148 	.clock_base = { { .cpu_base = &migration_cpu_base, }, },
174 			raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
178 			raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
199 	return expires < new_base->cpu_base->expires_next;  in hrtimer_check_target()
253 		raw_spin_unlock(&base->cpu_base->lock);  in switch_hrtimer_base()
254 		raw_spin_lock(&new_base->cpu_base->lock);  in switch_hrtimer_base()
258 			raw_spin_unlock(&new_base->cpu_base->lock);  in switch_hrtimer_base()
259 			raw_spin_lock(&base->cpu_base->lock);  in switch_hrtimer_base()
282 	raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
476 __next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)  in __next_base()  argument
486 	return &cpu_base->clock_base[idx];  in __next_base()
489 #define for_each_active_base(base, cpu_base, active)	\  argument
490 	while ((base = __next_base((cpu_base), &(active))))
492 static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,  in __hrtimer_next_event_base()  argument
500 	for_each_active_base(base, cpu_base, active) {  in __hrtimer_next_event_base()
523 				cpu_base->softirq_next_timer = timer;  in __hrtimer_next_event_base()
525 				cpu_base->next_timer = timer;  in __hrtimer_next_event_base()
556 __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)  in __hrtimer_get_next_event()  argument
562 	if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {  in __hrtimer_get_next_event()
563 		active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;  in __hrtimer_get_next_event()
564 		cpu_base->softirq_next_timer = NULL;  in __hrtimer_get_next_event()
565 		expires_next = __hrtimer_next_event_base(cpu_base, NULL,  in __hrtimer_get_next_event()
568 		next_timer = cpu_base->softirq_next_timer;  in __hrtimer_get_next_event()
572 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;  in __hrtimer_get_next_event()
573 		cpu_base->next_timer = next_timer;  in __hrtimer_get_next_event()
574 		expires_next = __hrtimer_next_event_base(cpu_base, NULL, active,  in __hrtimer_get_next_event()
600 static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)  in __hrtimer_hres_active()  argument
603 		cpu_base->hres_active : 0;  in __hrtimer_hres_active()
617 hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)  in hrtimer_force_reprogram()  argument
624 	expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);  in hrtimer_force_reprogram()
626 	if (cpu_base->next_timer && cpu_base->next_timer->is_soft) {  in hrtimer_force_reprogram()
632 		if (cpu_base->softirq_activated)  in hrtimer_force_reprogram()
633 			expires_next = __hrtimer_get_next_event(cpu_base,  in hrtimer_force_reprogram()
636 			cpu_base->softirq_expires_next = expires_next;  in hrtimer_force_reprogram()
639 	if (skip_equal && expires_next == cpu_base->expires_next)  in hrtimer_force_reprogram()
642 	cpu_base->expires_next = expires_next;  in hrtimer_force_reprogram()
661 	if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)  in hrtimer_force_reprogram()
664 	tick_program_event(cpu_base->expires_next, 1);  in hrtimer_force_reprogram()
766 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_reprogram()  local
787 		struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;  in hrtimer_reprogram()
807 	if (base->cpu_base != cpu_base)  in hrtimer_reprogram()
817 	if (cpu_base->in_hrtirq)  in hrtimer_reprogram()
820 	if (expires >= cpu_base->expires_next)  in hrtimer_reprogram()
824 	cpu_base->next_timer = timer;  in hrtimer_reprogram()
825 	cpu_base->expires_next = expires;  in hrtimer_reprogram()
836 	if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)  in hrtimer_reprogram()
887 	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);  in unlock_hrtimer_base()
956 	base->cpu_base->active_bases |= 1 << base->index;  in enqueue_hrtimer()
977 	struct hrtimer_cpu_base *cpu_base = base->cpu_base;  in __remove_hrtimer()  local
985 		cpu_base->active_bases &= ~(1 << base->index);  in __remove_hrtimer()
995 	if (reprogram && timer == cpu_base->next_timer)  in __remove_hrtimer()
996 		hrtimer_force_reprogram(cpu_base, 1);  in __remove_hrtimer()
1018 		reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);  in remove_hrtimer()
1046 hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)  in hrtimer_update_softirq_timer()  argument
1053 	expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);  in hrtimer_update_softirq_timer()
1067 	hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);  in hrtimer_update_softirq_timer()
1208 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_get_next_event()  local
1212 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_get_next_event()
1214 	if (!__hrtimer_hres_active(cpu_base))  in hrtimer_get_next_event()
1215 		expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);  in hrtimer_get_next_event()
1217 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_get_next_event()
1231 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_next_event_without()  local
1235 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_next_event_without()
1237 	if (__hrtimer_hres_active(cpu_base)) {  in hrtimer_next_event_without()
1240 		if (!cpu_base->softirq_activated) {  in hrtimer_next_event_without()
1241 			active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;  in hrtimer_next_event_without()
1242 			expires = __hrtimer_next_event_base(cpu_base, exclude,  in hrtimer_next_event_without()
1245 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;  in hrtimer_next_event_without()
1246 		expires = __hrtimer_next_event_base(cpu_base, exclude, active,  in hrtimer_next_event_without()
1250 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_next_event_without()
1273 	struct hrtimer_cpu_base *cpu_base;  in __hrtimer_init()  local
1277 	cpu_base = raw_cpu_ptr(&hrtimer_bases);  in __hrtimer_init()
1289 	timer->base = &cpu_base->clock_base[base];  in __hrtimer_init()
1358 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,  in __run_hrtimer()  argument
1366 	lockdep_assert_held(&cpu_base->lock);  in __run_hrtimer()
1396 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in __run_hrtimer()
1400 	raw_spin_lock_irq(&cpu_base->lock);  in __run_hrtimer()
1428 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,  in __hrtimer_run_queues()  argument
1432 	unsigned int active = cpu_base->active_bases & active_mask;  in __hrtimer_run_queues()
1434 	for_each_active_base(base, cpu_base, active) {  in __hrtimer_run_queues()
1460 			__run_hrtimer(cpu_base, base, timer, &basenow, flags);  in __hrtimer_run_queues()
1467 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_run_softirq()  local
1471 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_run_softirq()
1473 	now = hrtimer_update_base(cpu_base);  in hrtimer_run_softirq()
1474 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);  in hrtimer_run_softirq()
1476 	cpu_base->softirq_activated = 0;  in hrtimer_run_softirq()
1477 	hrtimer_update_softirq_timer(cpu_base, true);  in hrtimer_run_softirq()
1479 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_run_softirq()
1490 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_interrupt()  local
1495 	BUG_ON(!cpu_base->hres_active);  in hrtimer_interrupt()
1496 	cpu_base->nr_events++;  in hrtimer_interrupt()
1499 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_interrupt()
1500 	entry_time = now = hrtimer_update_base(cpu_base);  in hrtimer_interrupt()
1502 	cpu_base->in_hrtirq = 1;  in hrtimer_interrupt()
1510 	cpu_base->expires_next = KTIME_MAX;  in hrtimer_interrupt()
1512 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {  in hrtimer_interrupt()
1513 		cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimer_interrupt()
1514 		cpu_base->softirq_activated = 1;  in hrtimer_interrupt()
1518 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);  in hrtimer_interrupt()
1521 	expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);  in hrtimer_interrupt()
1526 	cpu_base->expires_next = expires_next;  in hrtimer_interrupt()
1527 	cpu_base->in_hrtirq = 0;  in hrtimer_interrupt()
1528 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_interrupt()
1532 		cpu_base->hang_detected = 0;  in hrtimer_interrupt()
1549 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_interrupt()
1550 	now = hrtimer_update_base(cpu_base);  in hrtimer_interrupt()
1551 	cpu_base->nr_retries++;  in hrtimer_interrupt()
1560 	cpu_base->nr_hangs++;  in hrtimer_interrupt()
1561 	cpu_base->hang_detected = 1;  in hrtimer_interrupt()
1562 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_interrupt()
1565 	if ((unsigned int)delta > cpu_base->max_hang_time)  in hrtimer_interrupt()
1566 		cpu_base->max_hang_time = (unsigned int) delta;  in hrtimer_interrupt()
1603 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_run_queues()  local
1607 	if (__hrtimer_hres_active(cpu_base))  in hrtimer_run_queues()
1622 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_run_queues()
1623 	now = hrtimer_update_base(cpu_base);  in hrtimer_run_queues()
1625 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {  in hrtimer_run_queues()
1626 		cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimer_run_queues()
1627 		cpu_base->softirq_activated = 1;  in hrtimer_run_queues()
1631 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);  in hrtimer_run_queues()
1632 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_run_queues()
1805 	struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);  in hrtimers_prepare_cpu()  local
1809 		cpu_base->clock_base[i].cpu_base = cpu_base;  in hrtimers_prepare_cpu()
1810 		timerqueue_init_head(&cpu_base->clock_base[i].active);  in hrtimers_prepare_cpu()
1813 	cpu_base->cpu = cpu;  in hrtimers_prepare_cpu()
1814 	cpu_base->active_bases = 0;  in hrtimers_prepare_cpu()
1815 	cpu_base->hres_active = 0;  in hrtimers_prepare_cpu()
1816 	cpu_base->hang_detected = 0;  in hrtimers_prepare_cpu()
1817 	cpu_base->next_timer = NULL;  in hrtimers_prepare_cpu()
1818 	cpu_base->softirq_next_timer = NULL;  in hrtimers_prepare_cpu()
1819 	cpu_base->expires_next = KTIME_MAX;  in hrtimers_prepare_cpu()
1820 	cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimers_prepare_cpu()