139 		.cpu_base = &migration_cpu_base,
174 			raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
178 			raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
199 	return expires < new_base->cpu_base->expires_next;  in hrtimer_check_target()
253 		raw_spin_unlock(&base->cpu_base->lock);  in switch_hrtimer_base()
254 		raw_spin_lock(&new_base->cpu_base->lock);  in switch_hrtimer_base()
258 			raw_spin_unlock(&new_base->cpu_base->lock);  in switch_hrtimer_base()
259 			raw_spin_lock(&base->cpu_base->lock);  in switch_hrtimer_base()
284 	__acquires(&timer->base->cpu_base->lock)  in lock_hrtimer_base()
288 	raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);  in lock_hrtimer_base()
488 __next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)  in __next_base()  argument
498 	return &cpu_base->clock_base[idx];  in __next_base()
501 #define for_each_active_base(base, cpu_base, active)	\  argument
502 	while ((base = __next_base((cpu_base), &(active))))
504 static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,  in __hrtimer_next_event_base()  argument
512 	for_each_active_base(base, cpu_base, active) {  in __hrtimer_next_event_base()
535 				cpu_base->softirq_next_timer = timer;  in __hrtimer_next_event_base()
537 				cpu_base->next_timer = timer;  in __hrtimer_next_event_base()
571 __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)  in __hrtimer_get_next_event()  argument
577 	if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {  in __hrtimer_get_next_event()
578 		active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;  in __hrtimer_get_next_event()
579 		cpu_base->softirq_next_timer = NULL;  in __hrtimer_get_next_event()
580 		expires_next = __hrtimer_next_event_base(cpu_base, NULL,  in __hrtimer_get_next_event()
583 		next_timer = cpu_base->softirq_next_timer;  in __hrtimer_get_next_event()
587 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;  in __hrtimer_get_next_event()
588 		cpu_base->next_timer = next_timer;  in __hrtimer_get_next_event()
589 		expires_next = __hrtimer_next_event_base(cpu_base, NULL, active,  in __hrtimer_get_next_event()
596 static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)  in hrtimer_update_next_event()  argument
605 	if (!cpu_base->softirq_activated) {  in hrtimer_update_next_event()
606 		soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);  in hrtimer_update_next_event()
611 		cpu_base->softirq_expires_next = soft;  in hrtimer_update_next_event()
614 	expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD);  in hrtimer_update_next_event()
620 		cpu_base->next_timer = cpu_base->softirq_next_timer;  in hrtimer_update_next_event()
646 static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)  in __hrtimer_hres_active()  argument
649 		cpu_base->hres_active : 0;  in __hrtimer_hres_active()
657 static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,  in __hrtimer_reprogram()  argument
661 	cpu_base->expires_next = expires_next;  in __hrtimer_reprogram()
680 	if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)  in __hrtimer_reprogram()
692 hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)  in hrtimer_force_reprogram()  argument
696 	expires_next = hrtimer_update_next_event(cpu_base);  in hrtimer_force_reprogram()
698 	if (skip_equal && expires_next == cpu_base->expires_next)  in hrtimer_force_reprogram()
701 	__hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next);  in hrtimer_force_reprogram()
812 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_reprogram()  local
833 		struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;  in hrtimer_reprogram()
853 	if (base->cpu_base != cpu_base)  in hrtimer_reprogram()
856 	if (expires >= cpu_base->expires_next)  in hrtimer_reprogram()
863 	if (cpu_base->in_hrtirq)  in hrtimer_reprogram()
866 	cpu_base->next_timer = timer;  in hrtimer_reprogram()
868 	__hrtimer_reprogram(cpu_base, timer, expires);  in hrtimer_reprogram()
871 static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base,  in update_needs_ipi()  argument
888 	seq = cpu_base->clock_was_set_seq;  in update_needs_ipi()
889 	hrtimer_update_base(cpu_base);  in update_needs_ipi()
895 	if (seq == cpu_base->clock_was_set_seq)  in update_needs_ipi()
903 	if (cpu_base->in_hrtirq)  in update_needs_ipi()
912 	active &= cpu_base->active_bases;  in update_needs_ipi()
914 	for_each_active_base(base, cpu_base, active) {  in update_needs_ipi()
919 		if (expires < cpu_base->expires_next)  in update_needs_ipi()
925 		if (cpu_base->softirq_activated)  in update_needs_ipi()
927 		if (expires < cpu_base->softirq_expires_next)  in update_needs_ipi()
949 	struct hrtimer_cpu_base *cpu_base = raw_cpu_ptr(&hrtimer_bases);  in clock_was_set()  local
953 	if (!__hrtimer_hres_active(cpu_base) && !tick_nohz_active)  in clock_was_set()
966 		cpu_base = &per_cpu(hrtimer_bases, cpu);  in clock_was_set()
967 		raw_spin_lock_irqsave(&cpu_base->lock, flags);  in clock_was_set()
969 		if (update_needs_ipi(cpu_base, bases))  in clock_was_set()
972 		raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in clock_was_set()
1018 	__releases(&timer->base->cpu_base->lock)  in unlock_hrtimer_base()
1020 	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);  in unlock_hrtimer_base()
1088 	WARN_ON_ONCE(!base->cpu_base->online);  in enqueue_hrtimer()
1090 	base->cpu_base->active_bases |= 1 << base->index;  in enqueue_hrtimer()
1112 	struct hrtimer_cpu_base *cpu_base = base->cpu_base;  in __remove_hrtimer()  local
1121 		cpu_base->active_bases &= ~(1 << base->index);  in __remove_hrtimer()
1131 	if (reprogram && timer == cpu_base->next_timer)  in __remove_hrtimer()
1132 		hrtimer_force_reprogram(cpu_base, 1);  in __remove_hrtimer()
1156 		reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);  in remove_hrtimer()
1192 hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)  in hrtimer_update_softirq_timer()  argument
1199 	expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);  in hrtimer_update_softirq_timer()
1213 	hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);  in hrtimer_update_softirq_timer()
1231 	force_local = base->cpu_base == this_cpu_ptr(&hrtimer_bases);  in __hrtimer_start_range_ns()
1232 	force_local &= base->cpu_base->next_timer == timer;  in __hrtimer_start_range_ns()
1271 	hrtimer_force_reprogram(new_base->cpu_base, 1);  in __hrtimer_start_range_ns()
1370 static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base,  in hrtimer_sync_wait_running()  argument
1373 	if (atomic_read(&cpu_base->timer_waiters)) {  in hrtimer_sync_wait_running()
1374 		raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_sync_wait_running()
1375 		spin_unlock(&cpu_base->softirq_expiry_lock);  in hrtimer_sync_wait_running()
1376 		spin_lock(&cpu_base->softirq_expiry_lock);  in hrtimer_sync_wait_running()
1377 		raw_spin_lock_irq(&cpu_base->lock);  in hrtimer_sync_wait_running()
1418 	atomic_inc(&base->cpu_base->timer_waiters);  in hrtimer_cancel_wait_running()
1419 	spin_lock_bh(&base->cpu_base->softirq_expiry_lock);  in hrtimer_cancel_wait_running()
1420 	atomic_dec(&base->cpu_base->timer_waiters);  in hrtimer_cancel_wait_running()
1421 	spin_unlock_bh(&base->cpu_base->softirq_expiry_lock);  in hrtimer_cancel_wait_running()
1485 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_get_next_event()  local
1489 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_get_next_event()
1491 	if (!__hrtimer_hres_active(cpu_base))  in hrtimer_get_next_event()
1492 		expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);  in hrtimer_get_next_event()
1494 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_get_next_event()
1508 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_next_event_without()  local
1512 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_next_event_without()
1514 	if (__hrtimer_hres_active(cpu_base)) {  in hrtimer_next_event_without()
1517 		if (!cpu_base->softirq_activated) {  in hrtimer_next_event_without()
1518 			active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;  in hrtimer_next_event_without()
1519 			expires = __hrtimer_next_event_base(cpu_base, exclude,  in hrtimer_next_event_without()
1522 		active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;  in hrtimer_next_event_without()
1523 		expires = __hrtimer_next_event_base(cpu_base, exclude, active,  in hrtimer_next_event_without()
1527 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_next_event_without()
1549 	struct hrtimer_cpu_base *cpu_base;  in __hrtimer_init()  local
1563 	cpu_base = raw_cpu_ptr(&hrtimer_bases);  in __hrtimer_init()
1577 	timer->base = &cpu_base->clock_base[base];  in __hrtimer_init()
1646 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,  in __run_hrtimer()  argument
1649 			  unsigned long flags) __must_hold(&cpu_base->lock)  in __run_hrtimer()
1655 	lockdep_assert_held(&cpu_base->lock);  in __run_hrtimer()
1685 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in __run_hrtimer()
1693 	raw_spin_lock_irq(&cpu_base->lock);  in __run_hrtimer()
1721 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,  in __hrtimer_run_queues()  argument
1725 	unsigned int active = cpu_base->active_bases & active_mask;  in __hrtimer_run_queues()
1727 	for_each_active_base(base, cpu_base, active) {  in __hrtimer_run_queues()
1753 			__run_hrtimer(cpu_base, base, timer, &basenow, flags);  in __hrtimer_run_queues()
1755 				hrtimer_sync_wait_running(cpu_base, flags);  in __hrtimer_run_queues()
1762 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_run_softirq()  local
1766 	hrtimer_cpu_base_lock_expiry(cpu_base);  in hrtimer_run_softirq()
1767 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_run_softirq()
1769 	now = hrtimer_update_base(cpu_base);  in hrtimer_run_softirq()
1770 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);  in hrtimer_run_softirq()
1772 	cpu_base->softirq_activated = 0;  in hrtimer_run_softirq()
1773 	hrtimer_update_softirq_timer(cpu_base, true);  in hrtimer_run_softirq()
1775 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_run_softirq()
1776 	hrtimer_cpu_base_unlock_expiry(cpu_base);  in hrtimer_run_softirq()
1787 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_interrupt()  local
1792 	BUG_ON(!cpu_base->hres_active);  in hrtimer_interrupt()
1793 	cpu_base->nr_events++;  in hrtimer_interrupt()
1796 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_interrupt()
1797 	entry_time = now = hrtimer_update_base(cpu_base);  in hrtimer_interrupt()
1799 	cpu_base->in_hrtirq = 1;  in hrtimer_interrupt()
1807 	cpu_base->expires_next = KTIME_MAX;  in hrtimer_interrupt()
1809 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {  in hrtimer_interrupt()
1810 		cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimer_interrupt()
1811 		cpu_base->softirq_activated = 1;  in hrtimer_interrupt()
1815 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);  in hrtimer_interrupt()
1818 	expires_next = hrtimer_update_next_event(cpu_base);  in hrtimer_interrupt()
1823 	cpu_base->expires_next = expires_next;  in hrtimer_interrupt()
1824 	cpu_base->in_hrtirq = 0;  in hrtimer_interrupt()
1825 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_interrupt()
1829 		cpu_base->hang_detected = 0;  in hrtimer_interrupt()
1846 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_interrupt()
1847 	now = hrtimer_update_base(cpu_base);  in hrtimer_interrupt()
1848 	cpu_base->nr_retries++;  in hrtimer_interrupt()
1857 	cpu_base->nr_hangs++;  in hrtimer_interrupt()
1858 	cpu_base->hang_detected = 1;  in hrtimer_interrupt()
1859 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_interrupt()
1862 	if ((unsigned int)delta > cpu_base->max_hang_time)  in hrtimer_interrupt()
1863 		cpu_base->max_hang_time = (unsigned int) delta;  in hrtimer_interrupt()
1900 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);  in hrtimer_run_queues()  local
1904 	if (__hrtimer_hres_active(cpu_base))  in hrtimer_run_queues()
1919 	raw_spin_lock_irqsave(&cpu_base->lock, flags);  in hrtimer_run_queues()
1920 	now = hrtimer_update_base(cpu_base);  in hrtimer_run_queues()
1922 	if (!ktime_before(now, cpu_base->softirq_expires_next)) {  in hrtimer_run_queues()
1923 		cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimer_run_queues()
1924 		cpu_base->softirq_activated = 1;  in hrtimer_run_queues()
1928 	__hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);  in hrtimer_run_queues()
1929 	raw_spin_unlock_irqrestore(&cpu_base->lock, flags);  in hrtimer_run_queues()
2168 	struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);  in hrtimers_prepare_cpu()  local
2172 		struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i];  in hrtimers_prepare_cpu()
2174 		clock_b->cpu_base = cpu_base;  in hrtimers_prepare_cpu()
2175 		seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock);  in hrtimers_prepare_cpu()
2179 	cpu_base->cpu = cpu;  in hrtimers_prepare_cpu()
2180 	cpu_base->active_bases = 0;  in hrtimers_prepare_cpu()
2181 	cpu_base->hres_active = 0;  in hrtimers_prepare_cpu()
2182 	cpu_base->hang_detected = 0;  in hrtimers_prepare_cpu()
2183 	cpu_base->next_timer = NULL;  in hrtimers_prepare_cpu()
2184 	cpu_base->softirq_next_timer = NULL;  in hrtimers_prepare_cpu()
2185 	cpu_base->expires_next = KTIME_MAX;  in hrtimers_prepare_cpu()
2186 	cpu_base->softirq_expires_next = KTIME_MAX;  in hrtimers_prepare_cpu()
2187 	cpu_base->online = 1;  in hrtimers_prepare_cpu()
2188 	hrtimer_cpu_base_init_expiry_lock(cpu_base);  in hrtimers_prepare_cpu()