36 					struct rt_mutex *lock,  in __ww_mutex_add_waiter()  argument
42 static inline void __ww_mutex_check_waiters(struct rt_mutex *lock,  in __ww_mutex_check_waiters()  argument
47 static inline void ww_mutex_lock_acquired(struct ww_mutex *lock,  in ww_mutex_lock_acquired()  argument
52 static inline int __ww_mutex_check_kill(struct rt_mutex *lock,  in __ww_mutex_check_kill()  argument
93 rt_mutex_owner_encode(struct rt_mutex_base *lock, struct task_struct *owner)  in rt_mutex_owner_encode()  argument
97 	if (rt_mutex_has_waiters(lock))  in rt_mutex_owner_encode()
104 rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner)  in rt_mutex_set_owner()  argument
110 	xchg_acquire(&lock->owner, rt_mutex_owner_encode(lock, owner));  in rt_mutex_set_owner()
113 static __always_inline void rt_mutex_clear_owner(struct rt_mutex_base *lock)  in rt_mutex_clear_owner()  argument
116 	WRITE_ONCE(lock->owner, rt_mutex_owner_encode(lock, NULL));  in rt_mutex_clear_owner()
119 static __always_inline void clear_rt_mutex_waiters(struct rt_mutex_base *lock)  in clear_rt_mutex_waiters()  argument
121 	lock->owner = (struct task_struct *)  in clear_rt_mutex_waiters()
122 			((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);  in clear_rt_mutex_waiters()
126 fixup_rt_mutex_waiters(struct rt_mutex_base *lock, bool acquire_lock)  in fixup_rt_mutex_waiters()  argument
128 	unsigned long owner, *p = (unsigned long *) &lock->owner;  in fixup_rt_mutex_waiters()
130 	if (rt_mutex_has_waiters(lock))  in fixup_rt_mutex_waiters()
214 static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_acquire()  argument
218 	return try_cmpxchg_acquire(&lock->owner, &old, new);  in rt_mutex_cmpxchg_acquire()
221 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_release()  argument
225 	return try_cmpxchg_release(&lock->owner, &old, new);  in rt_mutex_cmpxchg_release()
233 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)  in mark_rt_mutex_waiters()  argument
235 	unsigned long owner, *p = (unsigned long *) &lock->owner;  in mark_rt_mutex_waiters()
256 static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock,  in unlock_rt_mutex_safe()  argument
258 	__releases(lock->wait_lock)  in unlock_rt_mutex_safe()
260 	struct task_struct *owner = rt_mutex_owner(lock);  in unlock_rt_mutex_safe()
262 	clear_rt_mutex_waiters(lock);  in unlock_rt_mutex_safe()
263 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in unlock_rt_mutex_safe()
288 	return rt_mutex_cmpxchg_release(lock, owner, NULL);  in unlock_rt_mutex_safe()
292 static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_acquire()  argument
300 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_release()  argument
307 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)  in mark_rt_mutex_waiters()  argument
309 	lock->owner = (struct task_struct *)  in mark_rt_mutex_waiters()
310 			((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);  in mark_rt_mutex_waiters()
316 static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock,  in unlock_rt_mutex_safe()  argument
318 	__releases(lock->wait_lock)  in unlock_rt_mutex_safe()
320 	lock->owner = NULL;  in unlock_rt_mutex_safe()
321 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in unlock_rt_mutex_safe()
342 	lockdep_assert_held(&waiter->lock->wait_lock);  in waiter_update_prio()
355 	lockdep_assert_held(&waiter->lock->wait_lock);  in waiter_clone_prio()
457 rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue()  argument
459 	lockdep_assert_held(&lock->wait_lock);  in rt_mutex_enqueue()
461 	rb_add_cached(&waiter->tree.entry, &lock->waiters, __waiter_less);  in rt_mutex_enqueue()
465 rt_mutex_dequeue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue()  argument
467 	lockdep_assert_held(&lock->wait_lock);  in rt_mutex_dequeue()
472 	rb_erase_cached(&waiter->tree.entry, &lock->waiters);  in rt_mutex_dequeue()
504 static __always_inline void rt_mutex_adjust_prio(struct rt_mutex_base *lock,  in rt_mutex_adjust_prio()  argument
509 	lockdep_assert_held(&lock->wait_lock);  in rt_mutex_adjust_prio()
510 	lockdep_assert(rt_mutex_owner(lock) == p);  in rt_mutex_adjust_prio()
579 	return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;  in task_blocked_on_lock()
665 	struct rt_mutex_base *lock;  in rt_mutex_adjust_prio_chain()  local
744 	if (next_lock != waiter->lock)  in rt_mutex_adjust_prio_chain()
812 	lock = waiter->lock;  in rt_mutex_adjust_prio_chain()
822 	if (!raw_spin_trylock(&lock->wait_lock)) {  in rt_mutex_adjust_prio_chain()
837 	if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {  in rt_mutex_adjust_prio_chain()
852 		raw_spin_unlock(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
873 		if (!rt_mutex_owner(lock)) {  in rt_mutex_adjust_prio_chain()
874 			raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
879 		task = get_task_struct(rt_mutex_owner(lock));  in rt_mutex_adjust_prio_chain()
892 		top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
896 		raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
909 	prerequeue_top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
912 	rt_mutex_dequeue(lock, waiter);  in rt_mutex_adjust_prio_chain()
927 	rt_mutex_enqueue(lock, waiter);  in rt_mutex_adjust_prio_chain()
946 	if (!rt_mutex_owner(lock)) {  in rt_mutex_adjust_prio_chain()
952 		top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
955 		raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
965 	task = get_task_struct(rt_mutex_owner(lock));  in rt_mutex_adjust_prio_chain()
969 	if (waiter == rt_mutex_top_waiter(lock)) {  in rt_mutex_adjust_prio_chain()
979 		rt_mutex_adjust_prio(lock, task);  in rt_mutex_adjust_prio_chain()
993 		waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
996 		rt_mutex_adjust_prio(lock, task);  in rt_mutex_adjust_prio_chain()
1019 	top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
1023 	raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
1064 try_to_take_rt_mutex(struct rt_mutex_base *lock, struct task_struct *task,  in try_to_take_rt_mutex()  argument
1067 	lockdep_assert_held(&lock->wait_lock);  in try_to_take_rt_mutex()
1086 	mark_rt_mutex_waiters(lock);  in try_to_take_rt_mutex()
1091 	if (rt_mutex_owner(lock))  in try_to_take_rt_mutex()
1100 		struct rt_mutex_waiter *top_waiter = rt_mutex_top_waiter(lock);  in try_to_take_rt_mutex()
1111 			rt_mutex_dequeue(lock, waiter);  in try_to_take_rt_mutex()
1124 		if (rt_mutex_has_waiters(lock)) {  in try_to_take_rt_mutex()
1127 					    rt_mutex_top_waiter(lock)))  in try_to_take_rt_mutex()
1159 	if (rt_mutex_has_waiters(lock))  in try_to_take_rt_mutex()
1160 		rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));  in try_to_take_rt_mutex()
1168 	rt_mutex_set_owner(lock, task);  in try_to_take_rt_mutex()
1180 static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock,  in task_blocks_on_rt_mutex()  argument
1186 	struct task_struct *owner = rt_mutex_owner(lock);  in task_blocks_on_rt_mutex()
1191 	lockdep_assert_held(&lock->wait_lock);  in task_blocks_on_rt_mutex()
1210 	waiter->lock = lock;  in task_blocks_on_rt_mutex()
1215 	if (rt_mutex_has_waiters(lock))  in task_blocks_on_rt_mutex()
1216 		top_waiter = rt_mutex_top_waiter(lock);  in task_blocks_on_rt_mutex()
1217 	rt_mutex_enqueue(lock, waiter);  in task_blocks_on_rt_mutex()
1227 		rtm = container_of(lock, struct rt_mutex, rtmutex);  in task_blocks_on_rt_mutex()
1231 			rt_mutex_dequeue(lock, waiter);  in task_blocks_on_rt_mutex()
1242 	if (waiter == rt_mutex_top_waiter(lock)) {  in task_blocks_on_rt_mutex()
1246 		rt_mutex_adjust_prio(lock, owner);  in task_blocks_on_rt_mutex()
1272 	raw_spin_unlock_irq(&lock->wait_lock);  in task_blocks_on_rt_mutex()
1274 	res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,  in task_blocks_on_rt_mutex()
1277 	raw_spin_lock_irq(&lock->wait_lock);  in task_blocks_on_rt_mutex()
1289 					    struct rt_mutex_base *lock)  in mark_wakeup_next_waiter()  argument
1293 	lockdep_assert_held(&lock->wait_lock);  in mark_wakeup_next_waiter()
1297 	waiter = rt_mutex_top_waiter(lock);  in mark_wakeup_next_waiter()
1307 	rt_mutex_adjust_prio(lock, current);  in mark_wakeup_next_waiter()
1317 	lock->owner = (void *) RT_MUTEX_HAS_WAITERS;  in mark_wakeup_next_waiter()
1334 static int __sched __rt_mutex_slowtrylock(struct rt_mutex_base *lock)  in __rt_mutex_slowtrylock()  argument
1336 	int ret = try_to_take_rt_mutex(lock, current, NULL);  in __rt_mutex_slowtrylock()
1342 	fixup_rt_mutex_waiters(lock, true);  in __rt_mutex_slowtrylock()
1350 static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock)  in rt_mutex_slowtrylock()  argument
1360 	if (rt_mutex_owner(lock))  in rt_mutex_slowtrylock()
1367 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowtrylock()
1369 	ret = __rt_mutex_slowtrylock(lock);  in rt_mutex_slowtrylock()
1371 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowtrylock()
1376 static __always_inline int __rt_mutex_trylock(struct rt_mutex_base *lock)  in __rt_mutex_trylock()  argument
1378 	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))  in __rt_mutex_trylock()
1381 	return rt_mutex_slowtrylock(lock);  in __rt_mutex_trylock()
1387 static void __sched rt_mutex_slowunlock(struct rt_mutex_base *lock)  in rt_mutex_slowunlock()  argument
1393 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowunlock()
1395 	debug_rt_mutex_unlock(lock);  in rt_mutex_slowunlock()
1428 	while (!rt_mutex_has_waiters(lock)) {  in rt_mutex_slowunlock()
1430 		if (unlock_rt_mutex_safe(lock, flags) == true)  in rt_mutex_slowunlock()
1433 		raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowunlock()
1442 	mark_wakeup_next_waiter(&wqh, lock);  in rt_mutex_slowunlock()
1443 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowunlock()
1448 static __always_inline void __rt_mutex_unlock(struct rt_mutex_base *lock)  in __rt_mutex_unlock()  argument
1450 	if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))  in __rt_mutex_unlock()
1453 	rt_mutex_slowunlock(lock);  in __rt_mutex_unlock()
1457 static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,  in rtmutex_spin_on_owner()  argument
1466 		if (owner != rt_mutex_owner(lock))  in rtmutex_spin_on_owner()
1484 		    !rt_mutex_waiter_is_top_waiter(lock, waiter)) {  in rtmutex_spin_on_owner()
1494 static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,  in rtmutex_spin_on_owner()  argument
1515 static void __sched remove_waiter(struct rt_mutex_base *lock,  in remove_waiter()  argument
1518 	bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));  in remove_waiter()
1519 	struct task_struct *owner = rt_mutex_owner(lock);  in remove_waiter()
1522 	lockdep_assert_held(&lock->wait_lock);  in remove_waiter()
1525 	rt_mutex_dequeue(lock, waiter);  in remove_waiter()
1540 	if (rt_mutex_has_waiters(lock))  in remove_waiter()
1541 		rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock));  in remove_waiter()
1543 	rt_mutex_adjust_prio(lock, owner);  in remove_waiter()
1560 	raw_spin_unlock_irq(&lock->wait_lock);  in remove_waiter()
1562 	rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,  in remove_waiter()
1565 	raw_spin_lock_irq(&lock->wait_lock);  in remove_waiter()
1579 static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,  in rt_mutex_slowlock_block()  argument
1585 	struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex);  in rt_mutex_slowlock_block()
1591 		if (try_to_take_rt_mutex(lock, current, waiter))  in rt_mutex_slowlock_block()
1609 		if (waiter == rt_mutex_top_waiter(lock))  in rt_mutex_slowlock_block()
1610 			owner = rt_mutex_owner(lock);  in rt_mutex_slowlock_block()
1613 		raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_slowlock_block()
1615 		if (!owner || !rtmutex_spin_on_owner(lock, waiter, owner))  in rt_mutex_slowlock_block()
1618 		raw_spin_lock_irq(&lock->wait_lock);  in rt_mutex_slowlock_block()
1657 static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,  in __rt_mutex_slowlock()  argument
1663 	struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex);  in __rt_mutex_slowlock()
1667 	lockdep_assert_held(&lock->wait_lock);  in __rt_mutex_slowlock()
1670 	if (try_to_take_rt_mutex(lock, current, NULL)) {  in __rt_mutex_slowlock()
1680 	trace_contention_begin(lock, LCB_F_RT);  in __rt_mutex_slowlock()
1682 	ret = task_blocks_on_rt_mutex(lock, waiter, current, ww_ctx, chwalk);  in __rt_mutex_slowlock()
1684 		ret = rt_mutex_slowlock_block(lock, ww_ctx, state, NULL, waiter);  in __rt_mutex_slowlock()
1695 		remove_waiter(lock, waiter);  in __rt_mutex_slowlock()
1703 	fixup_rt_mutex_waiters(lock, true);  in __rt_mutex_slowlock()
1705 	trace_contention_end(lock, ret);  in __rt_mutex_slowlock()
1710 static inline int __rt_mutex_slowlock_locked(struct rt_mutex_base *lock,  in __rt_mutex_slowlock_locked()  argument
1720 	ret = __rt_mutex_slowlock(lock, ww_ctx, state, RT_MUTEX_MIN_CHAINWALK,  in __rt_mutex_slowlock_locked()
1733 static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock,  in rt_mutex_slowlock()  argument
1748 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowlock()
1749 	ret = __rt_mutex_slowlock_locked(lock, ww_ctx, state);  in rt_mutex_slowlock()
1750 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowlock()
1755 static __always_inline int __rt_mutex_lock(struct rt_mutex_base *lock,  in __rt_mutex_lock()  argument
1758 	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))  in __rt_mutex_lock()
1761 	return rt_mutex_slowlock(lock, NULL, state);  in __rt_mutex_lock()
1774 static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock)  in rtlock_slowlock_locked()  argument
1779 	lockdep_assert_held(&lock->wait_lock);  in rtlock_slowlock_locked()
1781 	if (try_to_take_rt_mutex(lock, current, NULL))  in rtlock_slowlock_locked()
1789 	trace_contention_begin(lock, LCB_F_RT);  in rtlock_slowlock_locked()
1791 	task_blocks_on_rt_mutex(lock, &waiter, current, NULL, RT_MUTEX_MIN_CHAINWALK);  in rtlock_slowlock_locked()
1795 		if (try_to_take_rt_mutex(lock, current, &waiter))  in rtlock_slowlock_locked()
1798 		if (&waiter == rt_mutex_top_waiter(lock))  in rtlock_slowlock_locked()
1799 			owner = rt_mutex_owner(lock);  in rtlock_slowlock_locked()
1802 		raw_spin_unlock_irq(&lock->wait_lock);  in rtlock_slowlock_locked()
1804 		if (!owner || !rtmutex_spin_on_owner(lock, &waiter, owner))  in rtlock_slowlock_locked()
1807 		raw_spin_lock_irq(&lock->wait_lock);  in rtlock_slowlock_locked()
1818 	fixup_rt_mutex_waiters(lock, true);  in rtlock_slowlock_locked()
1821 	trace_contention_end(lock, 0);  in rtlock_slowlock_locked()
1824 static __always_inline void __sched rtlock_slowlock(struct rt_mutex_base *lock)  in rtlock_slowlock()  argument
1828 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rtlock_slowlock()
1829 	rtlock_slowlock_locked(lock);  in rtlock_slowlock()
1830 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rtlock_slowlock()