1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * RT-Mutexes: simple blocking mutual exclusion locks with PI support
4  *
5  * started by Ingo Molnar and Thomas Gleixner.
6  *
7  *  Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
8  *  Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
9  *  Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
10  *  Copyright (C) 2006 Esben Nielsen
11  * Adaptive Spinlocks:
12  *  Copyright (C) 2008 Novell, Inc., Gregory Haskins, Sven Dietrich,
13  *				     and Peter Morreale,
14  * Adaptive Spinlocks simplification:
15  *  Copyright (C) 2008 Red Hat, Inc., Steven Rostedt <srostedt@redhat.com>
16  *
17  *  See Documentation/locking/rt-mutex-design.rst for details.
18  */
19 #include <linux/sched.h>
20 #include <linux/sched/debug.h>
21 #include <linux/sched/deadline.h>
22 #include <linux/sched/signal.h>
23 #include <linux/sched/rt.h>
24 #include <linux/sched/wake_q.h>
25 #include <linux/ww_mutex.h>
26 
27 #include <trace/events/lock.h>
28 
29 #include "rtmutex_common.h"
30 
31 #ifndef WW_RT
32 # define build_ww_mutex()	(false)
33 # define ww_container_of(rtm)	NULL
34 
__ww_mutex_add_waiter(struct rt_mutex_waiter * waiter,struct rt_mutex * lock,struct ww_acquire_ctx * ww_ctx)35 static inline int __ww_mutex_add_waiter(struct rt_mutex_waiter *waiter,
36 					struct rt_mutex *lock,
37 					struct ww_acquire_ctx *ww_ctx)
38 {
39 	return 0;
40 }
41 
__ww_mutex_check_waiters(struct rt_mutex * lock,struct ww_acquire_ctx * ww_ctx)42 static inline void __ww_mutex_check_waiters(struct rt_mutex *lock,
43 					    struct ww_acquire_ctx *ww_ctx)
44 {
45 }
46 
ww_mutex_lock_acquired(struct ww_mutex * lock,struct ww_acquire_ctx * ww_ctx)47 static inline void ww_mutex_lock_acquired(struct ww_mutex *lock,
48 					  struct ww_acquire_ctx *ww_ctx)
49 {
50 }
51 
__ww_mutex_check_kill(struct rt_mutex * lock,struct rt_mutex_waiter * waiter,struct ww_acquire_ctx * ww_ctx)52 static inline int __ww_mutex_check_kill(struct rt_mutex *lock,
53 					struct rt_mutex_waiter *waiter,
54 					struct ww_acquire_ctx *ww_ctx)
55 {
56 	return 0;
57 }
58 
59 #else
60 # define build_ww_mutex()	(true)
61 # define ww_container_of(rtm)	container_of(rtm, struct ww_mutex, base)
62 # include "ww_mutex.h"
63 #endif
64 
65 /*
66  * lock->owner state tracking:
67  *
68  * lock->owner holds the task_struct pointer of the owner. Bit 0
69  * is used to keep track of the "lock has waiters" state.
70  *
71  * owner	bit0
72  * NULL		0	lock is free (fast acquire possible)
73  * NULL		1	lock is free and has waiters and the top waiter
74  *				is going to take the lock*
75  * taskpointer	0	lock is held (fast release possible)
76  * taskpointer	1	lock is held and has waiters**
77  *
78  * The fast atomic compare exchange based acquire and release is only
79  * possible when bit 0 of lock->owner is 0.
80  *
81  * (*) It also can be a transitional state when grabbing the lock
82  * with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
83  * we need to set the bit0 before looking at the lock, and the owner may be
84  * NULL in this small time, hence this can be a transitional state.
85  *
86  * (**) There is a small time when bit 0 is set but there are no
87  * waiters. This can happen when grabbing the lock in the slow path.
88  * To prevent a cmpxchg of the owner releasing the lock, we need to
89  * set this bit before looking at the lock.
90  */
91 
92 static __always_inline struct task_struct *
rt_mutex_owner_encode(struct rt_mutex_base * lock,struct task_struct * owner)93 rt_mutex_owner_encode(struct rt_mutex_base *lock, struct task_struct *owner)
94 {
95 	unsigned long val = (unsigned long)owner;
96 
97 	if (rt_mutex_has_waiters(lock))
98 		val |= RT_MUTEX_HAS_WAITERS;
99 
100 	return (struct task_struct *)val;
101 }
102 
103 static __always_inline void
rt_mutex_set_owner(struct rt_mutex_base * lock,struct task_struct * owner)104 rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner)
105 {
106 	/*
107 	 * lock->wait_lock is held but explicit acquire semantics are needed
108 	 * for a new lock owner so WRITE_ONCE is insufficient.
109 	 */
110 	xchg_acquire(&lock->owner, rt_mutex_owner_encode(lock, owner));
111 }
112 
rt_mutex_clear_owner(struct rt_mutex_base * lock)113 static __always_inline void rt_mutex_clear_owner(struct rt_mutex_base *lock)
114 {
115 	/* lock->wait_lock is held so the unlock provides release semantics. */
116 	WRITE_ONCE(lock->owner, rt_mutex_owner_encode(lock, NULL));
117 }
118 
clear_rt_mutex_waiters(struct rt_mutex_base * lock)119 static __always_inline void clear_rt_mutex_waiters(struct rt_mutex_base *lock)
120 {
121 	lock->owner = (struct task_struct *)
122 			((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
123 }
124 
125 static __always_inline void
fixup_rt_mutex_waiters(struct rt_mutex_base * lock,bool acquire_lock)126 fixup_rt_mutex_waiters(struct rt_mutex_base *lock, bool acquire_lock)
127 {
128 	unsigned long owner, *p = (unsigned long *) &lock->owner;
129 
130 	if (rt_mutex_has_waiters(lock))
131 		return;
132 
133 	/*
134 	 * The rbtree has no waiters enqueued, now make sure that the
135 	 * lock->owner still has the waiters bit set, otherwise the
136 	 * following can happen:
137 	 *
138 	 * CPU 0	CPU 1		CPU2
139 	 * l->owner=T1
140 	 *		rt_mutex_lock(l)
141 	 *		lock(l->lock)
142 	 *		l->owner = T1 | HAS_WAITERS;
143 	 *		enqueue(T2)
144 	 *		boost()
145 	 *		  unlock(l->lock)
146 	 *		block()
147 	 *
148 	 *				rt_mutex_lock(l)
149 	 *				lock(l->lock)
150 	 *				l->owner = T1 | HAS_WAITERS;
151 	 *				enqueue(T3)
152 	 *				boost()
153 	 *				  unlock(l->lock)
154 	 *				block()
155 	 *		signal(->T2)	signal(->T3)
156 	 *		lock(l->lock)
157 	 *		dequeue(T2)
158 	 *		deboost()
159 	 *		  unlock(l->lock)
160 	 *				lock(l->lock)
161 	 *				dequeue(T3)
162 	 *				 ==> wait list is empty
163 	 *				deboost()
164 	 *				 unlock(l->lock)
165 	 *		lock(l->lock)
166 	 *		fixup_rt_mutex_waiters()
167 	 *		  if (wait_list_empty(l) {
168 	 *		    l->owner = owner
169 	 *		    owner = l->owner & ~HAS_WAITERS;
170 	 *		      ==> l->owner = T1
171 	 *		  }
172 	 *				lock(l->lock)
173 	 * rt_mutex_unlock(l)		fixup_rt_mutex_waiters()
174 	 *				  if (wait_list_empty(l) {
175 	 *				    owner = l->owner & ~HAS_WAITERS;
176 	 * cmpxchg(l->owner, T1, NULL)
177 	 *  ===> Success (l->owner = NULL)
178 	 *
179 	 *				    l->owner = owner
180 	 *				      ==> l->owner = T1
181 	 *				  }
182 	 *
183 	 * With the check for the waiter bit in place T3 on CPU2 will not
184 	 * overwrite. All tasks fiddling with the waiters bit are
185 	 * serialized by l->lock, so nothing else can modify the waiters
186 	 * bit. If the bit is set then nothing can change l->owner either
187 	 * so the simple RMW is safe. The cmpxchg() will simply fail if it
188 	 * happens in the middle of the RMW because the waiters bit is
189 	 * still set.
190 	 */
191 	owner = READ_ONCE(*p);
192 	if (owner & RT_MUTEX_HAS_WAITERS) {
193 		/*
194 		 * See rt_mutex_set_owner() and rt_mutex_clear_owner() on
195 		 * why xchg_acquire() is used for updating owner for
196 		 * locking and WRITE_ONCE() for unlocking.
197 		 *
198 		 * WRITE_ONCE() would work for the acquire case too, but
199 		 * in case that the lock acquisition failed it might
200 		 * force other lockers into the slow path unnecessarily.
201 		 */
202 		if (acquire_lock)
203 			xchg_acquire(p, owner & ~RT_MUTEX_HAS_WAITERS);
204 		else
205 			WRITE_ONCE(*p, owner & ~RT_MUTEX_HAS_WAITERS);
206 	}
207 }
208 
209 /*
210  * We can speed up the acquire/release, if there's no debugging state to be
211  * set up.
212  */
213 #ifndef CONFIG_DEBUG_RT_MUTEXES
rt_mutex_cmpxchg_acquire(struct rt_mutex_base * lock,struct task_struct * old,struct task_struct * new)214 static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,
215 						     struct task_struct *old,
216 						     struct task_struct *new)
217 {
218 	return try_cmpxchg_acquire(&lock->owner, &old, new);
219 }
220 
rt_mutex_cmpxchg_release(struct rt_mutex_base * lock,struct task_struct * old,struct task_struct * new)221 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,
222 						     struct task_struct *old,
223 						     struct task_struct *new)
224 {
225 	return try_cmpxchg_release(&lock->owner, &old, new);
226 }
227 
228 /*
229  * Callers must hold the ->wait_lock -- which is the whole purpose as we force
230  * all future threads that attempt to [Rmw] the lock to the slowpath. As such
231  * relaxed semantics suffice.
232  */
mark_rt_mutex_waiters(struct rt_mutex_base * lock)233 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)
234 {
235 	unsigned long owner, *p = (unsigned long *) &lock->owner;
236 
237 	do {
238 		owner = *p;
239 	} while (cmpxchg_relaxed(p, owner,
240 				 owner | RT_MUTEX_HAS_WAITERS) != owner);
241 
242 	/*
243 	 * The cmpxchg loop above is relaxed to avoid back-to-back ACQUIRE
244 	 * operations in the event of contention. Ensure the successful
245 	 * cmpxchg is visible.
246 	 */
247 	smp_mb__after_atomic();
248 }
249 
250 /*
251  * Safe fastpath aware unlock:
252  * 1) Clear the waiters bit
253  * 2) Drop lock->wait_lock
254  * 3) Try to unlock the lock with cmpxchg
255  */
unlock_rt_mutex_safe(struct rt_mutex_base * lock,unsigned long flags)256 static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock,
257 						 unsigned long flags)
258 	__releases(lock->wait_lock)
259 {
260 	struct task_struct *owner = rt_mutex_owner(lock);
261 
262 	clear_rt_mutex_waiters(lock);
263 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
264 	/*
265 	 * If a new waiter comes in between the unlock and the cmpxchg
266 	 * we have two situations:
267 	 *
268 	 * unlock(wait_lock);
269 	 *					lock(wait_lock);
270 	 * cmpxchg(p, owner, 0) == owner
271 	 *					mark_rt_mutex_waiters(lock);
272 	 *					acquire(lock);
273 	 * or:
274 	 *
275 	 * unlock(wait_lock);
276 	 *					lock(wait_lock);
277 	 *					mark_rt_mutex_waiters(lock);
278 	 *
279 	 * cmpxchg(p, owner, 0) != owner
280 	 *					enqueue_waiter();
281 	 *					unlock(wait_lock);
282 	 * lock(wait_lock);
283 	 * wake waiter();
284 	 * unlock(wait_lock);
285 	 *					lock(wait_lock);
286 	 *					acquire(lock);
287 	 */
288 	return rt_mutex_cmpxchg_release(lock, owner, NULL);
289 }
290 
291 #else
rt_mutex_cmpxchg_acquire(struct rt_mutex_base * lock,struct task_struct * old,struct task_struct * new)292 static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,
293 						     struct task_struct *old,
294 						     struct task_struct *new)
295 {
296 	return false;
297 
298 }
299 
rt_mutex_cmpxchg_release(struct rt_mutex_base * lock,struct task_struct * old,struct task_struct * new)300 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,
301 						     struct task_struct *old,
302 						     struct task_struct *new)
303 {
304 	return false;
305 }
306 
mark_rt_mutex_waiters(struct rt_mutex_base * lock)307 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)
308 {
309 	lock->owner = (struct task_struct *)
310 			((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
311 }
312 
313 /*
314  * Simple slow path only version: lock->owner is protected by lock->wait_lock.
315  */
unlock_rt_mutex_safe(struct rt_mutex_base * lock,unsigned long flags)316 static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock,
317 						 unsigned long flags)
318 	__releases(lock->wait_lock)
319 {
320 	lock->owner = NULL;
321 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
322 	return true;
323 }
324 #endif
325 
__waiter_prio(struct task_struct * task)326 static __always_inline int __waiter_prio(struct task_struct *task)
327 {
328 	int prio = task->prio;
329 
330 	if (!rt_prio(prio))
331 		return DEFAULT_PRIO;
332 
333 	return prio;
334 }
335 
336 static __always_inline void
waiter_update_prio(struct rt_mutex_waiter * waiter,struct task_struct * task)337 waiter_update_prio(struct rt_mutex_waiter *waiter, struct task_struct *task)
338 {
339 	waiter->prio = __waiter_prio(task);
340 	waiter->deadline = task->dl.deadline;
341 }
342 
343 /*
344  * Only use with rt_mutex_waiter_{less,equal}()
345  */
346 #define task_to_waiter(p)	\
347 	&(struct rt_mutex_waiter){ .prio = __waiter_prio(p), .deadline = (p)->dl.deadline }
348 
rt_mutex_waiter_less(struct rt_mutex_waiter * left,struct rt_mutex_waiter * right)349 static __always_inline int rt_mutex_waiter_less(struct rt_mutex_waiter *left,
350 						struct rt_mutex_waiter *right)
351 {
352 	if (left->prio < right->prio)
353 		return 1;
354 
355 	/*
356 	 * If both waiters have dl_prio(), we check the deadlines of the
357 	 * associated tasks.
358 	 * If left waiter has a dl_prio(), and we didn't return 1 above,
359 	 * then right waiter has a dl_prio() too.
360 	 */
361 	if (dl_prio(left->prio))
362 		return dl_time_before(left->deadline, right->deadline);
363 
364 	return 0;
365 }
366 
rt_mutex_waiter_equal(struct rt_mutex_waiter * left,struct rt_mutex_waiter * right)367 static __always_inline int rt_mutex_waiter_equal(struct rt_mutex_waiter *left,
368 						 struct rt_mutex_waiter *right)
369 {
370 	if (left->prio != right->prio)
371 		return 0;
372 
373 	/*
374 	 * If both waiters have dl_prio(), we check the deadlines of the
375 	 * associated tasks.
376 	 * If left waiter has a dl_prio(), and we didn't return 0 above,
377 	 * then right waiter has a dl_prio() too.
378 	 */
379 	if (dl_prio(left->prio))
380 		return left->deadline == right->deadline;
381 
382 	return 1;
383 }
384 
rt_mutex_steal(struct rt_mutex_waiter * waiter,struct rt_mutex_waiter * top_waiter)385 static inline bool rt_mutex_steal(struct rt_mutex_waiter *waiter,
386 				  struct rt_mutex_waiter *top_waiter)
387 {
388 	if (rt_mutex_waiter_less(waiter, top_waiter))
389 		return true;
390 
391 #ifdef RT_MUTEX_BUILD_SPINLOCKS
392 	/*
393 	 * Note that RT tasks are excluded from same priority (lateral)
394 	 * steals to prevent the introduction of an unbounded latency.
395 	 */
396 	if (rt_prio(waiter->prio) || dl_prio(waiter->prio))
397 		return false;
398 
399 	return rt_mutex_waiter_equal(waiter, top_waiter);
400 #else
401 	return false;
402 #endif
403 }
404 
405 #define __node_2_waiter(node) \
406 	rb_entry((node), struct rt_mutex_waiter, tree_entry)
407 
__waiter_less(struct rb_node * a,const struct rb_node * b)408 static __always_inline bool __waiter_less(struct rb_node *a, const struct rb_node *b)
409 {
410 	struct rt_mutex_waiter *aw = __node_2_waiter(a);
411 	struct rt_mutex_waiter *bw = __node_2_waiter(b);
412 
413 	if (rt_mutex_waiter_less(aw, bw))
414 		return 1;
415 
416 	if (!build_ww_mutex())
417 		return 0;
418 
419 	if (rt_mutex_waiter_less(bw, aw))
420 		return 0;
421 
422 	/* NOTE: relies on waiter->ww_ctx being set before insertion */
423 	if (aw->ww_ctx) {
424 		if (!bw->ww_ctx)
425 			return 1;
426 
427 		return (signed long)(aw->ww_ctx->stamp -
428 				     bw->ww_ctx->stamp) < 0;
429 	}
430 
431 	return 0;
432 }
433 
434 static __always_inline void
rt_mutex_enqueue(struct rt_mutex_base * lock,struct rt_mutex_waiter * waiter)435 rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)
436 {
437 	rb_add_cached(&waiter->tree_entry, &lock->waiters, __waiter_less);
438 }
439 
440 static __always_inline void
rt_mutex_dequeue(struct rt_mutex_base * lock,struct rt_mutex_waiter * waiter)441 rt_mutex_dequeue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)
442 {
443 	if (RB_EMPTY_NODE(&waiter->tree_entry))
444 		return;
445 
446 	rb_erase_cached(&waiter->tree_entry, &lock->waiters);
447 	RB_CLEAR_NODE(&waiter->tree_entry);
448 }
449 
450 #define __node_2_pi_waiter(node) \
451 	rb_entry((node), struct rt_mutex_waiter, pi_tree_entry)
452 
453 static __always_inline bool
__pi_waiter_less(struct rb_node * a,const struct rb_node * b)454 __pi_waiter_less(struct rb_node *a, const struct rb_node *b)
455 {
456 	return rt_mutex_waiter_less(__node_2_pi_waiter(a), __node_2_pi_waiter(b));
457 }
458 
459 static __always_inline void
rt_mutex_enqueue_pi(struct task_struct * task,struct rt_mutex_waiter * waiter)460 rt_mutex_enqueue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)
461 {
462 	rb_add_cached(&waiter->pi_tree_entry, &task->pi_waiters, __pi_waiter_less);
463 }
464 
465 static __always_inline void
rt_mutex_dequeue_pi(struct task_struct * task,struct rt_mutex_waiter * waiter)466 rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)
467 {
468 	if (RB_EMPTY_NODE(&waiter->pi_tree_entry))
469 		return;
470 
471 	rb_erase_cached(&waiter->pi_tree_entry, &task->pi_waiters);
472 	RB_CLEAR_NODE(&waiter->pi_tree_entry);
473 }
474 
rt_mutex_adjust_prio(struct task_struct * p)475 static __always_inline void rt_mutex_adjust_prio(struct task_struct *p)
476 {
477 	struct task_struct *pi_task = NULL;
478 
479 	lockdep_assert_held(&p->pi_lock);
480 
481 	if (task_has_pi_waiters(p))
482 		pi_task = task_top_pi_waiter(p)->task;
483 
484 	rt_mutex_setprio(p, pi_task);
485 }
486 
487 /* RT mutex specific wake_q wrappers */
rt_mutex_wake_q_add_task(struct rt_wake_q_head * wqh,struct task_struct * task,unsigned int wake_state)488 static __always_inline void rt_mutex_wake_q_add_task(struct rt_wake_q_head *wqh,
489 						     struct task_struct *task,
490 						     unsigned int wake_state)
491 {
492 	if (IS_ENABLED(CONFIG_PREEMPT_RT) && wake_state == TASK_RTLOCK_WAIT) {
493 		if (IS_ENABLED(CONFIG_PROVE_LOCKING))
494 			WARN_ON_ONCE(wqh->rtlock_task);
495 		get_task_struct(task);
496 		wqh->rtlock_task = task;
497 	} else {
498 		wake_q_add(&wqh->head, task);
499 	}
500 }
501 
rt_mutex_wake_q_add(struct rt_wake_q_head * wqh,struct rt_mutex_waiter * w)502 static __always_inline void rt_mutex_wake_q_add(struct rt_wake_q_head *wqh,
503 						struct rt_mutex_waiter *w)
504 {
505 	rt_mutex_wake_q_add_task(wqh, w->task, w->wake_state);
506 }
507 
rt_mutex_wake_up_q(struct rt_wake_q_head * wqh)508 static __always_inline void rt_mutex_wake_up_q(struct rt_wake_q_head *wqh)
509 {
510 	if (IS_ENABLED(CONFIG_PREEMPT_RT) && wqh->rtlock_task) {
511 		wake_up_state(wqh->rtlock_task, TASK_RTLOCK_WAIT);
512 		put_task_struct(wqh->rtlock_task);
513 		wqh->rtlock_task = NULL;
514 	}
515 
516 	if (!wake_q_empty(&wqh->head))
517 		wake_up_q(&wqh->head);
518 
519 	/* Pairs with preempt_disable() in mark_wakeup_next_waiter() */
520 	preempt_enable();
521 }
522 
523 /*
524  * Deadlock detection is conditional:
525  *
526  * If CONFIG_DEBUG_RT_MUTEXES=n, deadlock detection is only conducted
527  * if the detect argument is == RT_MUTEX_FULL_CHAINWALK.
528  *
529  * If CONFIG_DEBUG_RT_MUTEXES=y, deadlock detection is always
530  * conducted independent of the detect argument.
531  *
532  * If the waiter argument is NULL this indicates the deboost path and
533  * deadlock detection is disabled independent of the detect argument
534  * and the config settings.
535  */
536 static __always_inline bool
rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter * waiter,enum rtmutex_chainwalk chwalk)537 rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter,
538 			      enum rtmutex_chainwalk chwalk)
539 {
540 	if (IS_ENABLED(CONFIG_DEBUG_RT_MUTEXES))
541 		return waiter != NULL;
542 	return chwalk == RT_MUTEX_FULL_CHAINWALK;
543 }
544 
task_blocked_on_lock(struct task_struct * p)545 static __always_inline struct rt_mutex_base *task_blocked_on_lock(struct task_struct *p)
546 {
547 	return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;
548 }
549 
550 /*
551  * Adjust the priority chain. Also used for deadlock detection.
552  * Decreases task's usage by one - may thus free the task.
553  *
554  * @task:	the task owning the mutex (owner) for which a chain walk is
555  *		probably needed
556  * @chwalk:	do we have to carry out deadlock detection?
557  * @orig_lock:	the mutex (can be NULL if we are walking the chain to recheck
558  *		things for a task that has just got its priority adjusted, and
559  *		is waiting on a mutex)
560  * @next_lock:	the mutex on which the owner of @orig_lock was blocked before
561  *		we dropped its pi_lock. Is never dereferenced, only used for
562  *		comparison to detect lock chain changes.
563  * @orig_waiter: rt_mutex_waiter struct for the task that has just donated
564  *		its priority to the mutex owner (can be NULL in the case
565  *		depicted above or if the top waiter is gone away and we are
566  *		actually deboosting the owner)
567  * @top_task:	the current top waiter
568  *
569  * Returns 0 or -EDEADLK.
570  *
571  * Chain walk basics and protection scope
572  *
573  * [R] refcount on task
574  * [P] task->pi_lock held
575  * [L] rtmutex->wait_lock held
576  *
577  * Step	Description				Protected by
578  *	function arguments:
579  *	@task					[R]
580  *	@orig_lock if != NULL			@top_task is blocked on it
581  *	@next_lock				Unprotected. Cannot be
582  *						dereferenced. Only used for
583  *						comparison.
584  *	@orig_waiter if != NULL			@top_task is blocked on it
585  *	@top_task				current, or in case of proxy
586  *						locking protected by calling
587  *						code
588  *	again:
589  *	  loop_sanity_check();
590  *	retry:
591  * [1]	  lock(task->pi_lock);			[R] acquire [P]
592  * [2]	  waiter = task->pi_blocked_on;		[P]
593  * [3]	  check_exit_conditions_1();		[P]
594  * [4]	  lock = waiter->lock;			[P]
595  * [5]	  if (!try_lock(lock->wait_lock)) {	[P] try to acquire [L]
596  *	    unlock(task->pi_lock);		release [P]
597  *	    goto retry;
598  *	  }
599  * [6]	  check_exit_conditions_2();		[P] + [L]
600  * [7]	  requeue_lock_waiter(lock, waiter);	[P] + [L]
601  * [8]	  unlock(task->pi_lock);		release [P]
602  *	  put_task_struct(task);		release [R]
603  * [9]	  check_exit_conditions_3();		[L]
604  * [10]	  task = owner(lock);			[L]
605  *	  get_task_struct(task);		[L] acquire [R]
606  *	  lock(task->pi_lock);			[L] acquire [P]
607  * [11]	  requeue_pi_waiter(tsk, waiters(lock));[P] + [L]
608  * [12]	  check_exit_conditions_4();		[P] + [L]
609  * [13]	  unlock(task->pi_lock);		release [P]
610  *	  unlock(lock->wait_lock);		release [L]
611  *	  goto again;
612  */
rt_mutex_adjust_prio_chain(struct task_struct * task,enum rtmutex_chainwalk chwalk,struct rt_mutex_base * orig_lock,struct rt_mutex_base * next_lock,struct rt_mutex_waiter * orig_waiter,struct task_struct * top_task)613 static int __sched rt_mutex_adjust_prio_chain(struct task_struct *task,
614 					      enum rtmutex_chainwalk chwalk,
615 					      struct rt_mutex_base *orig_lock,
616 					      struct rt_mutex_base *next_lock,
617 					      struct rt_mutex_waiter *orig_waiter,
618 					      struct task_struct *top_task)
619 {
620 	struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
621 	struct rt_mutex_waiter *prerequeue_top_waiter;
622 	int ret = 0, depth = 0;
623 	struct rt_mutex_base *lock;
624 	bool detect_deadlock;
625 	bool requeue = true;
626 
627 	detect_deadlock = rt_mutex_cond_detect_deadlock(orig_waiter, chwalk);
628 
629 	/*
630 	 * The (de)boosting is a step by step approach with a lot of
631 	 * pitfalls. We want this to be preemptible and we want hold a
632 	 * maximum of two locks per step. So we have to check
633 	 * carefully whether things change under us.
634 	 */
635  again:
636 	/*
637 	 * We limit the lock chain length for each invocation.
638 	 */
639 	if (++depth > max_lock_depth) {
640 		static int prev_max;
641 
642 		/*
643 		 * Print this only once. If the admin changes the limit,
644 		 * print a new message when reaching the limit again.
645 		 */
646 		if (prev_max != max_lock_depth) {
647 			prev_max = max_lock_depth;
648 			printk(KERN_WARNING "Maximum lock depth %d reached "
649 			       "task: %s (%d)\n", max_lock_depth,
650 			       top_task->comm, task_pid_nr(top_task));
651 		}
652 		put_task_struct(task);
653 
654 		return -EDEADLK;
655 	}
656 
657 	/*
658 	 * We are fully preemptible here and only hold the refcount on
659 	 * @task. So everything can have changed under us since the
660 	 * caller or our own code below (goto retry/again) dropped all
661 	 * locks.
662 	 */
663  retry:
664 	/*
665 	 * [1] Task cannot go away as we did a get_task() before !
666 	 */
667 	raw_spin_lock_irq(&task->pi_lock);
668 
669 	/*
670 	 * [2] Get the waiter on which @task is blocked on.
671 	 */
672 	waiter = task->pi_blocked_on;
673 
674 	/*
675 	 * [3] check_exit_conditions_1() protected by task->pi_lock.
676 	 */
677 
678 	/*
679 	 * Check whether the end of the boosting chain has been
680 	 * reached or the state of the chain has changed while we
681 	 * dropped the locks.
682 	 */
683 	if (!waiter)
684 		goto out_unlock_pi;
685 
686 	/*
687 	 * Check the orig_waiter state. After we dropped the locks,
688 	 * the previous owner of the lock might have released the lock.
689 	 */
690 	if (orig_waiter && !rt_mutex_owner(orig_lock))
691 		goto out_unlock_pi;
692 
693 	/*
694 	 * We dropped all locks after taking a refcount on @task, so
695 	 * the task might have moved on in the lock chain or even left
696 	 * the chain completely and blocks now on an unrelated lock or
697 	 * on @orig_lock.
698 	 *
699 	 * We stored the lock on which @task was blocked in @next_lock,
700 	 * so we can detect the chain change.
701 	 */
702 	if (next_lock != waiter->lock)
703 		goto out_unlock_pi;
704 
705 	/*
706 	 * There could be 'spurious' loops in the lock graph due to ww_mutex,
707 	 * consider:
708 	 *
709 	 *   P1: A, ww_A, ww_B
710 	 *   P2: ww_B, ww_A
711 	 *   P3: A
712 	 *
713 	 * P3 should not return -EDEADLK because it gets trapped in the cycle
714 	 * created by P1 and P2 (which will resolve -- and runs into
715 	 * max_lock_depth above). Therefore disable detect_deadlock such that
716 	 * the below termination condition can trigger once all relevant tasks
717 	 * are boosted.
718 	 *
719 	 * Even when we start with ww_mutex we can disable deadlock detection,
720 	 * since we would supress a ww_mutex induced deadlock at [6] anyway.
721 	 * Supressing it here however is not sufficient since we might still
722 	 * hit [6] due to adjustment driven iteration.
723 	 *
724 	 * NOTE: if someone were to create a deadlock between 2 ww_classes we'd
725 	 * utterly fail to report it; lockdep should.
726 	 */
727 	if (IS_ENABLED(CONFIG_PREEMPT_RT) && waiter->ww_ctx && detect_deadlock)
728 		detect_deadlock = false;
729 
730 	/*
731 	 * Drop out, when the task has no waiters. Note,
732 	 * top_waiter can be NULL, when we are in the deboosting
733 	 * mode!
734 	 */
735 	if (top_waiter) {
736 		if (!task_has_pi_waiters(task))
737 			goto out_unlock_pi;
738 		/*
739 		 * If deadlock detection is off, we stop here if we
740 		 * are not the top pi waiter of the task. If deadlock
741 		 * detection is enabled we continue, but stop the
742 		 * requeueing in the chain walk.
743 		 */
744 		if (top_waiter != task_top_pi_waiter(task)) {
745 			if (!detect_deadlock)
746 				goto out_unlock_pi;
747 			else
748 				requeue = false;
749 		}
750 	}
751 
752 	/*
753 	 * If the waiter priority is the same as the task priority
754 	 * then there is no further priority adjustment necessary.  If
755 	 * deadlock detection is off, we stop the chain walk. If its
756 	 * enabled we continue, but stop the requeueing in the chain
757 	 * walk.
758 	 */
759 	if (rt_mutex_waiter_equal(waiter, task_to_waiter(task))) {
760 		if (!detect_deadlock)
761 			goto out_unlock_pi;
762 		else
763 			requeue = false;
764 	}
765 
766 	/*
767 	 * [4] Get the next lock
768 	 */
769 	lock = waiter->lock;
770 	/*
771 	 * [5] We need to trylock here as we are holding task->pi_lock,
772 	 * which is the reverse lock order versus the other rtmutex
773 	 * operations.
774 	 */
775 	if (!raw_spin_trylock(&lock->wait_lock)) {
776 		raw_spin_unlock_irq(&task->pi_lock);
777 		cpu_relax();
778 		goto retry;
779 	}
780 
781 	/*
782 	 * [6] check_exit_conditions_2() protected by task->pi_lock and
783 	 * lock->wait_lock.
784 	 *
785 	 * Deadlock detection. If the lock is the same as the original
786 	 * lock which caused us to walk the lock chain or if the
787 	 * current lock is owned by the task which initiated the chain
788 	 * walk, we detected a deadlock.
789 	 */
790 	if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
791 		ret = -EDEADLK;
792 
793 		/*
794 		 * When the deadlock is due to ww_mutex; also see above. Don't
795 		 * report the deadlock and instead let the ww_mutex wound/die
796 		 * logic pick which of the contending threads gets -EDEADLK.
797 		 *
798 		 * NOTE: assumes the cycle only contains a single ww_class; any
799 		 * other configuration and we fail to report; also, see
800 		 * lockdep.
801 		 */
802 		if (IS_ENABLED(CONFIG_PREEMPT_RT) && orig_waiter && orig_waiter->ww_ctx)
803 			ret = 0;
804 
805 		raw_spin_unlock(&lock->wait_lock);
806 		goto out_unlock_pi;
807 	}
808 
809 	/*
810 	 * If we just follow the lock chain for deadlock detection, no
811 	 * need to do all the requeue operations. To avoid a truckload
812 	 * of conditionals around the various places below, just do the
813 	 * minimum chain walk checks.
814 	 */
815 	if (!requeue) {
816 		/*
817 		 * No requeue[7] here. Just release @task [8]
818 		 */
819 		raw_spin_unlock(&task->pi_lock);
820 		put_task_struct(task);
821 
822 		/*
823 		 * [9] check_exit_conditions_3 protected by lock->wait_lock.
824 		 * If there is no owner of the lock, end of chain.
825 		 */
826 		if (!rt_mutex_owner(lock)) {
827 			raw_spin_unlock_irq(&lock->wait_lock);
828 			return 0;
829 		}
830 
831 		/* [10] Grab the next task, i.e. owner of @lock */
832 		task = get_task_struct(rt_mutex_owner(lock));
833 		raw_spin_lock(&task->pi_lock);
834 
835 		/*
836 		 * No requeue [11] here. We just do deadlock detection.
837 		 *
838 		 * [12] Store whether owner is blocked
839 		 * itself. Decision is made after dropping the locks
840 		 */
841 		next_lock = task_blocked_on_lock(task);
842 		/*
843 		 * Get the top waiter for the next iteration
844 		 */
845 		top_waiter = rt_mutex_top_waiter(lock);
846 
847 		/* [13] Drop locks */
848 		raw_spin_unlock(&task->pi_lock);
849 		raw_spin_unlock_irq(&lock->wait_lock);
850 
851 		/* If owner is not blocked, end of chain. */
852 		if (!next_lock)
853 			goto out_put_task;
854 		goto again;
855 	}
856 
857 	/*
858 	 * Store the current top waiter before doing the requeue
859 	 * operation on @lock. We need it for the boost/deboost
860 	 * decision below.
861 	 */
862 	prerequeue_top_waiter = rt_mutex_top_waiter(lock);
863 
864 	/* [7] Requeue the waiter in the lock waiter tree. */
865 	rt_mutex_dequeue(lock, waiter);
866 
867 	/*
868 	 * Update the waiter prio fields now that we're dequeued.
869 	 *
870 	 * These values can have changed through either:
871 	 *
872 	 *   sys_sched_set_scheduler() / sys_sched_setattr()
873 	 *
874 	 * or
875 	 *
876 	 *   DL CBS enforcement advancing the effective deadline.
877 	 *
878 	 * Even though pi_waiters also uses these fields, and that tree is only
879 	 * updated in [11], we can do this here, since we hold [L], which
880 	 * serializes all pi_waiters access and rb_erase() does not care about
881 	 * the values of the node being removed.
882 	 */
883 	waiter_update_prio(waiter, task);
884 
885 	rt_mutex_enqueue(lock, waiter);
886 
887 	/* [8] Release the task */
888 	raw_spin_unlock(&task->pi_lock);
889 	put_task_struct(task);
890 
891 	/*
892 	 * [9] check_exit_conditions_3 protected by lock->wait_lock.
893 	 *
894 	 * We must abort the chain walk if there is no lock owner even
895 	 * in the dead lock detection case, as we have nothing to
896 	 * follow here. This is the end of the chain we are walking.
897 	 */
898 	if (!rt_mutex_owner(lock)) {
899 		/*
900 		 * If the requeue [7] above changed the top waiter,
901 		 * then we need to wake the new top waiter up to try
902 		 * to get the lock.
903 		 */
904 		if (prerequeue_top_waiter != rt_mutex_top_waiter(lock))
905 			wake_up_state(waiter->task, waiter->wake_state);
906 		raw_spin_unlock_irq(&lock->wait_lock);
907 		return 0;
908 	}
909 
910 	/* [10] Grab the next task, i.e. the owner of @lock */
911 	task = get_task_struct(rt_mutex_owner(lock));
912 	raw_spin_lock(&task->pi_lock);
913 
914 	/* [11] requeue the pi waiters if necessary */
915 	if (waiter == rt_mutex_top_waiter(lock)) {
916 		/*
917 		 * The waiter became the new top (highest priority)
918 		 * waiter on the lock. Replace the previous top waiter
919 		 * in the owner tasks pi waiters tree with this waiter
920 		 * and adjust the priority of the owner.
921 		 */
922 		rt_mutex_dequeue_pi(task, prerequeue_top_waiter);
923 		rt_mutex_enqueue_pi(task, waiter);
924 		rt_mutex_adjust_prio(task);
925 
926 	} else if (prerequeue_top_waiter == waiter) {
927 		/*
928 		 * The waiter was the top waiter on the lock, but is
929 		 * no longer the top priority waiter. Replace waiter in
930 		 * the owner tasks pi waiters tree with the new top
931 		 * (highest priority) waiter and adjust the priority
932 		 * of the owner.
933 		 * The new top waiter is stored in @waiter so that
934 		 * @waiter == @top_waiter evaluates to true below and
935 		 * we continue to deboost the rest of the chain.
936 		 */
937 		rt_mutex_dequeue_pi(task, waiter);
938 		waiter = rt_mutex_top_waiter(lock);
939 		rt_mutex_enqueue_pi(task, waiter);
940 		rt_mutex_adjust_prio(task);
941 	} else {
942 		/*
943 		 * Nothing changed. No need to do any priority
944 		 * adjustment.
945 		 */
946 	}
947 
948 	/*
949 	 * [12] check_exit_conditions_4() protected by task->pi_lock
950 	 * and lock->wait_lock. The actual decisions are made after we
951 	 * dropped the locks.
952 	 *
953 	 * Check whether the task which owns the current lock is pi
954 	 * blocked itself. If yes we store a pointer to the lock for
955 	 * the lock chain change detection above. After we dropped
956 	 * task->pi_lock next_lock cannot be dereferenced anymore.
957 	 */
958 	next_lock = task_blocked_on_lock(task);
959 	/*
960 	 * Store the top waiter of @lock for the end of chain walk
961 	 * decision below.
962 	 */
963 	top_waiter = rt_mutex_top_waiter(lock);
964 
965 	/* [13] Drop the locks */
966 	raw_spin_unlock(&task->pi_lock);
967 	raw_spin_unlock_irq(&lock->wait_lock);
968 
969 	/*
970 	 * Make the actual exit decisions [12], based on the stored
971 	 * values.
972 	 *
973 	 * We reached the end of the lock chain. Stop right here. No
974 	 * point to go back just to figure that out.
975 	 */
976 	if (!next_lock)
977 		goto out_put_task;
978 
979 	/*
980 	 * If the current waiter is not the top waiter on the lock,
981 	 * then we can stop the chain walk here if we are not in full
982 	 * deadlock detection mode.
983 	 */
984 	if (!detect_deadlock && waiter != top_waiter)
985 		goto out_put_task;
986 
987 	goto again;
988 
989  out_unlock_pi:
990 	raw_spin_unlock_irq(&task->pi_lock);
991  out_put_task:
992 	put_task_struct(task);
993 
994 	return ret;
995 }
996 
997 /*
998  * Try to take an rt-mutex
999  *
1000  * Must be called with lock->wait_lock held and interrupts disabled
1001  *
1002  * @lock:   The lock to be acquired.
1003  * @task:   The task which wants to acquire the lock
1004  * @waiter: The waiter that is queued to the lock's wait tree if the
1005  *	    callsite called task_blocked_on_lock(), otherwise NULL
1006  */
1007 static int __sched
try_to_take_rt_mutex(struct rt_mutex_base * lock,struct task_struct * task,struct rt_mutex_waiter * waiter)1008 try_to_take_rt_mutex(struct rt_mutex_base *lock, struct task_struct *task,
1009 		     struct rt_mutex_waiter *waiter)
1010 {
1011 	lockdep_assert_held(&lock->wait_lock);
1012 
1013 	/*
1014 	 * Before testing whether we can acquire @lock, we set the
1015 	 * RT_MUTEX_HAS_WAITERS bit in @lock->owner. This forces all
1016 	 * other tasks which try to modify @lock into the slow path
1017 	 * and they serialize on @lock->wait_lock.
1018 	 *
1019 	 * The RT_MUTEX_HAS_WAITERS bit can have a transitional state
1020 	 * as explained at the top of this file if and only if:
1021 	 *
1022 	 * - There is a lock owner. The caller must fixup the
1023 	 *   transient state if it does a trylock or leaves the lock
1024 	 *   function due to a signal or timeout.
1025 	 *
1026 	 * - @task acquires the lock and there are no other
1027 	 *   waiters. This is undone in rt_mutex_set_owner(@task) at
1028 	 *   the end of this function.
1029 	 */
1030 	mark_rt_mutex_waiters(lock);
1031 
1032 	/*
1033 	 * If @lock has an owner, give up.
1034 	 */
1035 	if (rt_mutex_owner(lock))
1036 		return 0;
1037 
1038 	/*
1039 	 * If @waiter != NULL, @task has already enqueued the waiter
1040 	 * into @lock waiter tree. If @waiter == NULL then this is a
1041 	 * trylock attempt.
1042 	 */
1043 	if (waiter) {
1044 		struct rt_mutex_waiter *top_waiter = rt_mutex_top_waiter(lock);
1045 
1046 		/*
1047 		 * If waiter is the highest priority waiter of @lock,
1048 		 * or allowed to steal it, take it over.
1049 		 */
1050 		if (waiter == top_waiter || rt_mutex_steal(waiter, top_waiter)) {
1051 			/*
1052 			 * We can acquire the lock. Remove the waiter from the
1053 			 * lock waiters tree.
1054 			 */
1055 			rt_mutex_dequeue(lock, waiter);
1056 		} else {
1057 			return 0;
1058 		}
1059 	} else {
1060 		/*
1061 		 * If the lock has waiters already we check whether @task is
1062 		 * eligible to take over the lock.
1063 		 *
1064 		 * If there are no other waiters, @task can acquire
1065 		 * the lock.  @task->pi_blocked_on is NULL, so it does
1066 		 * not need to be dequeued.
1067 		 */
1068 		if (rt_mutex_has_waiters(lock)) {
1069 			/* Check whether the trylock can steal it. */
1070 			if (!rt_mutex_steal(task_to_waiter(task),
1071 					    rt_mutex_top_waiter(lock)))
1072 				return 0;
1073 
1074 			/*
1075 			 * The current top waiter stays enqueued. We
1076 			 * don't have to change anything in the lock
1077 			 * waiters order.
1078 			 */
1079 		} else {
1080 			/*
1081 			 * No waiters. Take the lock without the
1082 			 * pi_lock dance.@task->pi_blocked_on is NULL
1083 			 * and we have no waiters to enqueue in @task
1084 			 * pi waiters tree.
1085 			 */
1086 			goto takeit;
1087 		}
1088 	}
1089 
1090 	/*
1091 	 * Clear @task->pi_blocked_on. Requires protection by
1092 	 * @task->pi_lock. Redundant operation for the @waiter == NULL
1093 	 * case, but conditionals are more expensive than a redundant
1094 	 * store.
1095 	 */
1096 	raw_spin_lock(&task->pi_lock);
1097 	task->pi_blocked_on = NULL;
1098 	/*
1099 	 * Finish the lock acquisition. @task is the new owner. If
1100 	 * other waiters exist we have to insert the highest priority
1101 	 * waiter into @task->pi_waiters tree.
1102 	 */
1103 	if (rt_mutex_has_waiters(lock))
1104 		rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));
1105 	raw_spin_unlock(&task->pi_lock);
1106 
1107 takeit:
1108 	/*
1109 	 * This either preserves the RT_MUTEX_HAS_WAITERS bit if there
1110 	 * are still waiters or clears it.
1111 	 */
1112 	rt_mutex_set_owner(lock, task);
1113 
1114 	return 1;
1115 }
1116 
1117 /*
1118  * Task blocks on lock.
1119  *
1120  * Prepare waiter and propagate pi chain
1121  *
1122  * This must be called with lock->wait_lock held and interrupts disabled
1123  */
task_blocks_on_rt_mutex(struct rt_mutex_base * lock,struct rt_mutex_waiter * waiter,struct task_struct * task,struct ww_acquire_ctx * ww_ctx,enum rtmutex_chainwalk chwalk)1124 static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock,
1125 					   struct rt_mutex_waiter *waiter,
1126 					   struct task_struct *task,
1127 					   struct ww_acquire_ctx *ww_ctx,
1128 					   enum rtmutex_chainwalk chwalk)
1129 {
1130 	struct task_struct *owner = rt_mutex_owner(lock);
1131 	struct rt_mutex_waiter *top_waiter = waiter;
1132 	struct rt_mutex_base *next_lock;
1133 	int chain_walk = 0, res;
1134 
1135 	lockdep_assert_held(&lock->wait_lock);
1136 
1137 	/*
1138 	 * Early deadlock detection. We really don't want the task to
1139 	 * enqueue on itself just to untangle the mess later. It's not
1140 	 * only an optimization. We drop the locks, so another waiter
1141 	 * can come in before the chain walk detects the deadlock. So
1142 	 * the other will detect the deadlock and return -EDEADLOCK,
1143 	 * which is wrong, as the other waiter is not in a deadlock
1144 	 * situation.
1145 	 *
1146 	 * Except for ww_mutex, in that case the chain walk must already deal
1147 	 * with spurious cycles, see the comments at [3] and [6].
1148 	 */
1149 	if (owner == task && !(build_ww_mutex() && ww_ctx))
1150 		return -EDEADLK;
1151 
1152 	raw_spin_lock(&task->pi_lock);
1153 	waiter->task = task;
1154 	waiter->lock = lock;
1155 	waiter_update_prio(waiter, task);
1156 
1157 	/* Get the top priority waiter on the lock */
1158 	if (rt_mutex_has_waiters(lock))
1159 		top_waiter = rt_mutex_top_waiter(lock);
1160 	rt_mutex_enqueue(lock, waiter);
1161 
1162 	task->pi_blocked_on = waiter;
1163 
1164 	raw_spin_unlock(&task->pi_lock);
1165 
1166 	if (build_ww_mutex() && ww_ctx) {
1167 		struct rt_mutex *rtm;
1168 
1169 		/* Check whether the waiter should back out immediately */
1170 		rtm = container_of(lock, struct rt_mutex, rtmutex);
1171 		res = __ww_mutex_add_waiter(waiter, rtm, ww_ctx);
1172 		if (res) {
1173 			raw_spin_lock(&task->pi_lock);
1174 			rt_mutex_dequeue(lock, waiter);
1175 			task->pi_blocked_on = NULL;
1176 			raw_spin_unlock(&task->pi_lock);
1177 			return res;
1178 		}
1179 	}
1180 
1181 	if (!owner)
1182 		return 0;
1183 
1184 	raw_spin_lock(&owner->pi_lock);
1185 	if (waiter == rt_mutex_top_waiter(lock)) {
1186 		rt_mutex_dequeue_pi(owner, top_waiter);
1187 		rt_mutex_enqueue_pi(owner, waiter);
1188 
1189 		rt_mutex_adjust_prio(owner);
1190 		if (owner->pi_blocked_on)
1191 			chain_walk = 1;
1192 	} else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) {
1193 		chain_walk = 1;
1194 	}
1195 
1196 	/* Store the lock on which owner is blocked or NULL */
1197 	next_lock = task_blocked_on_lock(owner);
1198 
1199 	raw_spin_unlock(&owner->pi_lock);
1200 	/*
1201 	 * Even if full deadlock detection is on, if the owner is not
1202 	 * blocked itself, we can avoid finding this out in the chain
1203 	 * walk.
1204 	 */
1205 	if (!chain_walk || !next_lock)
1206 		return 0;
1207 
1208 	/*
1209 	 * The owner can't disappear while holding a lock,
1210 	 * so the owner struct is protected by wait_lock.
1211 	 * Gets dropped in rt_mutex_adjust_prio_chain()!
1212 	 */
1213 	get_task_struct(owner);
1214 
1215 	raw_spin_unlock_irq(&lock->wait_lock);
1216 
1217 	res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,
1218 					 next_lock, waiter, task);
1219 
1220 	raw_spin_lock_irq(&lock->wait_lock);
1221 
1222 	return res;
1223 }
1224 
1225 /*
1226  * Remove the top waiter from the current tasks pi waiter tree and
1227  * queue it up.
1228  *
1229  * Called with lock->wait_lock held and interrupts disabled.
1230  */
mark_wakeup_next_waiter(struct rt_wake_q_head * wqh,struct rt_mutex_base * lock)1231 static void __sched mark_wakeup_next_waiter(struct rt_wake_q_head *wqh,
1232 					    struct rt_mutex_base *lock)
1233 {
1234 	struct rt_mutex_waiter *waiter;
1235 
1236 	raw_spin_lock(&current->pi_lock);
1237 
1238 	waiter = rt_mutex_top_waiter(lock);
1239 
1240 	/*
1241 	 * Remove it from current->pi_waiters and deboost.
1242 	 *
1243 	 * We must in fact deboost here in order to ensure we call
1244 	 * rt_mutex_setprio() to update p->pi_top_task before the
1245 	 * task unblocks.
1246 	 */
1247 	rt_mutex_dequeue_pi(current, waiter);
1248 	rt_mutex_adjust_prio(current);
1249 
1250 	/*
1251 	 * As we are waking up the top waiter, and the waiter stays
1252 	 * queued on the lock until it gets the lock, this lock
1253 	 * obviously has waiters. Just set the bit here and this has
1254 	 * the added benefit of forcing all new tasks into the
1255 	 * slow path making sure no task of lower priority than
1256 	 * the top waiter can steal this lock.
1257 	 */
1258 	lock->owner = (void *) RT_MUTEX_HAS_WAITERS;
1259 
1260 	/*
1261 	 * We deboosted before waking the top waiter task such that we don't
1262 	 * run two tasks with the 'same' priority (and ensure the
1263 	 * p->pi_top_task pointer points to a blocked task). This however can
1264 	 * lead to priority inversion if we would get preempted after the
1265 	 * deboost but before waking our donor task, hence the preempt_disable()
1266 	 * before unlock.
1267 	 *
1268 	 * Pairs with preempt_enable() in rt_mutex_wake_up_q();
1269 	 */
1270 	preempt_disable();
1271 	rt_mutex_wake_q_add(wqh, waiter);
1272 	raw_spin_unlock(&current->pi_lock);
1273 }
1274 
__rt_mutex_slowtrylock(struct rt_mutex_base * lock)1275 static int __sched __rt_mutex_slowtrylock(struct rt_mutex_base *lock)
1276 {
1277 	int ret = try_to_take_rt_mutex(lock, current, NULL);
1278 
1279 	/*
1280 	 * try_to_take_rt_mutex() sets the lock waiters bit
1281 	 * unconditionally. Clean this up.
1282 	 */
1283 	fixup_rt_mutex_waiters(lock, true);
1284 
1285 	return ret;
1286 }
1287 
1288 /*
1289  * Slow path try-lock function:
1290  */
rt_mutex_slowtrylock(struct rt_mutex_base * lock)1291 static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock)
1292 {
1293 	unsigned long flags;
1294 	int ret;
1295 
1296 	/*
1297 	 * If the lock already has an owner we fail to get the lock.
1298 	 * This can be done without taking the @lock->wait_lock as
1299 	 * it is only being read, and this is a trylock anyway.
1300 	 */
1301 	if (rt_mutex_owner(lock))
1302 		return 0;
1303 
1304 	/*
1305 	 * The mutex has currently no owner. Lock the wait lock and try to
1306 	 * acquire the lock. We use irqsave here to support early boot calls.
1307 	 */
1308 	raw_spin_lock_irqsave(&lock->wait_lock, flags);
1309 
1310 	ret = __rt_mutex_slowtrylock(lock);
1311 
1312 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1313 
1314 	return ret;
1315 }
1316 
__rt_mutex_trylock(struct rt_mutex_base * lock)1317 static __always_inline int __rt_mutex_trylock(struct rt_mutex_base *lock)
1318 {
1319 	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
1320 		return 1;
1321 
1322 	return rt_mutex_slowtrylock(lock);
1323 }
1324 
1325 /*
1326  * Slow path to release a rt-mutex.
1327  */
rt_mutex_slowunlock(struct rt_mutex_base * lock)1328 static void __sched rt_mutex_slowunlock(struct rt_mutex_base *lock)
1329 {
1330 	DEFINE_RT_WAKE_Q(wqh);
1331 	unsigned long flags;
1332 
1333 	/* irqsave required to support early boot calls */
1334 	raw_spin_lock_irqsave(&lock->wait_lock, flags);
1335 
1336 	debug_rt_mutex_unlock(lock);
1337 
1338 	/*
1339 	 * We must be careful here if the fast path is enabled. If we
1340 	 * have no waiters queued we cannot set owner to NULL here
1341 	 * because of:
1342 	 *
1343 	 * foo->lock->owner = NULL;
1344 	 *			rtmutex_lock(foo->lock);   <- fast path
1345 	 *			free = atomic_dec_and_test(foo->refcnt);
1346 	 *			rtmutex_unlock(foo->lock); <- fast path
1347 	 *			if (free)
1348 	 *				kfree(foo);
1349 	 * raw_spin_unlock(foo->lock->wait_lock);
1350 	 *
1351 	 * So for the fastpath enabled kernel:
1352 	 *
1353 	 * Nothing can set the waiters bit as long as we hold
1354 	 * lock->wait_lock. So we do the following sequence:
1355 	 *
1356 	 *	owner = rt_mutex_owner(lock);
1357 	 *	clear_rt_mutex_waiters(lock);
1358 	 *	raw_spin_unlock(&lock->wait_lock);
1359 	 *	if (cmpxchg(&lock->owner, owner, 0) == owner)
1360 	 *		return;
1361 	 *	goto retry;
1362 	 *
1363 	 * The fastpath disabled variant is simple as all access to
1364 	 * lock->owner is serialized by lock->wait_lock:
1365 	 *
1366 	 *	lock->owner = NULL;
1367 	 *	raw_spin_unlock(&lock->wait_lock);
1368 	 */
1369 	while (!rt_mutex_has_waiters(lock)) {
1370 		/* Drops lock->wait_lock ! */
1371 		if (unlock_rt_mutex_safe(lock, flags) == true)
1372 			return;
1373 		/* Relock the rtmutex and try again */
1374 		raw_spin_lock_irqsave(&lock->wait_lock, flags);
1375 	}
1376 
1377 	/*
1378 	 * The wakeup next waiter path does not suffer from the above
1379 	 * race. See the comments there.
1380 	 *
1381 	 * Queue the next waiter for wakeup once we release the wait_lock.
1382 	 */
1383 	mark_wakeup_next_waiter(&wqh, lock);
1384 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1385 
1386 	rt_mutex_wake_up_q(&wqh);
1387 }
1388 
__rt_mutex_unlock(struct rt_mutex_base * lock)1389 static __always_inline void __rt_mutex_unlock(struct rt_mutex_base *lock)
1390 {
1391 	if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))
1392 		return;
1393 
1394 	rt_mutex_slowunlock(lock);
1395 }
1396 
1397 #ifdef CONFIG_SMP
rtmutex_spin_on_owner(struct rt_mutex_base * lock,struct rt_mutex_waiter * waiter,struct task_struct * owner)1398 static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,
1399 				  struct rt_mutex_waiter *waiter,
1400 				  struct task_struct *owner)
1401 {
1402 	bool res = true;
1403 
1404 	rcu_read_lock();
1405 	for (;;) {
1406 		/* If owner changed, trylock again. */
1407 		if (owner != rt_mutex_owner(lock))
1408 			break;
1409 		/*
1410 		 * Ensure that @owner is dereferenced after checking that
1411 		 * the lock owner still matches @owner. If that fails,
1412 		 * @owner might point to freed memory. If it still matches,
1413 		 * the rcu_read_lock() ensures the memory stays valid.
1414 		 */
1415 		barrier();
1416 		/*
1417 		 * Stop spinning when:
1418 		 *  - the lock owner has been scheduled out
1419 		 *  - current is not longer the top waiter
1420 		 *  - current is requested to reschedule (redundant
1421 		 *    for CONFIG_PREEMPT_RCU=y)
1422 		 *  - the VCPU on which owner runs is preempted
1423 		 */
1424 		if (!owner_on_cpu(owner) || need_resched() ||
1425 		    !rt_mutex_waiter_is_top_waiter(lock, waiter)) {
1426 			res = false;
1427 			break;
1428 		}
1429 		cpu_relax();
1430 	}
1431 	rcu_read_unlock();
1432 	return res;
1433 }
1434 #else
rtmutex_spin_on_owner(struct rt_mutex_base * lock,struct rt_mutex_waiter * waiter,struct task_struct * owner)1435 static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,
1436 				  struct rt_mutex_waiter *waiter,
1437 				  struct task_struct *owner)
1438 {
1439 	return false;
1440 }
1441 #endif
1442 
1443 #ifdef RT_MUTEX_BUILD_MUTEX
1444 /*
1445  * Functions required for:
1446  *	- rtmutex, futex on all kernels
1447  *	- mutex and rwsem substitutions on RT kernels
1448  */
1449 
1450 /*
1451  * Remove a waiter from a lock and give up
1452  *
1453  * Must be called with lock->wait_lock held and interrupts disabled. It must
1454  * have just failed to try_to_take_rt_mutex().
1455  */
remove_waiter(struct rt_mutex_base * lock,struct rt_mutex_waiter * waiter)1456 static void __sched remove_waiter(struct rt_mutex_base *lock,
1457 				  struct rt_mutex_waiter *waiter)
1458 {
1459 	bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
1460 	struct task_struct *owner = rt_mutex_owner(lock);
1461 	struct rt_mutex_base *next_lock;
1462 
1463 	lockdep_assert_held(&lock->wait_lock);
1464 
1465 	raw_spin_lock(&current->pi_lock);
1466 	rt_mutex_dequeue(lock, waiter);
1467 	current->pi_blocked_on = NULL;
1468 	raw_spin_unlock(&current->pi_lock);
1469 
1470 	/*
1471 	 * Only update priority if the waiter was the highest priority
1472 	 * waiter of the lock and there is an owner to update.
1473 	 */
1474 	if (!owner || !is_top_waiter)
1475 		return;
1476 
1477 	raw_spin_lock(&owner->pi_lock);
1478 
1479 	rt_mutex_dequeue_pi(owner, waiter);
1480 
1481 	if (rt_mutex_has_waiters(lock))
1482 		rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock));
1483 
1484 	rt_mutex_adjust_prio(owner);
1485 
1486 	/* Store the lock on which owner is blocked or NULL */
1487 	next_lock = task_blocked_on_lock(owner);
1488 
1489 	raw_spin_unlock(&owner->pi_lock);
1490 
1491 	/*
1492 	 * Don't walk the chain, if the owner task is not blocked
1493 	 * itself.
1494 	 */
1495 	if (!next_lock)
1496 		return;
1497 
1498 	/* gets dropped in rt_mutex_adjust_prio_chain()! */
1499 	get_task_struct(owner);
1500 
1501 	raw_spin_unlock_irq(&lock->wait_lock);
1502 
1503 	rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,
1504 				   next_lock, NULL, current);
1505 
1506 	raw_spin_lock_irq(&lock->wait_lock);
1507 }
1508 
1509 /**
1510  * rt_mutex_slowlock_block() - Perform the wait-wake-try-to-take loop
1511  * @lock:		 the rt_mutex to take
1512  * @ww_ctx:		 WW mutex context pointer
1513  * @state:		 the state the task should block in (TASK_INTERRUPTIBLE
1514  *			 or TASK_UNINTERRUPTIBLE)
1515  * @timeout:		 the pre-initialized and started timer, or NULL for none
1516  * @waiter:		 the pre-initialized rt_mutex_waiter
1517  *
1518  * Must be called with lock->wait_lock held and interrupts disabled
1519  */
rt_mutex_slowlock_block(struct rt_mutex_base * lock,struct ww_acquire_ctx * ww_ctx,unsigned int state,struct hrtimer_sleeper * timeout,struct rt_mutex_waiter * waiter)1520 static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,
1521 					   struct ww_acquire_ctx *ww_ctx,
1522 					   unsigned int state,
1523 					   struct hrtimer_sleeper *timeout,
1524 					   struct rt_mutex_waiter *waiter)
1525 {
1526 	struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex);
1527 	struct task_struct *owner;
1528 	int ret = 0;
1529 
1530 	for (;;) {
1531 		/* Try to acquire the lock: */
1532 		if (try_to_take_rt_mutex(lock, current, waiter))
1533 			break;
1534 
1535 		if (timeout && !timeout->task) {
1536 			ret = -ETIMEDOUT;
1537 			break;
1538 		}
1539 		if (signal_pending_state(state, current)) {
1540 			ret = -EINTR;
1541 			break;
1542 		}
1543 
1544 		if (build_ww_mutex() && ww_ctx) {
1545 			ret = __ww_mutex_check_kill(rtm, waiter, ww_ctx);
1546 			if (ret)
1547 				break;
1548 		}
1549 
1550 		if (waiter == rt_mutex_top_waiter(lock))
1551 			owner = rt_mutex_owner(lock);
1552 		else
1553 			owner = NULL;
1554 		raw_spin_unlock_irq(&lock->wait_lock);
1555 
1556 		if (!owner || !rtmutex_spin_on_owner(lock, waiter, owner))
1557 			schedule();
1558 
1559 		raw_spin_lock_irq(&lock->wait_lock);
1560 		set_current_state(state);
1561 	}
1562 
1563 	__set_current_state(TASK_RUNNING);
1564 	return ret;
1565 }
1566 
rt_mutex_handle_deadlock(int res,int detect_deadlock,struct rt_mutex_waiter * w)1567 static void __sched rt_mutex_handle_deadlock(int res, int detect_deadlock,
1568 					     struct rt_mutex_waiter *w)
1569 {
1570 	/*
1571 	 * If the result is not -EDEADLOCK or the caller requested
1572 	 * deadlock detection, nothing to do here.
1573 	 */
1574 	if (res != -EDEADLOCK || detect_deadlock)
1575 		return;
1576 
1577 	if (build_ww_mutex() && w->ww_ctx)
1578 		return;
1579 
1580 	/*
1581 	 * Yell loudly and stop the task right here.
1582 	 */
1583 	WARN(1, "rtmutex deadlock detected\n");
1584 	while (1) {
1585 		set_current_state(TASK_INTERRUPTIBLE);
1586 		schedule();
1587 	}
1588 }
1589 
1590 /**
1591  * __rt_mutex_slowlock - Locking slowpath invoked with lock::wait_lock held
1592  * @lock:	The rtmutex to block lock
1593  * @ww_ctx:	WW mutex context pointer
1594  * @state:	The task state for sleeping
1595  * @chwalk:	Indicator whether full or partial chainwalk is requested
1596  * @waiter:	Initializer waiter for blocking
1597  */
__rt_mutex_slowlock(struct rt_mutex_base * lock,struct ww_acquire_ctx * ww_ctx,unsigned int state,enum rtmutex_chainwalk chwalk,struct rt_mutex_waiter * waiter)1598 static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,
1599 				       struct ww_acquire_ctx *ww_ctx,
1600 				       unsigned int state,
1601 				       enum rtmutex_chainwalk chwalk,
1602 				       struct rt_mutex_waiter *waiter)
1603 {
1604 	struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex);
1605 	struct ww_mutex *ww = ww_container_of(rtm);
1606 	int ret;
1607 
1608 	lockdep_assert_held(&lock->wait_lock);
1609 
1610 	/* Try to acquire the lock again: */
1611 	if (try_to_take_rt_mutex(lock, current, NULL)) {
1612 		if (build_ww_mutex() && ww_ctx) {
1613 			__ww_mutex_check_waiters(rtm, ww_ctx);
1614 			ww_mutex_lock_acquired(ww, ww_ctx);
1615 		}
1616 		return 0;
1617 	}
1618 
1619 	set_current_state(state);
1620 
1621 	trace_contention_begin(lock, LCB_F_RT);
1622 
1623 	ret = task_blocks_on_rt_mutex(lock, waiter, current, ww_ctx, chwalk);
1624 	if (likely(!ret))
1625 		ret = rt_mutex_slowlock_block(lock, ww_ctx, state, NULL, waiter);
1626 
1627 	if (likely(!ret)) {
1628 		/* acquired the lock */
1629 		if (build_ww_mutex() && ww_ctx) {
1630 			if (!ww_ctx->is_wait_die)
1631 				__ww_mutex_check_waiters(rtm, ww_ctx);
1632 			ww_mutex_lock_acquired(ww, ww_ctx);
1633 		}
1634 	} else {
1635 		__set_current_state(TASK_RUNNING);
1636 		remove_waiter(lock, waiter);
1637 		rt_mutex_handle_deadlock(ret, chwalk, waiter);
1638 	}
1639 
1640 	/*
1641 	 * try_to_take_rt_mutex() sets the waiter bit
1642 	 * unconditionally. We might have to fix that up.
1643 	 */
1644 	fixup_rt_mutex_waiters(lock, true);
1645 
1646 	trace_contention_end(lock, ret);
1647 
1648 	return ret;
1649 }
1650 
__rt_mutex_slowlock_locked(struct rt_mutex_base * lock,struct ww_acquire_ctx * ww_ctx,unsigned int state)1651 static inline int __rt_mutex_slowlock_locked(struct rt_mutex_base *lock,
1652 					     struct ww_acquire_ctx *ww_ctx,
1653 					     unsigned int state)
1654 {
1655 	struct rt_mutex_waiter waiter;
1656 	int ret;
1657 
1658 	rt_mutex_init_waiter(&waiter);
1659 	waiter.ww_ctx = ww_ctx;
1660 
1661 	ret = __rt_mutex_slowlock(lock, ww_ctx, state, RT_MUTEX_MIN_CHAINWALK,
1662 				  &waiter);
1663 
1664 	debug_rt_mutex_free_waiter(&waiter);
1665 	return ret;
1666 }
1667 
1668 /*
1669  * rt_mutex_slowlock - Locking slowpath invoked when fast path fails
1670  * @lock:	The rtmutex to block lock
1671  * @ww_ctx:	WW mutex context pointer
1672  * @state:	The task state for sleeping
1673  */
rt_mutex_slowlock(struct rt_mutex_base * lock,struct ww_acquire_ctx * ww_ctx,unsigned int state)1674 static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock,
1675 				     struct ww_acquire_ctx *ww_ctx,
1676 				     unsigned int state)
1677 {
1678 	unsigned long flags;
1679 	int ret;
1680 
1681 	/*
1682 	 * Technically we could use raw_spin_[un]lock_irq() here, but this can
1683 	 * be called in early boot if the cmpxchg() fast path is disabled
1684 	 * (debug, no architecture support). In this case we will acquire the
1685 	 * rtmutex with lock->wait_lock held. But we cannot unconditionally
1686 	 * enable interrupts in that early boot case. So we need to use the
1687 	 * irqsave/restore variants.
1688 	 */
1689 	raw_spin_lock_irqsave(&lock->wait_lock, flags);
1690 	ret = __rt_mutex_slowlock_locked(lock, ww_ctx, state);
1691 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1692 
1693 	return ret;
1694 }
1695 
__rt_mutex_lock(struct rt_mutex_base * lock,unsigned int state)1696 static __always_inline int __rt_mutex_lock(struct rt_mutex_base *lock,
1697 					   unsigned int state)
1698 {
1699 	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
1700 		return 0;
1701 
1702 	return rt_mutex_slowlock(lock, NULL, state);
1703 }
1704 #endif /* RT_MUTEX_BUILD_MUTEX */
1705 
1706 #ifdef RT_MUTEX_BUILD_SPINLOCKS
1707 /*
1708  * Functions required for spin/rw_lock substitution on RT kernels
1709  */
1710 
1711 /**
1712  * rtlock_slowlock_locked - Slow path lock acquisition for RT locks
1713  * @lock:	The underlying RT mutex
1714  */
rtlock_slowlock_locked(struct rt_mutex_base * lock)1715 static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock)
1716 {
1717 	struct rt_mutex_waiter waiter;
1718 	struct task_struct *owner;
1719 
1720 	lockdep_assert_held(&lock->wait_lock);
1721 
1722 	if (try_to_take_rt_mutex(lock, current, NULL))
1723 		return;
1724 
1725 	rt_mutex_init_rtlock_waiter(&waiter);
1726 
1727 	/* Save current state and set state to TASK_RTLOCK_WAIT */
1728 	current_save_and_set_rtlock_wait_state();
1729 
1730 	trace_contention_begin(lock, LCB_F_RT);
1731 
1732 	task_blocks_on_rt_mutex(lock, &waiter, current, NULL, RT_MUTEX_MIN_CHAINWALK);
1733 
1734 	for (;;) {
1735 		/* Try to acquire the lock again */
1736 		if (try_to_take_rt_mutex(lock, current, &waiter))
1737 			break;
1738 
1739 		if (&waiter == rt_mutex_top_waiter(lock))
1740 			owner = rt_mutex_owner(lock);
1741 		else
1742 			owner = NULL;
1743 		raw_spin_unlock_irq(&lock->wait_lock);
1744 
1745 		if (!owner || !rtmutex_spin_on_owner(lock, &waiter, owner))
1746 			schedule_rtlock();
1747 
1748 		raw_spin_lock_irq(&lock->wait_lock);
1749 		set_current_state(TASK_RTLOCK_WAIT);
1750 	}
1751 
1752 	/* Restore the task state */
1753 	current_restore_rtlock_saved_state();
1754 
1755 	/*
1756 	 * try_to_take_rt_mutex() sets the waiter bit unconditionally.
1757 	 * We might have to fix that up:
1758 	 */
1759 	fixup_rt_mutex_waiters(lock, true);
1760 	debug_rt_mutex_free_waiter(&waiter);
1761 
1762 	trace_contention_end(lock, 0);
1763 }
1764 
rtlock_slowlock(struct rt_mutex_base * lock)1765 static __always_inline void __sched rtlock_slowlock(struct rt_mutex_base *lock)
1766 {
1767 	unsigned long flags;
1768 
1769 	raw_spin_lock_irqsave(&lock->wait_lock, flags);
1770 	rtlock_slowlock_locked(lock);
1771 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1772 }
1773 
1774 #endif /* RT_MUTEX_BUILD_SPINLOCKS */
1775