22 static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock,  in __rt_mutex_lock_common()  argument
30 	mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, _RET_IP_);  in __rt_mutex_lock_common()
31 	ret = __rt_mutex_lock(&lock->rtmutex, state);  in __rt_mutex_lock_common()
33 		mutex_release(&lock->dep_map, _RET_IP_);  in __rt_mutex_lock_common()
50 void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass)  in rt_mutex_lock_nested()  argument
52 	__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass);  in rt_mutex_lock_nested()
56 void __sched _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock)  in _rt_mutex_lock_nest_lock()  argument
58 	__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0);  in _rt_mutex_lock_nest_lock()
69 void __sched rt_mutex_lock(struct rt_mutex *lock)  in rt_mutex_lock()  argument
71 	__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0);  in rt_mutex_lock()
85 int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock)  in rt_mutex_lock_interruptible()  argument
87 	return __rt_mutex_lock_common(lock, TASK_INTERRUPTIBLE, NULL, 0);  in rt_mutex_lock_interruptible()
100 int __sched rt_mutex_lock_killable(struct rt_mutex *lock)  in rt_mutex_lock_killable()  argument
102 	return __rt_mutex_lock_common(lock, TASK_KILLABLE, NULL, 0);  in rt_mutex_lock_killable()
118 int __sched rt_mutex_trylock(struct rt_mutex *lock)  in rt_mutex_trylock()  argument
125 	ret = __rt_mutex_trylock(&lock->rtmutex);  in rt_mutex_trylock()
127 		mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);  in rt_mutex_trylock()
138 void __sched rt_mutex_unlock(struct rt_mutex *lock)  in rt_mutex_unlock()  argument
140 	mutex_release(&lock->dep_map, _RET_IP_);  in rt_mutex_unlock()
141 	__rt_mutex_unlock(&lock->rtmutex);  in rt_mutex_unlock()
148 int __sched rt_mutex_futex_trylock(struct rt_mutex_base *lock)  in rt_mutex_futex_trylock()  argument
150 	return rt_mutex_slowtrylock(lock);  in rt_mutex_futex_trylock()
153 int __sched __rt_mutex_futex_trylock(struct rt_mutex_base *lock)  in __rt_mutex_futex_trylock()  argument
155 	return __rt_mutex_slowtrylock(lock);  in __rt_mutex_futex_trylock()
165 bool __sched __rt_mutex_futex_unlock(struct rt_mutex_base *lock,  in __rt_mutex_futex_unlock()  argument
168 	lockdep_assert_held(&lock->wait_lock);  in __rt_mutex_futex_unlock()
170 	debug_rt_mutex_unlock(lock);  in __rt_mutex_futex_unlock()
172 	if (!rt_mutex_has_waiters(lock)) {  in __rt_mutex_futex_unlock()
173 		lock->owner = NULL;  in __rt_mutex_futex_unlock()
183 	mark_wakeup_next_waiter(wqh, lock);  in __rt_mutex_futex_unlock()
188 void __sched rt_mutex_futex_unlock(struct rt_mutex_base *lock)  in rt_mutex_futex_unlock()  argument
194 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_futex_unlock()
195 	postunlock = __rt_mutex_futex_unlock(lock, &wqh);  in rt_mutex_futex_unlock()
196 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_futex_unlock()
213 void __sched __rt_mutex_init(struct rt_mutex *lock, const char *name,  in __rt_mutex_init()  argument
216 	debug_check_no_locks_freed((void *)lock, sizeof(*lock));  in __rt_mutex_init()
217 	__rt_mutex_base_init(&lock->rtmutex);  in __rt_mutex_init()
218 	lockdep_init_map_wait(&lock->dep_map, name, key, 0, LD_WAIT_SLEEP);  in __rt_mutex_init()
236 void __sched rt_mutex_init_proxy_locked(struct rt_mutex_base *lock,  in rt_mutex_init_proxy_locked()  argument
241 	__rt_mutex_base_init(lock);  in rt_mutex_init_proxy_locked()
251 	lockdep_set_class(&lock->wait_lock, &pi_futex_key);  in rt_mutex_init_proxy_locked()
252 	rt_mutex_set_owner(lock, proxy_owner);  in rt_mutex_init_proxy_locked()
267 void __sched rt_mutex_proxy_unlock(struct rt_mutex_base *lock)  in rt_mutex_proxy_unlock()  argument
269 	debug_rt_mutex_proxy_unlock(lock);  in rt_mutex_proxy_unlock()
270 	rt_mutex_clear_owner(lock);  in rt_mutex_proxy_unlock()
292 int __sched __rt_mutex_start_proxy_lock(struct rt_mutex_base *lock,  in __rt_mutex_start_proxy_lock()  argument
298 	lockdep_assert_held(&lock->wait_lock);  in __rt_mutex_start_proxy_lock()
300 	if (try_to_take_rt_mutex(lock, task, NULL))  in __rt_mutex_start_proxy_lock()
304 	ret = task_blocks_on_rt_mutex(lock, waiter, task, NULL,  in __rt_mutex_start_proxy_lock()
307 	if (ret && !rt_mutex_owner(lock)) {  in __rt_mutex_start_proxy_lock()
339 int __sched rt_mutex_start_proxy_lock(struct rt_mutex_base *lock,  in rt_mutex_start_proxy_lock()  argument
345 	raw_spin_lock_irq(&lock->wait_lock);  in rt_mutex_start_proxy_lock()
346 	ret = __rt_mutex_start_proxy_lock(lock, waiter, task);  in rt_mutex_start_proxy_lock()
348 		remove_waiter(lock, waiter);  in rt_mutex_start_proxy_lock()
349 	raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_start_proxy_lock()
371 int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock,  in rt_mutex_wait_proxy_lock()  argument
377 	raw_spin_lock_irq(&lock->wait_lock);  in rt_mutex_wait_proxy_lock()
380 	ret = rt_mutex_slowlock_block(lock, NULL, TASK_INTERRUPTIBLE, to, waiter);  in rt_mutex_wait_proxy_lock()
385 	fixup_rt_mutex_waiters(lock, true);  in rt_mutex_wait_proxy_lock()
386 	raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_wait_proxy_lock()
411 bool __sched rt_mutex_cleanup_proxy_lock(struct rt_mutex_base *lock,  in rt_mutex_cleanup_proxy_lock()  argument
416 	raw_spin_lock_irq(&lock->wait_lock);  in rt_mutex_cleanup_proxy_lock()
428 	try_to_take_rt_mutex(lock, current, waiter);  in rt_mutex_cleanup_proxy_lock()
433 	if (rt_mutex_owner(lock) != current) {  in rt_mutex_cleanup_proxy_lock()
434 		remove_waiter(lock, waiter);  in rt_mutex_cleanup_proxy_lock()
441 	fixup_rt_mutex_waiters(lock, false);  in rt_mutex_cleanup_proxy_lock()
443 	raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_cleanup_proxy_lock()
466 	next_lock = waiter->lock;  in rt_mutex_adjust_pi()
502 static __always_inline int __mutex_lock_common(struct mutex *lock,  in __mutex_lock_common()  argument
511 	mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);  in __mutex_lock_common()
512 	ret = __rt_mutex_lock(&lock->rtmutex, state);  in __mutex_lock_common()
514 		mutex_release(&lock->dep_map, ip);  in __mutex_lock_common()
516 		lock_acquired(&lock->dep_map, ip);  in __mutex_lock_common()
521 void __sched mutex_lock_nested(struct mutex *lock, unsigned int subclass)  in mutex_lock_nested()  argument
523 	__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);  in mutex_lock_nested()
527 void __sched _mutex_lock_nest_lock(struct mutex *lock,  in _mutex_lock_nest_lock()  argument
530 	__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, nest_lock, _RET_IP_);  in _mutex_lock_nest_lock()
534 int __sched mutex_lock_interruptible_nested(struct mutex *lock,  in mutex_lock_interruptible_nested()  argument
537 	return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, subclass, NULL, _RET_IP_);  in mutex_lock_interruptible_nested()
541 int __sched mutex_lock_killable_nested(struct mutex *lock,  in mutex_lock_killable_nested()  argument
544 	return __mutex_lock_common(lock, TASK_KILLABLE, subclass, NULL, _RET_IP_);  in mutex_lock_killable_nested()
548 void __sched mutex_lock_io_nested(struct mutex *lock, unsigned int subclass)  in mutex_lock_io_nested()  argument
555 	__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);  in mutex_lock_io_nested()
562 void __sched mutex_lock(struct mutex *lock)  in mutex_lock()  argument
564 	__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);  in mutex_lock()
568 int __sched mutex_lock_interruptible(struct mutex *lock)  in mutex_lock_interruptible()  argument
570 	return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_);  in mutex_lock_interruptible()
574 int __sched mutex_lock_killable(struct mutex *lock)  in mutex_lock_killable()  argument
576 	return __mutex_lock_common(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);  in mutex_lock_killable()
580 void __sched mutex_lock_io(struct mutex *lock)  in mutex_lock_io()  argument
584 	__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);  in mutex_lock_io()
590 int __sched mutex_trylock(struct mutex *lock)  in mutex_trylock()  argument
597 	ret = __rt_mutex_trylock(&lock->rtmutex);  in mutex_trylock()
599 		mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);  in mutex_trylock()
605 void __sched mutex_unlock(struct mutex *lock)  in mutex_unlock()  argument
607 	mutex_release(&lock->dep_map, _RET_IP_);  in mutex_unlock()
608 	__rt_mutex_unlock(&lock->rtmutex);  in mutex_unlock()