53 static __always_inline int rwbase_read_trylock(struct rwbase_rt *rwb)  in rwbase_read_trylock()  argument
61 	for (r = atomic_read(&rwb->readers); r < 0;) {  in rwbase_read_trylock()
62 		if (likely(atomic_try_cmpxchg_acquire(&rwb->readers, &r, r + 1)))  in rwbase_read_trylock()
68 static int __sched __rwbase_read_lock(struct rwbase_rt *rwb,  in __rwbase_read_lock()  argument
71 	struct rt_mutex_base *rtm = &rwb->rtmutex;  in __rwbase_read_lock()
79 	if (atomic_read(&rwb->readers) != WRITER_BIAS) {  in __rwbase_read_lock()
80 		atomic_inc(&rwb->readers);  in __rwbase_read_lock()
115 	trace_contention_begin(rwb, LCB_F_RT | LCB_F_READ);  in __rwbase_read_lock()
131 		atomic_inc(&rwb->readers);  in __rwbase_read_lock()
136 	trace_contention_end(rwb, ret);  in __rwbase_read_lock()
140 static __always_inline int rwbase_read_lock(struct rwbase_rt *rwb,  in rwbase_read_lock()  argument
143 	if (rwbase_read_trylock(rwb))  in rwbase_read_lock()
146 	return __rwbase_read_lock(rwb, state);  in rwbase_read_lock()
149 static void __sched __rwbase_read_unlock(struct rwbase_rt *rwb,  in __rwbase_read_unlock()  argument
152 	struct rt_mutex_base *rtm = &rwb->rtmutex;  in __rwbase_read_unlock()
173 static __always_inline void rwbase_read_unlock(struct rwbase_rt *rwb,  in rwbase_read_unlock()  argument
182 	if (unlikely(atomic_dec_and_test(&rwb->readers)))  in rwbase_read_unlock()
183 		__rwbase_read_unlock(rwb, state);  in rwbase_read_unlock()
186 static inline void __rwbase_write_unlock(struct rwbase_rt *rwb, int bias,  in __rwbase_write_unlock()  argument
189 	struct rt_mutex_base *rtm = &rwb->rtmutex;  in __rwbase_write_unlock()
195 	(void)atomic_add_return_release(READER_BIAS - bias, &rwb->readers);  in __rwbase_write_unlock()
200 static inline void rwbase_write_unlock(struct rwbase_rt *rwb)  in rwbase_write_unlock()  argument
202 	struct rt_mutex_base *rtm = &rwb->rtmutex;  in rwbase_write_unlock()
206 	__rwbase_write_unlock(rwb, WRITER_BIAS, flags);  in rwbase_write_unlock()
209 static inline void rwbase_write_downgrade(struct rwbase_rt *rwb)  in rwbase_write_downgrade()  argument
211 	struct rt_mutex_base *rtm = &rwb->rtmutex;  in rwbase_write_downgrade()
216 	__rwbase_write_unlock(rwb, WRITER_BIAS - 1, flags);  in rwbase_write_downgrade()
219 static inline bool __rwbase_write_trylock(struct rwbase_rt *rwb)  in __rwbase_write_trylock()  argument
222 	lockdep_assert_held(&rwb->rtmutex.wait_lock);  in __rwbase_write_trylock()
228 	if (!atomic_read_acquire(&rwb->readers)) {  in __rwbase_write_trylock()
229 		atomic_set(&rwb->readers, WRITER_BIAS);  in __rwbase_write_trylock()
236 static int __sched rwbase_write_lock(struct rwbase_rt *rwb,  in rwbase_write_lock()  argument
239 	struct rt_mutex_base *rtm = &rwb->rtmutex;  in rwbase_write_lock()
247 	atomic_sub(READER_BIAS, &rwb->readers);  in rwbase_write_lock()
250 	if (__rwbase_write_trylock(rwb))  in rwbase_write_lock()
254 	trace_contention_begin(rwb, LCB_F_RT | LCB_F_WRITE);  in rwbase_write_lock()
259 			__rwbase_write_unlock(rwb, 0, flags);  in rwbase_write_lock()
260 			trace_contention_end(rwb, -EINTR);  in rwbase_write_lock()
264 		if (__rwbase_write_trylock(rwb))  in rwbase_write_lock()
274 	trace_contention_end(rwb, 0);  in rwbase_write_lock()
281 static inline int rwbase_write_trylock(struct rwbase_rt *rwb)  in rwbase_write_trylock()  argument
283 	struct rt_mutex_base *rtm = &rwb->rtmutex;  in rwbase_write_trylock()
289 	atomic_sub(READER_BIAS, &rwb->readers);  in rwbase_write_trylock()
292 	if (__rwbase_write_trylock(rwb)) {  in rwbase_write_trylock()
296 	__rwbase_write_unlock(rwb, 0, flags);  in rwbase_write_trylock()