1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * PREEMPT_RT substitution for spin/rw_locks
4  *
5  * spinlocks and rwlocks on RT are based on rtmutexes, with a few twists to
6  * resemble the non RT semantics:
7  *
8  * - Contrary to plain rtmutexes, spinlocks and rwlocks are state
9  *   preserving. The task state is saved before blocking on the underlying
10  *   rtmutex, and restored when the lock has been acquired. Regular wakeups
11  *   during that time are redirected to the saved state so no wake up is
12  *   missed.
13  *
14  * - Non RT spin/rwlocks disable preemption and eventually interrupts.
15  *   Disabling preemption has the side effect of disabling migration and
16  *   preventing RCU grace periods.
17  *
18  *   The RT substitutions explicitly disable migration and take
19  *   rcu_read_lock() across the lock held section.
20  */
21 #include <linux/spinlock.h>
22 #include <linux/export.h>
23 
24 #define RT_MUTEX_BUILD_SPINLOCKS
25 #include "rtmutex.c"
26 
27 /*
28  * __might_resched() skips the state check as rtlocks are state
29  * preserving. Take RCU nesting into account as spin/read/write_lock() can
30  * legitimately nest into an RCU read side critical section.
31  */
32 #define RTLOCK_RESCHED_OFFSETS						\
33 	(rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT)
34 
35 #define rtlock_might_resched()						\
36 	__might_resched(__FILE__, __LINE__, RTLOCK_RESCHED_OFFSETS)
37 
rtlock_lock(struct rt_mutex_base * rtm)38 static __always_inline void rtlock_lock(struct rt_mutex_base *rtm)
39 {
40 	if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
41 		rtlock_slowlock(rtm);
42 }
43 
__rt_spin_lock(spinlock_t * lock)44 static __always_inline void __rt_spin_lock(spinlock_t *lock)
45 {
46 	rtlock_might_resched();
47 	rtlock_lock(&lock->lock);
48 	rcu_read_lock();
49 	migrate_disable();
50 }
51 
rt_spin_lock(spinlock_t * lock)52 void __sched rt_spin_lock(spinlock_t *lock)
53 {
54 	spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
55 	__rt_spin_lock(lock);
56 }
57 EXPORT_SYMBOL(rt_spin_lock);
58 
59 #ifdef CONFIG_DEBUG_LOCK_ALLOC
rt_spin_lock_nested(spinlock_t * lock,int subclass)60 void __sched rt_spin_lock_nested(spinlock_t *lock, int subclass)
61 {
62 	spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
63 	__rt_spin_lock(lock);
64 }
65 EXPORT_SYMBOL(rt_spin_lock_nested);
66 
rt_spin_lock_nest_lock(spinlock_t * lock,struct lockdep_map * nest_lock)67 void __sched rt_spin_lock_nest_lock(spinlock_t *lock,
68 				    struct lockdep_map *nest_lock)
69 {
70 	spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_);
71 	__rt_spin_lock(lock);
72 }
73 EXPORT_SYMBOL(rt_spin_lock_nest_lock);
74 #endif
75 
rt_spin_unlock(spinlock_t * lock)76 void __sched rt_spin_unlock(spinlock_t *lock)
77 {
78 	spin_release(&lock->dep_map, _RET_IP_);
79 	migrate_enable();
80 	rcu_read_unlock();
81 
82 	if (unlikely(!rt_mutex_cmpxchg_release(&lock->lock, current, NULL)))
83 		rt_mutex_slowunlock(&lock->lock);
84 }
85 EXPORT_SYMBOL(rt_spin_unlock);
86 
87 /*
88  * Wait for the lock to get unlocked: instead of polling for an unlock
89  * (like raw spinlocks do), lock and unlock, to force the kernel to
90  * schedule if there's contention:
91  */
rt_spin_lock_unlock(spinlock_t * lock)92 void __sched rt_spin_lock_unlock(spinlock_t *lock)
93 {
94 	spin_lock(lock);
95 	spin_unlock(lock);
96 }
97 EXPORT_SYMBOL(rt_spin_lock_unlock);
98 
__rt_spin_trylock(spinlock_t * lock)99 static __always_inline int __rt_spin_trylock(spinlock_t *lock)
100 {
101 	int ret = 1;
102 
103 	if (unlikely(!rt_mutex_cmpxchg_acquire(&lock->lock, NULL, current)))
104 		ret = rt_mutex_slowtrylock(&lock->lock);
105 
106 	if (ret) {
107 		spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
108 		rcu_read_lock();
109 		migrate_disable();
110 	}
111 	return ret;
112 }
113 
rt_spin_trylock(spinlock_t * lock)114 int __sched rt_spin_trylock(spinlock_t *lock)
115 {
116 	return __rt_spin_trylock(lock);
117 }
118 EXPORT_SYMBOL(rt_spin_trylock);
119 
rt_spin_trylock_bh(spinlock_t * lock)120 int __sched rt_spin_trylock_bh(spinlock_t *lock)
121 {
122 	int ret;
123 
124 	local_bh_disable();
125 	ret = __rt_spin_trylock(lock);
126 	if (!ret)
127 		local_bh_enable();
128 	return ret;
129 }
130 EXPORT_SYMBOL(rt_spin_trylock_bh);
131 
132 #ifdef CONFIG_DEBUG_LOCK_ALLOC
__rt_spin_lock_init(spinlock_t * lock,const char * name,struct lock_class_key * key,bool percpu)133 void __rt_spin_lock_init(spinlock_t *lock, const char *name,
134 			 struct lock_class_key *key, bool percpu)
135 {
136 	u8 type = percpu ? LD_LOCK_PERCPU : LD_LOCK_NORMAL;
137 
138 	debug_check_no_locks_freed((void *)lock, sizeof(*lock));
139 	lockdep_init_map_type(&lock->dep_map, name, key, 0, LD_WAIT_CONFIG,
140 			      LD_WAIT_INV, type);
141 }
142 EXPORT_SYMBOL(__rt_spin_lock_init);
143 #endif
144 
145 /*
146  * RT-specific reader/writer locks
147  */
148 #define rwbase_set_and_save_current_state(state)	\
149 	current_save_and_set_rtlock_wait_state()
150 
151 #define rwbase_restore_current_state()			\
152 	current_restore_rtlock_saved_state()
153 
154 static __always_inline int
rwbase_rtmutex_lock_state(struct rt_mutex_base * rtm,unsigned int state)155 rwbase_rtmutex_lock_state(struct rt_mutex_base *rtm, unsigned int state)
156 {
157 	if (unlikely(!rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
158 		rtlock_slowlock(rtm);
159 	return 0;
160 }
161 
162 static __always_inline int
rwbase_rtmutex_slowlock_locked(struct rt_mutex_base * rtm,unsigned int state)163 rwbase_rtmutex_slowlock_locked(struct rt_mutex_base *rtm, unsigned int state)
164 {
165 	rtlock_slowlock_locked(rtm);
166 	return 0;
167 }
168 
rwbase_rtmutex_unlock(struct rt_mutex_base * rtm)169 static __always_inline void rwbase_rtmutex_unlock(struct rt_mutex_base *rtm)
170 {
171 	if (likely(rt_mutex_cmpxchg_acquire(rtm, current, NULL)))
172 		return;
173 
174 	rt_mutex_slowunlock(rtm);
175 }
176 
rwbase_rtmutex_trylock(struct rt_mutex_base * rtm)177 static __always_inline int  rwbase_rtmutex_trylock(struct rt_mutex_base *rtm)
178 {
179 	if (likely(rt_mutex_cmpxchg_acquire(rtm, NULL, current)))
180 		return 1;
181 
182 	return rt_mutex_slowtrylock(rtm);
183 }
184 
185 #define rwbase_signal_pending_state(state, current)	(0)
186 
187 #define rwbase_schedule()				\
188 	schedule_rtlock()
189 
190 #include "rwbase_rt.c"
191 /*
192  * The common functions which get wrapped into the rwlock API.
193  */
rt_read_trylock(rwlock_t * rwlock)194 int __sched rt_read_trylock(rwlock_t *rwlock)
195 {
196 	int ret;
197 
198 	ret = rwbase_read_trylock(&rwlock->rwbase);
199 	if (ret) {
200 		rwlock_acquire_read(&rwlock->dep_map, 0, 1, _RET_IP_);
201 		rcu_read_lock();
202 		migrate_disable();
203 	}
204 	return ret;
205 }
206 EXPORT_SYMBOL(rt_read_trylock);
207 
rt_write_trylock(rwlock_t * rwlock)208 int __sched rt_write_trylock(rwlock_t *rwlock)
209 {
210 	int ret;
211 
212 	ret = rwbase_write_trylock(&rwlock->rwbase);
213 	if (ret) {
214 		rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_);
215 		rcu_read_lock();
216 		migrate_disable();
217 	}
218 	return ret;
219 }
220 EXPORT_SYMBOL(rt_write_trylock);
221 
rt_read_lock(rwlock_t * rwlock)222 void __sched rt_read_lock(rwlock_t *rwlock)
223 {
224 	rtlock_might_resched();
225 	rwlock_acquire_read(&rwlock->dep_map, 0, 0, _RET_IP_);
226 	rwbase_read_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
227 	rcu_read_lock();
228 	migrate_disable();
229 }
230 EXPORT_SYMBOL(rt_read_lock);
231 
rt_write_lock(rwlock_t * rwlock)232 void __sched rt_write_lock(rwlock_t *rwlock)
233 {
234 	rtlock_might_resched();
235 	rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_);
236 	rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
237 	rcu_read_lock();
238 	migrate_disable();
239 }
240 EXPORT_SYMBOL(rt_write_lock);
241 
242 #ifdef CONFIG_DEBUG_LOCK_ALLOC
rt_write_lock_nested(rwlock_t * rwlock,int subclass)243 void __sched rt_write_lock_nested(rwlock_t *rwlock, int subclass)
244 {
245 	rtlock_might_resched();
246 	rwlock_acquire(&rwlock->dep_map, subclass, 0, _RET_IP_);
247 	rwbase_write_lock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
248 	rcu_read_lock();
249 	migrate_disable();
250 }
251 EXPORT_SYMBOL(rt_write_lock_nested);
252 #endif
253 
rt_read_unlock(rwlock_t * rwlock)254 void __sched rt_read_unlock(rwlock_t *rwlock)
255 {
256 	rwlock_release(&rwlock->dep_map, _RET_IP_);
257 	migrate_enable();
258 	rcu_read_unlock();
259 	rwbase_read_unlock(&rwlock->rwbase, TASK_RTLOCK_WAIT);
260 }
261 EXPORT_SYMBOL(rt_read_unlock);
262 
rt_write_unlock(rwlock_t * rwlock)263 void __sched rt_write_unlock(rwlock_t *rwlock)
264 {
265 	rwlock_release(&rwlock->dep_map, _RET_IP_);
266 	rcu_read_unlock();
267 	migrate_enable();
268 	rwbase_write_unlock(&rwlock->rwbase);
269 }
270 EXPORT_SYMBOL(rt_write_unlock);
271 
272 #ifdef CONFIG_DEBUG_LOCK_ALLOC
__rt_rwlock_init(rwlock_t * rwlock,const char * name,struct lock_class_key * key)273 void __rt_rwlock_init(rwlock_t *rwlock, const char *name,
274 		      struct lock_class_key *key)
275 {
276 	debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock));
277 	lockdep_init_map_wait(&rwlock->dep_map, name, key, 0, LD_WAIT_CONFIG);
278 }
279 EXPORT_SYMBOL(__rt_rwlock_init);
280 #endif
281