1 /* rwsem.h: R/W semaphores implemented using XADD/CMPXCHG for x86_64+
2 *
3 * Written by David Howells (dhowells@redhat.com).
4 * Ported by Andi Kleen <ak@suse.de> to x86-64.
5 *
6 * Derived from asm-i386/semaphore.h and asm-i386/rwsem.h
7 *
8 *
9 * The MSW of the count is the negated number of active writers and waiting
10 * lockers, and the LSW is the total number of active locks
11 *
12 * The lock count is initialized to 0 (no active and no waiting lockers).
13 *
14 * When a writer subtracts WRITE_BIAS, it'll get 0xffff0001 for the case of an
15 * uncontended lock. This can be determined because XADD returns the old value.
16 * Readers increment by 1 and see a positive value when uncontended, negative
17 * if there are writers (and maybe) readers waiting (in which case it goes to
18 * sleep).
19 *
20 * The value of WAITING_BIAS supports up to 32766 waiting processes. This can
21 * be extended to 65534 by manually checking the whole MSW rather than relying
22 * on the S flag.
23 *
24 * The value of ACTIVE_BIAS supports up to 65535 active processes.
25 *
26 * This should be totally fair - if anything is waiting, a process that wants a
27 * lock will go to the back of the queue. When the currently active lock is
28 * released, if there's a writer at the front of the queue, then that and only
29 * that will be woken up; if there's a bunch of consequtive readers at the
30 * front, then they'll all be woken up, but no other readers will be.
31 */
32
33 #ifndef _X8664_RWSEM_H
34 #define _X8664_RWSEM_H
35
36 #ifndef _LINUX_RWSEM_H
37 #error please dont include asm/rwsem.h directly, use linux/rwsem.h instead
38 #endif
39
40 #ifdef __KERNEL__
41
42 #include <linux/list.h>
43 #include <linux/spinlock.h>
44
45 struct rwsem_waiter;
46
47 extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem);
48 extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem);
49 extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *);
50
51 /*
52 * the semaphore definition
53 */
54 struct rw_semaphore {
55 signed int count;
56 #define RWSEM_UNLOCKED_VALUE 0x00000000
57 #define RWSEM_ACTIVE_BIAS 0x00000001
58 #define RWSEM_ACTIVE_MASK 0x0000ffff
59 #define RWSEM_WAITING_BIAS (-0x00010000)
60 #define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
61 #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
62 spinlock_t wait_lock;
63 struct list_head wait_list;
64 #if RWSEM_DEBUG
65 int debug;
66 #endif
67 };
68
69 /*
70 * initialisation
71 */
72 #if RWSEM_DEBUG
73 #define __RWSEM_DEBUG_INIT , 0
74 #else
75 #define __RWSEM_DEBUG_INIT /* */
76 #endif
77
78 #define __RWSEM_INITIALIZER(name) \
79 { RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, LIST_HEAD_INIT((name).wait_list) \
80 __RWSEM_DEBUG_INIT }
81
82 #define DECLARE_RWSEM(name) \
83 struct rw_semaphore name = __RWSEM_INITIALIZER(name)
84
init_rwsem(struct rw_semaphore * sem)85 static inline void init_rwsem(struct rw_semaphore *sem)
86 {
87 sem->count = RWSEM_UNLOCKED_VALUE;
88 spin_lock_init(&sem->wait_lock);
89 INIT_LIST_HEAD(&sem->wait_list);
90 #if RWSEM_DEBUG
91 sem->debug = 0;
92 #endif
93 }
94
95 /*
96 * lock for reading
97 */
__down_read(struct rw_semaphore * sem)98 static inline void __down_read(struct rw_semaphore *sem)
99 {
100 __asm__ __volatile__(
101 "# beginning down_read\n\t"
102 LOCK_PREFIX " incl (%%rdi)\n\t" /* adds 0x00000001, returns the old value */
103 " js 2f\n\t" /* jump if we weren't granted the lock */
104 "1:\n\t"
105 LOCK_SECTION_START("") \
106 "2:\n\t"
107 " call rwsem_down_read_failed_thunk\n\t"
108 " jmp 1b\n"
109 LOCK_SECTION_END \
110 "# ending down_read\n\t"
111 : "+m"(sem->count)
112 : "D"(sem)
113 : "memory", "cc");
114 }
115
116 /*
117 * lock for writing
118 */
__down_write(struct rw_semaphore * sem)119 static inline void __down_write(struct rw_semaphore *sem)
120 {
121 int tmp;
122
123 tmp = RWSEM_ACTIVE_WRITE_BIAS;
124 __asm__ __volatile__(
125 "# beginning down_write\n\t"
126 LOCK_PREFIX " xaddl %0,(%%rdi)\n\t" /* subtract 0x0000ffff, returns the old value */
127 " testl %0,%0\n\t" /* was the count 0 before? */
128 " jnz 2f\n\t" /* jump if we weren't granted the lock */
129 "1:\n\t"
130 LOCK_SECTION_START("")
131 "2:\n\t"
132 " call rwsem_down_write_failed_thunk\n\t"
133 " jmp 1b\n"
134 LOCK_SECTION_END
135 "# ending down_write"
136 : "=&r" (tmp)
137 : "0"(tmp), "D"(sem)
138 : "memory", "cc");
139 }
140
141 /*
142 * unlock after reading
143 */
__up_read(struct rw_semaphore * sem)144 static inline void __up_read(struct rw_semaphore *sem)
145 {
146 __s32 tmp = -RWSEM_ACTIVE_READ_BIAS;
147 __asm__ __volatile__(
148 "# beginning __up_read\n\t"
149 LOCK_PREFIX " xaddl %%edx,(%%rdi)\n\t" /* subtracts 1, returns the old value */
150 " js 2f\n\t" /* jump if the lock is being waited upon */
151 "1:\n\t"
152 LOCK_SECTION_START("")
153 "2:\n\t"
154 " decw %%dx\n\t" /* do nothing if still outstanding active readers */
155 " jnz 1b\n\t"
156 " call rwsem_wake_thunk\n\t"
157 " jmp 1b\n"
158 LOCK_SECTION_END
159 "# ending __up_read\n"
160 : "+m"(sem->count), "+d"(tmp)
161 : "D"(sem)
162 : "memory", "cc");
163 }
164
165 /*
166 * unlock after writing
167 */
__up_write(struct rw_semaphore * sem)168 static inline void __up_write(struct rw_semaphore *sem)
169 {
170 __asm__ __volatile__(
171 "# beginning __up_write\n\t"
172 " movl %2,%%edx\n\t"
173 LOCK_PREFIX " xaddl %%edx,(%%rdi)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */
174 " jnz 2f\n\t" /* jump if the lock is being waited upon */
175 "1:\n\t"
176 LOCK_SECTION_START("")
177 "2:\n\t"
178 " decw %%dx\n\t" /* did the active count reduce to 0? */
179 " jnz 1b\n\t" /* jump back if not */
180 " call rwsem_wake_thunk\n\t"
181 " jmp 1b\n"
182 LOCK_SECTION_END
183 "# ending __up_write\n"
184 : "+m"(sem->count)
185 : "D"(sem), "i"(-RWSEM_ACTIVE_WRITE_BIAS)
186 : "memory", "cc", "rdx");
187 }
188
189 /*
190 * implement atomic add functionality
191 */
rwsem_atomic_add(int delta,struct rw_semaphore * sem)192 static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)
193 {
194 __asm__ __volatile__(
195 LOCK_PREFIX "addl %1,%0"
196 :"=m"(sem->count)
197 :"ir"(delta), "m"(sem->count));
198 }
199
200 /*
201 * implement exchange and add functionality
202 */
rwsem_atomic_update(int delta,struct rw_semaphore * sem)203 static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem)
204 {
205 int tmp = delta;
206
207 __asm__ __volatile__(
208 LOCK_PREFIX "xaddl %0,(%2)"
209 : "=r"(tmp), "=m"(sem->count)
210 : "r"(sem), "m"(sem->count), "0" (tmp)
211 : "memory");
212
213 return tmp+delta;
214 }
215
216 #endif /* __KERNEL__ */
217 #endif /* _X8664_RWSEM_H */
218