Lines Matching refs:mutex
35 lll_mutex_lock_optimized (pthread_mutex_t *mutex) in lll_mutex_lock_optimized() argument
44 int private = PTHREAD_MUTEX_PSHARED (mutex); in lll_mutex_lock_optimized()
45 if (private == LLL_PRIVATE && SINGLE_THREAD_P && mutex->__data.__lock == 0) in lll_mutex_lock_optimized()
46 mutex->__data.__lock = 1; in lll_mutex_lock_optimized()
48 lll_lock (mutex->__data.__lock, private); in lll_mutex_lock_optimized()
51 # define LLL_MUTEX_LOCK(mutex) \ argument
52 lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
53 # define LLL_MUTEX_LOCK_OPTIMIZED(mutex) lll_mutex_lock_optimized (mutex) argument
54 # define LLL_MUTEX_TRYLOCK(mutex) \ argument
55 lll_trylock ((mutex)->__data.__lock)
57 # define LLL_MUTEX_LOCK_ELISION(mutex) \ argument
58 lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \
59 PTHREAD_MUTEX_PSHARED (mutex))
60 # define LLL_MUTEX_TRYLOCK_ELISION(mutex) \ argument
61 lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \
62 PTHREAD_MUTEX_PSHARED (mutex))
68 # define LLL_MUTEX_READ_LOCK(mutex) \ argument
69 atomic_load_relaxed (&(mutex)->__data.__lock)
72 static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
76 PTHREAD_MUTEX_LOCK (pthread_mutex_t *mutex) in PTHREAD_MUTEX_LOCK() argument
80 unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex); in PTHREAD_MUTEX_LOCK()
82 LIBC_PROBE (mutex_entry, 1, mutex); in PTHREAD_MUTEX_LOCK()
86 return __pthread_mutex_lock_full (mutex); in PTHREAD_MUTEX_LOCK()
90 FORCE_ELISION (mutex, goto elision); in PTHREAD_MUTEX_LOCK()
93 LLL_MUTEX_LOCK_OPTIMIZED (mutex); in PTHREAD_MUTEX_LOCK()
94 assert (mutex->__data.__owner == 0); in PTHREAD_MUTEX_LOCK()
105 return LLL_MUTEX_LOCK_ELISION (mutex); in PTHREAD_MUTEX_LOCK()
108 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) in PTHREAD_MUTEX_LOCK()
115 if (mutex->__data.__owner == id) in PTHREAD_MUTEX_LOCK()
118 if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) in PTHREAD_MUTEX_LOCK()
122 ++mutex->__data.__count; in PTHREAD_MUTEX_LOCK()
128 LLL_MUTEX_LOCK_OPTIMIZED (mutex); in PTHREAD_MUTEX_LOCK()
130 assert (mutex->__data.__owner == 0); in PTHREAD_MUTEX_LOCK()
131 mutex->__data.__count = 1; in PTHREAD_MUTEX_LOCK()
133 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) in PTHREAD_MUTEX_LOCK()
136 if (LLL_MUTEX_TRYLOCK (mutex) != 0) in PTHREAD_MUTEX_LOCK()
140 mutex->__data.__spins * 2 + 10); in PTHREAD_MUTEX_LOCK()
153 LLL_MUTEX_LOCK (mutex); in PTHREAD_MUTEX_LOCK()
162 while (LLL_MUTEX_READ_LOCK (mutex) != 0 in PTHREAD_MUTEX_LOCK()
163 || LLL_MUTEX_TRYLOCK (mutex) != 0); in PTHREAD_MUTEX_LOCK()
165 mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8; in PTHREAD_MUTEX_LOCK()
167 assert (mutex->__data.__owner == 0); in PTHREAD_MUTEX_LOCK()
172 assert (PTHREAD_MUTEX_TYPE (mutex) == PTHREAD_MUTEX_ERRORCHECK_NP); in PTHREAD_MUTEX_LOCK()
174 if (__glibc_unlikely (mutex->__data.__owner == id)) in PTHREAD_MUTEX_LOCK()
182 mutex->__data.__owner = id; in PTHREAD_MUTEX_LOCK()
184 ++mutex->__data.__nusers; in PTHREAD_MUTEX_LOCK()
187 LIBC_PROBE (mutex_acquired, 1, mutex); in PTHREAD_MUTEX_LOCK()
193 __pthread_mutex_lock_full (pthread_mutex_t *mutex) in __pthread_mutex_lock_full() argument
198 switch (PTHREAD_MUTEX_TYPE (mutex)) in __pthread_mutex_lock_full()
205 &mutex->__data.__list.__next); in __pthread_mutex_lock_full()
210 oldval = mutex->__data.__lock; in __pthread_mutex_lock_full()
226 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, in __pthread_mutex_lock_full()
245 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, in __pthread_mutex_lock_full()
255 mutex->__data.__count = 1; in __pthread_mutex_lock_full()
257 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT; in __pthread_mutex_lock_full()
262 ENQUEUE_MUTEX (mutex); in __pthread_mutex_lock_full()
274 --mutex->__data.__nusers; in __pthread_mutex_lock_full()
283 int kind = PTHREAD_MUTEX_TYPE (mutex); in __pthread_mutex_lock_full()
301 if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) in __pthread_mutex_lock_full()
305 ++mutex->__data.__count; in __pthread_mutex_lock_full()
319 (&mutex->__data.__lock, oldval | FUTEX_WAITERS, oldval); in __pthread_mutex_lock_full()
335 futex_wait ((unsigned int *) &mutex->__data.__lock, oldval, in __pthread_mutex_lock_full()
336 PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); in __pthread_mutex_lock_full()
337 oldval = mutex->__data.__lock; in __pthread_mutex_lock_full()
341 if (__builtin_expect (mutex->__data.__owner in __pthread_mutex_lock_full()
345 mutex->__data.__count = 0; in __pthread_mutex_lock_full()
346 int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex); in __pthread_mutex_lock_full()
347 lll_unlock (mutex->__data.__lock, private); in __pthread_mutex_lock_full()
354 mutex->__data.__count = 1; in __pthread_mutex_lock_full()
358 ENQUEUE_MUTEX (mutex); in __pthread_mutex_lock_full()
381 int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind)); in __pthread_mutex_lock_full()
390 (void *) (((uintptr_t) &mutex->__data.__list.__next) in __pthread_mutex_lock_full()
397 oldval = mutex->__data.__lock; in __pthread_mutex_lock_full()
417 if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) in __pthread_mutex_lock_full()
421 ++mutex->__data.__count; in __pthread_mutex_lock_full()
431 oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, in __pthread_mutex_lock_full()
439 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex) in __pthread_mutex_lock_full()
440 : PTHREAD_MUTEX_PSHARED (mutex)); in __pthread_mutex_lock_full()
441 int e = __futex_lock_pi64 (&mutex->__data.__lock, 0 /* ununsed */, in __pthread_mutex_lock_full()
458 oldval = mutex->__data.__lock; in __pthread_mutex_lock_full()
465 atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED); in __pthread_mutex_lock_full()
468 mutex->__data.__count = 1; in __pthread_mutex_lock_full()
470 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT; in __pthread_mutex_lock_full()
475 ENQUEUE_MUTEX_PI (mutex); in __pthread_mutex_lock_full()
486 --mutex->__data.__nusers; in __pthread_mutex_lock_full()
493 && __builtin_expect (mutex->__data.__owner in __pthread_mutex_lock_full()
497 mutex->__data.__count = 0; in __pthread_mutex_lock_full()
499 futex_unlock_pi ((unsigned int *) &mutex->__data.__lock, in __pthread_mutex_lock_full()
500 PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); in __pthread_mutex_lock_full()
508 mutex->__data.__count = 1; in __pthread_mutex_lock_full()
514 ENQUEUE_MUTEX_PI (mutex); in __pthread_mutex_lock_full()
530 int kind = atomic_load_relaxed (&(mutex->__data.__kind)) in __pthread_mutex_lock_full()
533 oldval = mutex->__data.__lock; in __pthread_mutex_lock_full()
536 if (mutex->__data.__owner == id) in __pthread_mutex_lock_full()
544 if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) in __pthread_mutex_lock_full()
548 ++mutex->__data.__count; in __pthread_mutex_lock_full()
575 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, in __pthread_mutex_lock_full()
589 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, in __pthread_mutex_lock_full()
597 futex_wait ((unsigned int * ) &mutex->__data.__lock, in __pthread_mutex_lock_full()
599 PTHREAD_MUTEX_PSHARED (mutex)); in __pthread_mutex_lock_full()
601 while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, in __pthread_mutex_lock_full()
607 assert (mutex->__data.__owner == 0); in __pthread_mutex_lock_full()
608 mutex->__data.__count = 1; in __pthread_mutex_lock_full()
618 mutex->__data.__owner = id; in __pthread_mutex_lock_full()
620 ++mutex->__data.__nusers; in __pthread_mutex_lock_full()
623 LIBC_PROBE (mutex_acquired, 1, mutex); in __pthread_mutex_lock_full()
645 __pthread_mutex_cond_lock_adjust (pthread_mutex_t *mutex) in __pthread_mutex_cond_lock_adjust() argument
649 int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind)); in __pthread_mutex_cond_lock_adjust()
656 mutex->__data.__owner = id; in __pthread_mutex_cond_lock_adjust()
659 ++mutex->__data.__count; in __pthread_mutex_cond_lock_adjust()