202 static int attach_to_pi_state(u32 __user *uaddr, u32 uval,  in attach_to_pi_state()  argument
206 	pid_t pid = uval & FUTEX_TID_MASK;  in attach_to_pi_state()
245 	if (uval != uval2)  in attach_to_pi_state()
251 	if (uval & FUTEX_OWNER_DIED) {  in attach_to_pi_state()
320 static int handle_exit_race(u32 __user *uaddr, u32 uval,  in handle_exit_race()  argument
365 	if (uval2 != uval)  in handle_exit_race()
410 static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key,  in attach_to_pi_owner()  argument
414 	pid_t pid = uval & FUTEX_TID_MASK;  in attach_to_pi_owner()
428 		return handle_exit_race(uaddr, uval, NULL);  in attach_to_pi_owner()
447 		int ret = handle_exit_race(uaddr, uval, p);  in attach_to_pi_owner()
474 static int lock_pi_update_atomic(u32 __user *uaddr, u32 uval, u32 newval)  in lock_pi_update_atomic()  argument
482 	err = futex_cmpxchg_value_locked(&curval, uaddr, uval, newval);  in lock_pi_update_atomic()
487 	return curval != uval ? -EAGAIN : 0;  in lock_pi_update_atomic()
521 	u32 uval, newval, vpid = task_pid_vnr(task);  in futex_lock_pi_atomic()  local
529 	if (futex_get_value_locked(&uval, uaddr))  in futex_lock_pi_atomic()
538 	if ((unlikely((uval & FUTEX_TID_MASK) == vpid)))  in futex_lock_pi_atomic()
550 		return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);  in futex_lock_pi_atomic()
558 	if (!(uval & FUTEX_TID_MASK)) {  in futex_lock_pi_atomic()
563 		newval = uval & FUTEX_OWNER_DIED;  in futex_lock_pi_atomic()
570 		ret = lock_pi_update_atomic(uaddr, uval, newval);  in futex_lock_pi_atomic()
598 	newval = uval | FUTEX_WAITERS;  in futex_lock_pi_atomic()
599 	ret = lock_pi_update_atomic(uaddr, uval, newval);  in futex_lock_pi_atomic()
613 static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_state)  in wake_futex_pi()  argument
650 	ret = futex_cmpxchg_value_locked(&curval, uaddr, uval, newval);  in wake_futex_pi()
651 	if (!ret && (curval != uval)) {  in wake_futex_pi()
658 		if ((FUTEX_TID_MASK & curval) == uval)  in wake_futex_pi()
688 	u32 uval, curval, newval, newtid;  in __fixup_pi_state_owner()  local
765 	err = futex_get_value_locked(&uval, uaddr);  in __fixup_pi_state_owner()
770 		newval = (uval & FUTEX_OWNER_DIED) | newtid;  in __fixup_pi_state_owner()
772 		err = futex_cmpxchg_value_locked(&curval, uaddr, uval, newval);  in __fixup_pi_state_owner()
776 		if (curval == uval)  in __fixup_pi_state_owner()
778 		uval = curval;  in __fixup_pi_state_owner()
1102 	u32 curval, uval, vpid = task_pid_vnr(current);  in futex_unlock_pi()  local
1112 	if (get_user(uval, uaddr))  in futex_unlock_pi()
1117 	if ((uval & FUTEX_TID_MASK) != vpid)  in futex_unlock_pi()
1162 		ret = wake_futex_pi(uaddr, uval, pi_state);  in futex_unlock_pi()
1197 	if ((ret = futex_cmpxchg_value_locked(&curval, uaddr, uval, 0))) {  in futex_unlock_pi()
1215 	ret = (curval == uval) ? 0 : -EAGAIN;  in futex_unlock_pi()