487 static int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key,  in get_futex_key()  argument
490 	unsigned long address = (unsigned long)uaddr;  in get_futex_key()
504 	if (unlikely(!access_ok(uaddr, sizeof(u32))))  in get_futex_key()
676 static int fault_in_user_writeable(u32 __user *uaddr)  in fault_in_user_writeable()  argument
682 	ret = fixup_user_fault(mm, (unsigned long)uaddr,  in fault_in_user_writeable()
708 static int cmpxchg_futex_value_locked(u32 *curval, u32 __user *uaddr,  in cmpxchg_futex_value_locked()  argument
714 	ret = futex_atomic_cmpxchg_inatomic(curval, uaddr, uval, newval);  in cmpxchg_futex_value_locked()
987 static int attach_to_pi_state(u32 __user *uaddr, u32 uval,  in attach_to_pi_state()  argument
1027 	if (get_futex_value_locked(&uval2, uaddr))  in attach_to_pi_state()
1136 static int handle_exit_race(u32 __user *uaddr, u32 uval,  in handle_exit_race()  argument
1177 	if (get_futex_value_locked(&uval2, uaddr))  in handle_exit_race()
1196 static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key,  in attach_to_pi_owner()  argument
1215 		return handle_exit_race(uaddr, uval, NULL);  in attach_to_pi_owner()
1234 		int ret = handle_exit_race(uaddr, uval, p);  in attach_to_pi_owner()
1286 static int lookup_pi_state(u32 __user *uaddr, u32 uval,  in lookup_pi_state()  argument
1298 		return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);  in lookup_pi_state()
1304 	return attach_to_pi_owner(uaddr, uval, key, ps, exiting);  in lookup_pi_state()
1307 static int lock_pi_update_atomic(u32 __user *uaddr, u32 uval, u32 newval)  in lock_pi_update_atomic()  argument
1315 	err = cmpxchg_futex_value_locked(&curval, uaddr, uval, newval);  in lock_pi_update_atomic()
1347 static int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,  in futex_lock_pi_atomic()  argument
1362 	if (get_futex_value_locked(&uval, uaddr))  in futex_lock_pi_atomic()
1383 		return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);  in futex_lock_pi_atomic()
1403 		ret = lock_pi_update_atomic(uaddr, uval, newval);  in futex_lock_pi_atomic()
1414 	ret = lock_pi_update_atomic(uaddr, uval, newval);  in futex_lock_pi_atomic()
1422 	return attach_to_pi_owner(uaddr, newval, key, ps, exiting);  in futex_lock_pi_atomic()
1478 static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_state)  in wake_futex_pi()  argument
1512 	ret = cmpxchg_futex_value_locked(&curval, uaddr, uval, newval);  in wake_futex_pi()
1584 futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)  in futex_wake()  argument
1595 	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_READ);  in futex_wake()
1629 static int futex_atomic_op_inuser(unsigned int encoded_op, u32 __user *uaddr)  in futex_atomic_op_inuser()  argument
1652 	ret = arch_futex_atomic_op_inuser(op, oparg, &oldval, uaddr);  in futex_atomic_op_inuser()
2328 static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,  in fixup_pi_state_owner()  argument
2418 	err = get_futex_value_locked(&uval, uaddr);  in fixup_pi_state_owner()
2425 		err = cmpxchg_futex_value_locked(&curval, uaddr, uval, newval);  in fixup_pi_state_owner()
2474 		ret = fault_in_user_writeable(uaddr);  in fixup_pi_state_owner()
2526 static int fixup_owner(u32 __user *uaddr, struct futex_q *q, int locked)  in fixup_owner()  argument
2540 			ret = fixup_pi_state_owner(uaddr, q, current);  in fixup_owner()
2553 		ret = fixup_pi_state_owner(uaddr, q, NULL);  in fixup_owner()
2626 static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,  in futex_wait_setup()  argument
2651 	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, FUTEX_READ);  in futex_wait_setup()
2658 	ret = get_futex_value_locked(&uval, uaddr);  in futex_wait_setup()
2663 		ret = get_user(uval, uaddr);  in futex_wait_setup()
2681 static int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,  in futex_wait()  argument
2701 	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);  in futex_wait()
2730 	restart->futex.uaddr = uaddr;  in futex_wait()
2749 	u32 __user *uaddr = restart->futex.uaddr;  in futex_wait_restart()  local
2758 	return (long)futex_wait(uaddr, restart->futex.flags,  in futex_wait_restart()
2772 static int futex_lock_pi(u32 __user *uaddr, unsigned int flags,  in futex_lock_pi()  argument
2792 	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key, FUTEX_WRITE);  in futex_lock_pi()
2799 	ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current,  in futex_lock_pi()
2904 	res = fixup_owner(uaddr, &q, !ret);  in futex_lock_pi()
2944 	ret = fault_in_user_writeable(uaddr);  in futex_lock_pi()
2959 static int futex_unlock_pi(u32 __user *uaddr, unsigned int flags)  in futex_unlock_pi()  argument
2971 	if (get_user(uval, uaddr))  in futex_unlock_pi()
2979 	ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_WRITE);  in futex_unlock_pi()
3021 		ret = wake_futex_pi(uaddr, uval, pi_state);  in futex_unlock_pi()
3056 	if ((ret = cmpxchg_futex_value_locked(&curval, uaddr, uval, 0))) {  in futex_unlock_pi()
3087 	ret = fault_in_user_writeable(uaddr);  in futex_unlock_pi()
3183 static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags,  in futex_wait_requeue_pi()  argument
3198 	if (uaddr == uaddr2)  in futex_wait_requeue_pi()
3225 	ret = futex_wait_setup(uaddr, val, flags, &q, &hb);  in futex_wait_requeue_pi()
3433 static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr,  in handle_futex_death()  argument
3440 	if ((((unsigned long)uaddr) % sizeof(*uaddr)) != 0)  in handle_futex_death()
3444 	if (get_user(uval, uaddr))  in handle_futex_death()
3479 		futex_wake(uaddr, 1, 1, FUTEX_BITSET_MATCH_ANY);  in handle_futex_death()
3507 	if ((err = cmpxchg_futex_value_locked(&nval, uaddr, uval, mval))) {  in handle_futex_death()
3510 			if (fault_in_user_writeable(uaddr))  in handle_futex_death()
3532 		futex_wake(uaddr, 1, 1, FUTEX_BITSET_MATCH_ANY);  in handle_futex_death()
3734 long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,  in do_futex()  argument
3765 		return futex_wait(uaddr, flags, val, timeout, val3);  in do_futex()
3770 		return futex_wake(uaddr, flags, val, val3);  in do_futex()
3772 		return futex_requeue(uaddr, flags, uaddr2, val, val2, NULL, 0);  in do_futex()
3774 		return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 0);  in do_futex()
3776 		return futex_wake_op(uaddr, flags, uaddr2, val, val2, val3);  in do_futex()
3778 		return futex_lock_pi(uaddr, flags, timeout, 0);  in do_futex()
3780 		return futex_unlock_pi(uaddr, flags);  in do_futex()
3782 		return futex_lock_pi(uaddr, flags, NULL, 1);  in do_futex()
3785 		return futex_wait_requeue_pi(uaddr, flags, val, timeout, val3,  in do_futex()
3788 		return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 1);  in do_futex()
3794 SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,  in SYSCALL_DEFINE6()  argument
3828 	return do_futex(uaddr, op, val, tp, uaddr2, val2, val3);  in SYSCALL_DEFINE6()
3852 	void __user *uaddr = compat_ptr(base + futex_offset);  in futex_uaddr()  local
3854 	return uaddr;  in futex_uaddr()
3908 			void __user *uaddr = futex_uaddr(entry, futex_offset);  in compat_exit_robust_list()  local
3910 			if (handle_futex_death(uaddr, curr, pi,  in compat_exit_robust_list()
3928 		void __user *uaddr = futex_uaddr(pending, futex_offset);  in compat_exit_robust_list()  local
3930 		handle_futex_death(uaddr, curr, pip, HANDLE_DEATH_PENDING);  in compat_exit_robust_list()
3990 SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val,  in SYSCALL_DEFINE6()  argument
4018 	return do_futex(uaddr, op, val, tp, uaddr2, val2, val3);  in SYSCALL_DEFINE6()