110 		atomic_long_read(&(sem)->owner), (long)current,	\
176 	atomic_long_set(&sem->owner, (long)current);  in rwsem_set_owner()
181 	atomic_long_set(&sem->owner, 0);  in rwsem_clear_owner()
189 	return atomic_long_read(&sem->owner) & flags;  in rwsem_test_oflags()
203 					    struct task_struct *owner)  in __rwsem_set_reader_owned()  argument
205 	unsigned long val = (unsigned long)owner | RWSEM_READER_OWNED |  in __rwsem_set_reader_owned()
206 		(atomic_long_read(&sem->owner) & RWSEM_RD_NONSPINNABLE);  in __rwsem_set_reader_owned()
208 	atomic_long_set(&sem->owner, val);  in __rwsem_set_reader_owned()
242 	unsigned long val = atomic_long_read(&sem->owner);  in rwsem_clear_reader_owned()
245 		if (atomic_long_try_cmpxchg(&sem->owner, &val,  in rwsem_clear_reader_owned()
262 	unsigned long owner = atomic_long_read(&sem->owner);  in rwsem_set_nonspinnable()  local
265 		if (!(owner & RWSEM_READER_OWNED))  in rwsem_set_nonspinnable()
267 		if (owner & RWSEM_NONSPINNABLE)  in rwsem_set_nonspinnable()
269 	} while (!atomic_long_try_cmpxchg(&sem->owner, &owner,  in rwsem_set_nonspinnable()
270 					  owner | RWSEM_NONSPINNABLE));  in rwsem_set_nonspinnable()
287 		(atomic_long_read(&sem->owner) & ~RWSEM_OWNER_FLAGS_MASK);  in rwsem_owner()
297 	unsigned long owner = atomic_long_read(&sem->owner);  in rwsem_owner_flags()  local
299 	*pflags = owner & RWSEM_OWNER_FLAGS_MASK;  in rwsem_owner_flags()
300 	return (struct task_struct *)(owner & ~RWSEM_OWNER_FLAGS_MASK);  in rwsem_owner_flags()
339 	atomic_long_set(&sem->owner, 0L);  in __init_rwsem()
444 		struct task_struct *owner;  in rwsem_mark_wake()  local
469 		owner = waiter->task;  in rwsem_mark_wake()
471 			owner = (void *)((unsigned long)owner | RWSEM_RD_NONSPINNABLE);  in rwsem_mark_wake()
474 		__rwsem_set_reader_owned(sem, owner);  in rwsem_mark_wake()
646 static inline bool owner_on_cpu(struct task_struct *owner)  in owner_on_cpu()  argument
652 	return owner->on_cpu && !vcpu_is_preempted(task_cpu(owner));  in owner_on_cpu()
658 	struct task_struct *owner;  in rwsem_can_spin_on_owner()  local
669 	owner = rwsem_owner_flags(sem, &flags);  in rwsem_can_spin_on_owner()
674 	    (owner && !(flags & RWSEM_READER_OWNED) && !owner_on_cpu(owner)))  in rwsem_can_spin_on_owner()
703 rwsem_owner_state(struct task_struct *owner, unsigned long flags, unsigned long nonspinnable)  in rwsem_owner_state()  argument
711 	return owner ? OWNER_WRITER : OWNER_NULL;  in rwsem_owner_state()
717 	struct task_struct *new, *owner;  in rwsem_spin_on_owner()  local
721 	owner = rwsem_owner_flags(sem, &flags);  in rwsem_spin_on_owner()
722 	state = rwsem_owner_state(owner, flags, nonspinnable);  in rwsem_spin_on_owner()
735 		if ((new != owner) || (new_flags != flags)) {  in rwsem_spin_on_owner()
748 		if (need_resched() || !owner_on_cpu(owner)) {  in rwsem_spin_on_owner()
922 		atomic_long_andnot(RWSEM_WR_NONSPINNABLE, &sem->owner);  in clear_wr_nonspinnable()
947 	unsigned long owner = atomic_long_read(&sem->owner);  in rwsem_reader_phase_trylock()  local
949 	if (!(owner & RWSEM_READER_OWNED))  in rwsem_reader_phase_trylock()
952 	if (((owner ^ last_rowner) & ~RWSEM_OWNER_FLAGS_MASK) &&  in rwsem_reader_phase_trylock()
1003 	waiter.last_rowner = atomic_long_read(&sem->owner);  in rwsem_down_read_slowpath()
1127 		atomic_long_or(RWSEM_RD_NONSPINNABLE, &sem->owner);  in rwsem_disable_reader_optspin()
1157 	disable_rspin = atomic_long_read(&sem->owner) & RWSEM_NONSPINNABLE;  in rwsem_down_write_slowpath()