111 		atomic_long_read(&(sem)->owner), (long)current,	\
177 	atomic_long_set(&sem->owner, (long)current);  in rwsem_set_owner()
182 	atomic_long_set(&sem->owner, 0);  in rwsem_clear_owner()
190 	return atomic_long_read(&sem->owner) & flags;  in rwsem_test_oflags()
204 					    struct task_struct *owner)  in __rwsem_set_reader_owned()  argument
206 	unsigned long val = (unsigned long)owner | RWSEM_READER_OWNED |  in __rwsem_set_reader_owned()
207 		(atomic_long_read(&sem->owner) & RWSEM_RD_NONSPINNABLE);  in __rwsem_set_reader_owned()
209 	atomic_long_set(&sem->owner, val);  in __rwsem_set_reader_owned()
243 	unsigned long val = atomic_long_read(&sem->owner);  in rwsem_clear_reader_owned()
246 		if (atomic_long_try_cmpxchg(&sem->owner, &val,  in rwsem_clear_reader_owned()
263 	unsigned long owner = atomic_long_read(&sem->owner);  in rwsem_set_nonspinnable()  local
266 		if (!(owner & RWSEM_READER_OWNED))  in rwsem_set_nonspinnable()
268 		if (owner & RWSEM_NONSPINNABLE)  in rwsem_set_nonspinnable()
270 	} while (!atomic_long_try_cmpxchg(&sem->owner, &owner,  in rwsem_set_nonspinnable()
271 					  owner | RWSEM_NONSPINNABLE));  in rwsem_set_nonspinnable()
288 		(atomic_long_read(&sem->owner) & ~RWSEM_OWNER_FLAGS_MASK);  in rwsem_owner()
298 	unsigned long owner = atomic_long_read(&sem->owner);  in rwsem_owner_flags()  local
300 	*pflags = owner & RWSEM_OWNER_FLAGS_MASK;  in rwsem_owner_flags()
301 	return (struct task_struct *)(owner & ~RWSEM_OWNER_FLAGS_MASK);  in rwsem_owner_flags()
340 	atomic_long_set(&sem->owner, 0L);  in __init_rwsem()
445 		struct task_struct *owner;  in rwsem_mark_wake()  local
470 		owner = waiter->task;  in rwsem_mark_wake()
472 			owner = (void *)((unsigned long)owner | RWSEM_RD_NONSPINNABLE);  in rwsem_mark_wake()
475 		__rwsem_set_reader_owned(sem, owner);  in rwsem_mark_wake()
647 static inline bool owner_on_cpu(struct task_struct *owner)  in owner_on_cpu()  argument
653 	return owner->on_cpu && !vcpu_is_preempted(task_cpu(owner));  in owner_on_cpu()
659 	struct task_struct *owner;  in rwsem_can_spin_on_owner()  local
672 	owner = rwsem_owner_flags(sem, &flags);  in rwsem_can_spin_on_owner()
677 	    (owner && !(flags & RWSEM_READER_OWNED) && !owner_on_cpu(owner)))  in rwsem_can_spin_on_owner()
706 rwsem_owner_state(struct task_struct *owner, unsigned long flags, unsigned long nonspinnable)  in rwsem_owner_state()  argument
714 	return owner ? OWNER_WRITER : OWNER_NULL;  in rwsem_owner_state()
720 	struct task_struct *new, *owner;  in rwsem_spin_on_owner()  local
724 	owner = rwsem_owner_flags(sem, &flags);  in rwsem_spin_on_owner()
725 	state = rwsem_owner_state(owner, flags, nonspinnable);  in rwsem_spin_on_owner()
738 		if ((new != owner) || (new_flags != flags)) {  in rwsem_spin_on_owner()
751 		if (need_resched() || !owner_on_cpu(owner)) {  in rwsem_spin_on_owner()
925 		atomic_long_andnot(RWSEM_WR_NONSPINNABLE, &sem->owner);  in clear_wr_nonspinnable()
950 	unsigned long owner = atomic_long_read(&sem->owner);  in rwsem_reader_phase_trylock()  local
952 	if (!(owner & RWSEM_READER_OWNED))  in rwsem_reader_phase_trylock()
955 	if (((owner ^ last_rowner) & ~RWSEM_OWNER_FLAGS_MASK) &&  in rwsem_reader_phase_trylock()
1006 	waiter.last_rowner = atomic_long_read(&sem->owner);  in rwsem_down_read_slowpath()
1130 		atomic_long_or(RWSEM_RD_NONSPINNABLE, &sem->owner);  in rwsem_disable_reader_optspin()
1160 	disable_rspin = atomic_long_read(&sem->owner) & RWSEM_NONSPINNABLE;  in rwsem_down_write_slowpath()