68 # define DEBUG_RWSEMS_WARN_ON(c, sem)	do {			\  argument
71 		#c, atomic_long_read(&(sem)->count),		\
72 		(unsigned long) sem->magic,			\
73 		atomic_long_read(&(sem)->owner), (long)current,	\
74 		list_empty(&(sem)->wait_list) ? "" : "not "))	\
78 # define DEBUG_RWSEMS_WARN_ON(c, sem)  argument
137 static inline void rwsem_set_owner(struct rw_semaphore *sem)  in rwsem_set_owner()  argument
139 	atomic_long_set(&sem->owner, (long)current);  in rwsem_set_owner()
142 static inline void rwsem_clear_owner(struct rw_semaphore *sem)  in rwsem_clear_owner()  argument
144 	atomic_long_set(&sem->owner, 0);  in rwsem_clear_owner()
150 static inline bool rwsem_test_oflags(struct rw_semaphore *sem, long flags)  in rwsem_test_oflags()  argument
152 	return atomic_long_read(&sem->owner) & flags;  in rwsem_test_oflags()
165 static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem,  in __rwsem_set_reader_owned()  argument
169 		(atomic_long_read(&sem->owner) & RWSEM_NONSPINNABLE);  in __rwsem_set_reader_owned()
171 	atomic_long_set(&sem->owner, val);  in __rwsem_set_reader_owned()
174 static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)  in rwsem_set_reader_owned()  argument
176 	__rwsem_set_reader_owned(sem, current);  in rwsem_set_reader_owned()
182 static inline bool is_rwsem_reader_owned(struct rw_semaphore *sem)  in is_rwsem_reader_owned()  argument
188 	long count = atomic_long_read(&sem->count);  in is_rwsem_reader_owned()
193 	return rwsem_test_oflags(sem, RWSEM_READER_OWNED);  in is_rwsem_reader_owned()
203 static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)  in rwsem_clear_reader_owned()  argument
205 	unsigned long val = atomic_long_read(&sem->owner);  in rwsem_clear_reader_owned()
208 		if (atomic_long_try_cmpxchg(&sem->owner, &val,  in rwsem_clear_reader_owned()
214 static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)  in rwsem_clear_reader_owned()  argument
223 static inline void rwsem_set_nonspinnable(struct rw_semaphore *sem)  in rwsem_set_nonspinnable()  argument
225 	unsigned long owner = atomic_long_read(&sem->owner);  in rwsem_set_nonspinnable()
232 	} while (!atomic_long_try_cmpxchg(&sem->owner, &owner,  in rwsem_set_nonspinnable()
236 static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp)  in rwsem_read_trylock()  argument
238 	*cntp = atomic_long_add_return_acquire(RWSEM_READER_BIAS, &sem->count);  in rwsem_read_trylock()
241 		rwsem_set_nonspinnable(sem);  in rwsem_read_trylock()
244 		rwsem_set_reader_owned(sem);  in rwsem_read_trylock()
251 static inline bool rwsem_write_trylock(struct rw_semaphore *sem)  in rwsem_write_trylock()  argument
255 	if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, RWSEM_WRITER_LOCKED)) {  in rwsem_write_trylock()
256 		rwsem_set_owner(sem);  in rwsem_write_trylock()
266 static inline struct task_struct *rwsem_owner(struct rw_semaphore *sem)  in rwsem_owner()  argument
269 		(atomic_long_read(&sem->owner) & ~RWSEM_OWNER_FLAGS_MASK);  in rwsem_owner()
277 rwsem_owner_flags(struct rw_semaphore *sem, unsigned long *pflags)  in rwsem_owner_flags()  argument
279 	unsigned long owner = atomic_long_read(&sem->owner);  in rwsem_owner_flags()
305 void __init_rwsem(struct rw_semaphore *sem, const char *name,  in __init_rwsem()  argument
312 	debug_check_no_locks_freed((void *)sem, sizeof(*sem));  in __init_rwsem()
313 	lockdep_init_map_wait(&sem->dep_map, name, key, 0, LD_WAIT_SLEEP);  in __init_rwsem()
316 	sem->magic = sem;  in __init_rwsem()
318 	atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE);  in __init_rwsem()
319 	raw_spin_lock_init(&sem->wait_lock);  in __init_rwsem()
320 	INIT_LIST_HEAD(&sem->wait_list);  in __init_rwsem()
321 	atomic_long_set(&sem->owner, 0L);  in __init_rwsem()
323 	osq_lock_init(&sem->osq);  in __init_rwsem()
339 #define rwsem_first_waiter(sem) \  argument
340 	list_first_entry(&sem->wait_list, struct rwsem_waiter, list)
381 static void rwsem_mark_wake(struct rw_semaphore *sem,  in rwsem_mark_wake()  argument
389 	lockdep_assert_held(&sem->wait_lock);  in rwsem_mark_wake()
395 	waiter = rwsem_first_waiter(sem);  in rwsem_mark_wake()
416 	if (unlikely(atomic_long_read(&sem->count) < 0))  in rwsem_mark_wake()
428 		oldcount = atomic_long_fetch_add(adjustment, &sem->count);  in rwsem_mark_wake()
441 			atomic_long_add(-adjustment, &sem->count);  in rwsem_mark_wake()
451 		__rwsem_set_reader_owned(sem, owner);  in rwsem_mark_wake()
478 	list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) {  in rwsem_mark_wake()
494 	if (list_empty(&sem->wait_list)) {  in rwsem_mark_wake()
503 	if (woken && (atomic_long_read(&sem->count) & RWSEM_FLAG_HANDOFF))  in rwsem_mark_wake()
507 		atomic_long_add(adjustment, &sem->count);  in rwsem_mark_wake()
539 static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,  in rwsem_try_write_lock()  argument
544 	lockdep_assert_held(&sem->wait_lock);  in rwsem_try_write_lock()
546 	count = atomic_long_read(&sem->count);  in rwsem_try_write_lock()
564 			if (list_is_singular(&sem->wait_list))  in rwsem_try_write_lock()
567 	} while (!atomic_long_try_cmpxchg_acquire(&sem->count, &count, new));  in rwsem_try_write_lock()
576 	rwsem_set_owner(sem);  in rwsem_try_write_lock()
584 static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem)  in rwsem_try_write_lock_unqueued()  argument
586 	long count = atomic_long_read(&sem->count);  in rwsem_try_write_lock_unqueued()
589 		if (atomic_long_try_cmpxchg_acquire(&sem->count, &count,  in rwsem_try_write_lock_unqueued()
591 			rwsem_set_owner(sem);  in rwsem_try_write_lock_unqueued()
608 static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)  in rwsem_can_spin_on_owner()  argument
621 	owner = rwsem_owner_flags(sem, &flags);  in rwsem_can_spin_on_owner()
667 rwsem_spin_on_owner(struct rw_semaphore *sem)  in rwsem_spin_on_owner()  argument
673 	owner = rwsem_owner_flags(sem, &flags);  in rwsem_spin_on_owner()
686 		new = rwsem_owner_flags(sem, &new_flags);  in rwsem_spin_on_owner()
724 static inline u64 rwsem_rspin_threshold(struct rw_semaphore *sem)  in rwsem_rspin_threshold()  argument
726 	long count = atomic_long_read(&sem->count);  in rwsem_rspin_threshold()
737 static bool rwsem_optimistic_spin(struct rw_semaphore *sem)  in rwsem_optimistic_spin()  argument
747 	if (!osq_lock(&sem->osq))  in rwsem_optimistic_spin()
759 		owner_state = rwsem_spin_on_owner(sem);  in rwsem_optimistic_spin()
766 		taken = rwsem_try_write_lock_unqueued(sem);  in rwsem_optimistic_spin()
783 				if (rwsem_test_oflags(sem, RWSEM_NONSPINNABLE))  in rwsem_optimistic_spin()
785 				rspin_threshold = rwsem_rspin_threshold(sem);  in rwsem_optimistic_spin()
797 				rwsem_set_nonspinnable(sem);  in rwsem_optimistic_spin()
851 	osq_unlock(&sem->osq);  in rwsem_optimistic_spin()
862 static inline void clear_nonspinnable(struct rw_semaphore *sem)  in clear_nonspinnable()  argument
864 	if (rwsem_test_oflags(sem, RWSEM_NONSPINNABLE))  in clear_nonspinnable()
865 		atomic_long_andnot(RWSEM_NONSPINNABLE, &sem->owner);  in clear_nonspinnable()
869 static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)  in rwsem_can_spin_on_owner()  argument
874 static inline bool rwsem_optimistic_spin(struct rw_semaphore *sem)  in rwsem_optimistic_spin()  argument
879 static inline void clear_nonspinnable(struct rw_semaphore *sem) { }  in clear_nonspinnable()  argument
882 rwsem_spin_on_owner(struct rw_semaphore *sem)  in rwsem_spin_on_owner()  argument
893 rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int state)  in rwsem_down_read_slowpath()  argument
906 	if ((atomic_long_read(&sem->owner) & RWSEM_READER_OWNED) &&  in rwsem_down_read_slowpath()
914 		rwsem_set_reader_owned(sem);  in rwsem_down_read_slowpath()
922 			raw_spin_lock_irq(&sem->wait_lock);  in rwsem_down_read_slowpath()
923 			if (!list_empty(&sem->wait_list))  in rwsem_down_read_slowpath()
924 				rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED,  in rwsem_down_read_slowpath()
926 			raw_spin_unlock_irq(&sem->wait_lock);  in rwsem_down_read_slowpath()
929 		return sem;  in rwsem_down_read_slowpath()
937 	raw_spin_lock_irq(&sem->wait_lock);  in rwsem_down_read_slowpath()
938 	if (list_empty(&sem->wait_list)) {  in rwsem_down_read_slowpath()
945 		if (!(atomic_long_read(&sem->count) &  in rwsem_down_read_slowpath()
949 			raw_spin_unlock_irq(&sem->wait_lock);  in rwsem_down_read_slowpath()
950 			rwsem_set_reader_owned(sem);  in rwsem_down_read_slowpath()
952 			return sem;  in rwsem_down_read_slowpath()
956 	list_add_tail(&waiter.list, &sem->wait_list);  in rwsem_down_read_slowpath()
959 	count = atomic_long_add_return(adjustment, &sem->count);  in rwsem_down_read_slowpath()
968 		clear_nonspinnable(sem);  in rwsem_down_read_slowpath()
973 		rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);  in rwsem_down_read_slowpath()
975 	raw_spin_unlock_irq(&sem->wait_lock);  in rwsem_down_read_slowpath()
986 			raw_spin_lock_irq(&sem->wait_lock);  in rwsem_down_read_slowpath()
989 			raw_spin_unlock_irq(&sem->wait_lock);  in rwsem_down_read_slowpath()
999 	return sem;  in rwsem_down_read_slowpath()
1003 	if (list_empty(&sem->wait_list)) {  in rwsem_down_read_slowpath()
1005 				   &sem->count);  in rwsem_down_read_slowpath()
1007 	raw_spin_unlock_irq(&sem->wait_lock);  in rwsem_down_read_slowpath()
1017 rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)  in rwsem_down_write_slowpath()  argument
1022 	struct rw_semaphore *ret = sem;  in rwsem_down_write_slowpath()
1026 	if (rwsem_can_spin_on_owner(sem) && rwsem_optimistic_spin(sem)) {  in rwsem_down_write_slowpath()
1028 		return sem;  in rwsem_down_write_slowpath()
1039 	raw_spin_lock_irq(&sem->wait_lock);  in rwsem_down_write_slowpath()
1042 	wstate = list_empty(&sem->wait_list) ? WRITER_FIRST : WRITER_NOT_FIRST;  in rwsem_down_write_slowpath()
1044 	list_add_tail(&waiter.list, &sem->wait_list);  in rwsem_down_write_slowpath()
1048 		count = atomic_long_read(&sem->count);  in rwsem_down_write_slowpath()
1061 		rwsem_mark_wake(sem, (count & RWSEM_READER_MASK)  in rwsem_down_write_slowpath()
1070 			raw_spin_unlock_irq(&sem->wait_lock);  in rwsem_down_write_slowpath()
1073 			raw_spin_lock_irq(&sem->wait_lock);  in rwsem_down_write_slowpath()
1076 		atomic_long_or(RWSEM_FLAG_WAITERS, &sem->count);  in rwsem_down_write_slowpath()
1083 		if (rwsem_try_write_lock(sem, wstate)) {  in rwsem_down_write_slowpath()
1088 		raw_spin_unlock_irq(&sem->wait_lock);  in rwsem_down_write_slowpath()
1099 		    rwsem_spin_on_owner(sem) == OWNER_NULL)  in rwsem_down_write_slowpath()
1118 			    (rwsem_first_waiter(sem) == &waiter))  in rwsem_down_write_slowpath()
1121 			count = atomic_long_read(&sem->count);  in rwsem_down_write_slowpath()
1137 		raw_spin_lock_irq(&sem->wait_lock);  in rwsem_down_write_slowpath()
1141 	raw_spin_unlock_irq(&sem->wait_lock);  in rwsem_down_write_slowpath()
1148 	raw_spin_lock_irq(&sem->wait_lock);  in rwsem_down_write_slowpath()
1152 		atomic_long_add(-RWSEM_FLAG_HANDOFF,  &sem->count);  in rwsem_down_write_slowpath()
1154 	if (list_empty(&sem->wait_list))  in rwsem_down_write_slowpath()
1155 		atomic_long_andnot(RWSEM_FLAG_WAITERS, &sem->count);  in rwsem_down_write_slowpath()
1157 		rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);  in rwsem_down_write_slowpath()
1158 	raw_spin_unlock_irq(&sem->wait_lock);  in rwsem_down_write_slowpath()
1169 static struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem)  in rwsem_wake()  argument
1174 	raw_spin_lock_irqsave(&sem->wait_lock, flags);  in rwsem_wake()
1176 	if (!list_empty(&sem->wait_list))  in rwsem_wake()
1177 		rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);  in rwsem_wake()
1179 	raw_spin_unlock_irqrestore(&sem->wait_lock, flags);  in rwsem_wake()
1182 	return sem;  in rwsem_wake()
1190 static struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem)  in rwsem_downgrade_wake()  argument
1195 	raw_spin_lock_irqsave(&sem->wait_lock, flags);  in rwsem_downgrade_wake()
1197 	if (!list_empty(&sem->wait_list))  in rwsem_downgrade_wake()
1198 		rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, &wake_q);  in rwsem_downgrade_wake()
1200 	raw_spin_unlock_irqrestore(&sem->wait_lock, flags);  in rwsem_downgrade_wake()
1203 	return sem;  in rwsem_downgrade_wake()
1209 static inline int __down_read_common(struct rw_semaphore *sem, int state)  in __down_read_common()  argument
1213 	if (!rwsem_read_trylock(sem, &count)) {  in __down_read_common()
1214 		if (IS_ERR(rwsem_down_read_slowpath(sem, count, state)))  in __down_read_common()
1216 		DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem);  in __down_read_common()
1221 static inline void __down_read(struct rw_semaphore *sem)  in __down_read()  argument
1223 	__down_read_common(sem, TASK_UNINTERRUPTIBLE);  in __down_read()
1226 static inline int __down_read_interruptible(struct rw_semaphore *sem)  in __down_read_interruptible()  argument
1228 	return __down_read_common(sem, TASK_INTERRUPTIBLE);  in __down_read_interruptible()
1231 static inline int __down_read_killable(struct rw_semaphore *sem)  in __down_read_killable()  argument
1233 	return __down_read_common(sem, TASK_KILLABLE);  in __down_read_killable()
1236 static inline int __down_read_trylock(struct rw_semaphore *sem)  in __down_read_trylock()  argument
1240 	DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem);  in __down_read_trylock()
1247 		if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp,  in __down_read_trylock()
1249 			rwsem_set_reader_owned(sem);  in __down_read_trylock()
1259 static inline int __down_write_common(struct rw_semaphore *sem, int state)  in __down_write_common()  argument
1261 	if (unlikely(!rwsem_write_trylock(sem))) {  in __down_write_common()
1262 		if (IS_ERR(rwsem_down_write_slowpath(sem, state)))  in __down_write_common()
1269 static inline void __down_write(struct rw_semaphore *sem)  in __down_write()  argument
1271 	__down_write_common(sem, TASK_UNINTERRUPTIBLE);  in __down_write()
1274 static inline int __down_write_killable(struct rw_semaphore *sem)  in __down_write_killable()  argument
1276 	return __down_write_common(sem, TASK_KILLABLE);  in __down_write_killable()
1279 static inline int __down_write_trylock(struct rw_semaphore *sem)  in __down_write_trylock()  argument
1281 	DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem);  in __down_write_trylock()
1282 	return rwsem_write_trylock(sem);  in __down_write_trylock()
1288 static inline void __up_read(struct rw_semaphore *sem)  in __up_read()  argument
1292 	DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem);  in __up_read()
1293 	DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem);  in __up_read()
1295 	rwsem_clear_reader_owned(sem);  in __up_read()
1296 	tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count);  in __up_read()
1297 	DEBUG_RWSEMS_WARN_ON(tmp < 0, sem);  in __up_read()
1300 		clear_nonspinnable(sem);  in __up_read()
1301 		rwsem_wake(sem);  in __up_read()
1308 static inline void __up_write(struct rw_semaphore *sem)  in __up_write()  argument
1312 	DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem);  in __up_write()
1317 	DEBUG_RWSEMS_WARN_ON((rwsem_owner(sem) != current) &&  in __up_write()
1318 			    !rwsem_test_oflags(sem, RWSEM_NONSPINNABLE), sem);  in __up_write()
1320 	rwsem_clear_owner(sem);  in __up_write()
1321 	tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count);  in __up_write()
1323 		rwsem_wake(sem);  in __up_write()
1329 static inline void __downgrade_write(struct rw_semaphore *sem)  in __downgrade_write()  argument
1340 	DEBUG_RWSEMS_WARN_ON(rwsem_owner(sem) != current, sem);  in __downgrade_write()
1342 		-RWSEM_WRITER_LOCKED+RWSEM_READER_BIAS, &sem->count);  in __downgrade_write()
1343 	rwsem_set_reader_owned(sem);  in __downgrade_write()
1345 		rwsem_downgrade_wake(sem);  in __downgrade_write()
1379 void __init_rwsem(struct rw_semaphore *sem, const char *name,  in __init_rwsem()  argument
1382 	init_rwbase_rt(&(sem)->rwbase);  in __init_rwsem()
1385 	debug_check_no_locks_freed((void *)sem, sizeof(*sem));  in __init_rwsem()
1386 	lockdep_init_map_wait(&sem->dep_map, name, key, 0, LD_WAIT_SLEEP);  in __init_rwsem()
1391 static inline void __down_read(struct rw_semaphore *sem)  in __down_read()  argument
1393 	rwbase_read_lock(&sem->rwbase, TASK_UNINTERRUPTIBLE);  in __down_read()
1396 static inline int __down_read_interruptible(struct rw_semaphore *sem)  in __down_read_interruptible()  argument
1398 	return rwbase_read_lock(&sem->rwbase, TASK_INTERRUPTIBLE);  in __down_read_interruptible()
1401 static inline int __down_read_killable(struct rw_semaphore *sem)  in __down_read_killable()  argument
1403 	return rwbase_read_lock(&sem->rwbase, TASK_KILLABLE);  in __down_read_killable()
1406 static inline int __down_read_trylock(struct rw_semaphore *sem)  in __down_read_trylock()  argument
1408 	return rwbase_read_trylock(&sem->rwbase);  in __down_read_trylock()
1411 static inline void __up_read(struct rw_semaphore *sem)  in __up_read()  argument
1413 	rwbase_read_unlock(&sem->rwbase, TASK_NORMAL);  in __up_read()
1416 static inline void __sched __down_write(struct rw_semaphore *sem)  in __down_write()  argument
1418 	rwbase_write_lock(&sem->rwbase, TASK_UNINTERRUPTIBLE);  in __down_write()
1421 static inline int __sched __down_write_killable(struct rw_semaphore *sem)  in __down_write_killable()  argument
1423 	return rwbase_write_lock(&sem->rwbase, TASK_KILLABLE);  in __down_write_killable()
1426 static inline int __down_write_trylock(struct rw_semaphore *sem)  in __down_write_trylock()  argument
1428 	return rwbase_write_trylock(&sem->rwbase);  in __down_write_trylock()
1431 static inline void __up_write(struct rw_semaphore *sem)  in __up_write()  argument
1433 	rwbase_write_unlock(&sem->rwbase);  in __up_write()
1436 static inline void __downgrade_write(struct rw_semaphore *sem)  in __downgrade_write()  argument
1438 	rwbase_write_downgrade(&sem->rwbase);  in __downgrade_write()
1442 #define DEBUG_RWSEMS_WARN_ON(c, sem)  argument
1444 static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem,  in __rwsem_set_reader_owned()  argument
1449 static inline bool is_rwsem_reader_owned(struct rw_semaphore *sem)  in is_rwsem_reader_owned()  argument
1451 	int count = atomic_read(&sem->rwbase.readers);  in is_rwsem_reader_owned()
1461 void __sched down_read(struct rw_semaphore *sem)  in down_read()  argument
1464 	rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);  in down_read()
1466 	LOCK_CONTENDED(sem, __down_read_trylock, __down_read);  in down_read()
1470 int __sched down_read_interruptible(struct rw_semaphore *sem)  in down_read_interruptible()  argument
1473 	rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);  in down_read_interruptible()
1475 	if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_interruptible)) {  in down_read_interruptible()
1476 		rwsem_release(&sem->dep_map, _RET_IP_);  in down_read_interruptible()
1484 int __sched down_read_killable(struct rw_semaphore *sem)  in down_read_killable()  argument
1487 	rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);  in down_read_killable()
1489 	if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable)) {  in down_read_killable()
1490 		rwsem_release(&sem->dep_map, _RET_IP_);  in down_read_killable()
1501 int down_read_trylock(struct rw_semaphore *sem)  in down_read_trylock()  argument
1503 	int ret = __down_read_trylock(sem);  in down_read_trylock()
1506 		rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_);  in down_read_trylock()
1514 void __sched down_write(struct rw_semaphore *sem)  in down_write()  argument
1517 	rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);  in down_write()
1518 	LOCK_CONTENDED(sem, __down_write_trylock, __down_write);  in down_write()
1525 int __sched down_write_killable(struct rw_semaphore *sem)  in down_write_killable()  argument
1528 	rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);  in down_write_killable()
1530 	if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock,  in down_write_killable()
1532 		rwsem_release(&sem->dep_map, _RET_IP_);  in down_write_killable()
1543 int down_write_trylock(struct rw_semaphore *sem)  in down_write_trylock()  argument
1545 	int ret = __down_write_trylock(sem);  in down_write_trylock()
1548 		rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_);  in down_write_trylock()
1557 void up_read(struct rw_semaphore *sem)  in up_read()  argument
1559 	rwsem_release(&sem->dep_map, _RET_IP_);  in up_read()
1560 	__up_read(sem);  in up_read()
1567 void up_write(struct rw_semaphore *sem)  in up_write()  argument
1569 	rwsem_release(&sem->dep_map, _RET_IP_);  in up_write()
1570 	__up_write(sem);  in up_write()
1577 void downgrade_write(struct rw_semaphore *sem)  in downgrade_write()  argument
1579 	lock_downgrade(&sem->dep_map, _RET_IP_);  in downgrade_write()
1580 	__downgrade_write(sem);  in downgrade_write()
1586 void down_read_nested(struct rw_semaphore *sem, int subclass)  in down_read_nested()  argument
1589 	rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);  in down_read_nested()
1590 	LOCK_CONTENDED(sem, __down_read_trylock, __down_read);  in down_read_nested()
1594 int down_read_killable_nested(struct rw_semaphore *sem, int subclass)  in down_read_killable_nested()  argument
1597 	rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);  in down_read_killable_nested()
1599 	if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable)) {  in down_read_killable_nested()
1600 		rwsem_release(&sem->dep_map, _RET_IP_);  in down_read_killable_nested()
1608 void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)  in _down_write_nest_lock()  argument
1611 	rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_);  in _down_write_nest_lock()
1612 	LOCK_CONTENDED(sem, __down_write_trylock, __down_write);  in _down_write_nest_lock()
1616 void down_read_non_owner(struct rw_semaphore *sem)  in down_read_non_owner()  argument
1619 	__down_read(sem);  in down_read_non_owner()
1620 	__rwsem_set_reader_owned(sem, NULL);  in down_read_non_owner()
1624 void down_write_nested(struct rw_semaphore *sem, int subclass)  in down_write_nested()  argument
1627 	rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);  in down_write_nested()
1628 	LOCK_CONTENDED(sem, __down_write_trylock, __down_write);  in down_write_nested()
1632 int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)  in down_write_killable_nested()  argument
1635 	rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);  in down_write_killable_nested()
1637 	if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock,  in down_write_killable_nested()
1639 		rwsem_release(&sem->dep_map, _RET_IP_);  in down_write_killable_nested()
1647 void up_read_non_owner(struct rw_semaphore *sem)  in up_read_non_owner()  argument
1649 	DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem);  in up_read_non_owner()
1650 	__up_read(sem);  in up_read_non_owner()