76 void __init_rwsem(struct rw_semaphore *sem, const char *name,  in __init_rwsem()  argument
83 	debug_check_no_locks_freed((void *)sem, sizeof(*sem));  in __init_rwsem()
84 	lockdep_init_map(&sem->dep_map, name, key, 0);  in __init_rwsem()
86 	atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE);  in __init_rwsem()
87 	raw_spin_lock_init(&sem->wait_lock);  in __init_rwsem()
88 	INIT_LIST_HEAD(&sem->wait_list);  in __init_rwsem()
90 	sem->owner = NULL;  in __init_rwsem()
91 	osq_lock_init(&sem->osq);  in __init_rwsem()
127 static void __rwsem_mark_wake(struct rw_semaphore *sem,  in __rwsem_mark_wake()  argument
138 	waiter = list_first_entry(&sem->wait_list, struct rwsem_waiter, list);  in __rwsem_mark_wake()
163 		oldcount = atomic_long_fetch_add(adjustment, &sem->count);  in __rwsem_mark_wake()
171 			if (atomic_long_add_return(-adjustment, &sem->count) <  in __rwsem_mark_wake()
183 		rwsem_set_reader_owned(sem);  in __rwsem_mark_wake()
192 	list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) {  in __rwsem_mark_wake()
213 	if (list_empty(&sem->wait_list)) {  in __rwsem_mark_wake()
219 		atomic_long_add(adjustment, &sem->count);  in __rwsem_mark_wake()
226 __rwsem_down_read_failed_common(struct rw_semaphore *sem, int state)  in __rwsem_down_read_failed_common()  argument
235 	raw_spin_lock_irq(&sem->wait_lock);  in __rwsem_down_read_failed_common()
236 	if (list_empty(&sem->wait_list))  in __rwsem_down_read_failed_common()
238 	list_add_tail(&waiter.list, &sem->wait_list);  in __rwsem_down_read_failed_common()
241 	count = atomic_long_add_return(adjustment, &sem->count);  in __rwsem_down_read_failed_common()
252 		__rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);  in __rwsem_down_read_failed_common()
254 	raw_spin_unlock_irq(&sem->wait_lock);  in __rwsem_down_read_failed_common()
263 			raw_spin_lock_irq(&sem->wait_lock);  in __rwsem_down_read_failed_common()
266 			raw_spin_unlock_irq(&sem->wait_lock);  in __rwsem_down_read_failed_common()
273 	return sem;  in __rwsem_down_read_failed_common()
276 	if (list_empty(&sem->wait_list))  in __rwsem_down_read_failed_common()
277 		atomic_long_add(-RWSEM_WAITING_BIAS, &sem->count);  in __rwsem_down_read_failed_common()
278 	raw_spin_unlock_irq(&sem->wait_lock);  in __rwsem_down_read_failed_common()
284 rwsem_down_read_failed(struct rw_semaphore *sem)  in rwsem_down_read_failed()  argument
286 	return __rwsem_down_read_failed_common(sem, TASK_UNINTERRUPTIBLE);  in rwsem_down_read_failed()
291 rwsem_down_read_failed_killable(struct rw_semaphore *sem)  in rwsem_down_read_failed_killable()  argument
293 	return __rwsem_down_read_failed_common(sem, TASK_KILLABLE);  in rwsem_down_read_failed_killable()
302 static inline bool rwsem_try_write_lock(long count, struct rw_semaphore *sem)  in rwsem_try_write_lock()  argument
314 	count = list_is_singular(&sem->wait_list) ?  in rwsem_try_write_lock()
318 	if (atomic_long_cmpxchg_acquire(&sem->count, RWSEM_WAITING_BIAS, count)  in rwsem_try_write_lock()
320 		rwsem_set_owner(sem);  in rwsem_try_write_lock()
331 static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem)  in rwsem_try_write_lock_unqueued()  argument
333 	long old, count = atomic_long_read(&sem->count);  in rwsem_try_write_lock_unqueued()
339 		old = atomic_long_cmpxchg_acquire(&sem->count, count,  in rwsem_try_write_lock_unqueued()
342 			rwsem_set_owner(sem);  in rwsem_try_write_lock_unqueued()
359 static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem)  in rwsem_can_spin_on_owner()  argument
370 	owner = READ_ONCE(sem->owner);  in rwsem_can_spin_on_owner()
382 static noinline bool rwsem_spin_on_owner(struct rw_semaphore *sem)  in rwsem_spin_on_owner()  argument
384 	struct task_struct *owner = READ_ONCE(sem->owner);  in rwsem_spin_on_owner()
390 	while (owner && (READ_ONCE(sem->owner) == owner)) {  in rwsem_spin_on_owner()
416 	return is_rwsem_owner_spinnable(READ_ONCE(sem->owner));  in rwsem_spin_on_owner()
419 static bool rwsem_optimistic_spin(struct rw_semaphore *sem)  in rwsem_optimistic_spin()  argument
426 	if (!rwsem_can_spin_on_owner(sem))  in rwsem_optimistic_spin()
429 	if (!osq_lock(&sem->osq))  in rwsem_optimistic_spin()
439 	while (rwsem_spin_on_owner(sem)) {  in rwsem_optimistic_spin()
443 		if (rwsem_try_write_lock_unqueued(sem)) {  in rwsem_optimistic_spin()
454 		if (!sem->owner && (need_resched() || rt_task(current)))  in rwsem_optimistic_spin()
465 	osq_unlock(&sem->osq);  in rwsem_optimistic_spin()
474 static inline bool rwsem_has_spinner(struct rw_semaphore *sem)  in rwsem_has_spinner()  argument
476 	return osq_is_locked(&sem->osq);  in rwsem_has_spinner()
480 static bool rwsem_optimistic_spin(struct rw_semaphore *sem)  in rwsem_optimistic_spin()  argument
485 static inline bool rwsem_has_spinner(struct rw_semaphore *sem)  in rwsem_has_spinner()  argument
495 __rwsem_down_write_failed_common(struct rw_semaphore *sem, int state)  in __rwsem_down_write_failed_common()  argument
500 	struct rw_semaphore *ret = sem;  in __rwsem_down_write_failed_common()
504 	count = atomic_long_sub_return(RWSEM_ACTIVE_WRITE_BIAS, &sem->count);  in __rwsem_down_write_failed_common()
507 	if (rwsem_optimistic_spin(sem))  in __rwsem_down_write_failed_common()
508 		return sem;  in __rwsem_down_write_failed_common()
517 	raw_spin_lock_irq(&sem->wait_lock);  in __rwsem_down_write_failed_common()
520 	if (list_empty(&sem->wait_list))  in __rwsem_down_write_failed_common()
523 	list_add_tail(&waiter.list, &sem->wait_list);  in __rwsem_down_write_failed_common()
527 		count = atomic_long_read(&sem->count);  in __rwsem_down_write_failed_common()
535 			__rwsem_mark_wake(sem, RWSEM_WAKE_READERS, &wake_q);  in __rwsem_down_write_failed_common()
552 		count = atomic_long_add_return(RWSEM_WAITING_BIAS, &sem->count);  in __rwsem_down_write_failed_common()
557 		if (rwsem_try_write_lock(count, sem))  in __rwsem_down_write_failed_common()
559 		raw_spin_unlock_irq(&sem->wait_lock);  in __rwsem_down_write_failed_common()
568 		} while ((count = atomic_long_read(&sem->count)) & RWSEM_ACTIVE_MASK);  in __rwsem_down_write_failed_common()
570 		raw_spin_lock_irq(&sem->wait_lock);  in __rwsem_down_write_failed_common()
574 	raw_spin_unlock_irq(&sem->wait_lock);  in __rwsem_down_write_failed_common()
580 	raw_spin_lock_irq(&sem->wait_lock);  in __rwsem_down_write_failed_common()
582 	if (list_empty(&sem->wait_list))  in __rwsem_down_write_failed_common()
583 		atomic_long_add(-RWSEM_WAITING_BIAS, &sem->count);  in __rwsem_down_write_failed_common()
585 		__rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);  in __rwsem_down_write_failed_common()
586 	raw_spin_unlock_irq(&sem->wait_lock);  in __rwsem_down_write_failed_common()
593 rwsem_down_write_failed(struct rw_semaphore *sem)  in rwsem_down_write_failed()  argument
595 	return __rwsem_down_write_failed_common(sem, TASK_UNINTERRUPTIBLE);  in rwsem_down_write_failed()
600 rwsem_down_write_failed_killable(struct rw_semaphore *sem)  in rwsem_down_write_failed_killable()  argument
602 	return __rwsem_down_write_failed_common(sem, TASK_KILLABLE);  in rwsem_down_write_failed_killable()
611 struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem)  in rwsem_wake()  argument
663 	if (rwsem_has_spinner(sem)) {  in rwsem_wake()
669 		if (!raw_spin_trylock_irqsave(&sem->wait_lock, flags))  in rwsem_wake()
670 			return sem;  in rwsem_wake()
673 	raw_spin_lock_irqsave(&sem->wait_lock, flags);  in rwsem_wake()
676 	if (!list_empty(&sem->wait_list))  in rwsem_wake()
677 		__rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q);  in rwsem_wake()
679 	raw_spin_unlock_irqrestore(&sem->wait_lock, flags);  in rwsem_wake()
682 	return sem;  in rwsem_wake()
692 struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem)  in rwsem_downgrade_wake()  argument
697 	raw_spin_lock_irqsave(&sem->wait_lock, flags);  in rwsem_downgrade_wake()
699 	if (!list_empty(&sem->wait_list))  in rwsem_downgrade_wake()
700 		__rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, &wake_q);  in rwsem_downgrade_wake()
702 	raw_spin_unlock_irqrestore(&sem->wait_lock, flags);  in rwsem_downgrade_wake()
705 	return sem;  in rwsem_downgrade_wake()