34 					struct rt_mutex *lock,  in __ww_mutex_add_waiter()  argument
40 static inline void __ww_mutex_check_waiters(struct rt_mutex *lock,  in __ww_mutex_check_waiters()  argument
45 static inline void ww_mutex_lock_acquired(struct ww_mutex *lock,  in ww_mutex_lock_acquired()  argument
50 static inline int __ww_mutex_check_kill(struct rt_mutex *lock,  in __ww_mutex_check_kill()  argument
91 rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner)  in rt_mutex_set_owner()  argument
95 	if (rt_mutex_has_waiters(lock))  in rt_mutex_set_owner()
98 	WRITE_ONCE(lock->owner, (struct task_struct *)val);  in rt_mutex_set_owner()
101 static __always_inline void clear_rt_mutex_waiters(struct rt_mutex_base *lock)  in clear_rt_mutex_waiters()  argument
103 	lock->owner = (struct task_struct *)  in clear_rt_mutex_waiters()
104 			((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);  in clear_rt_mutex_waiters()
107 static __always_inline void fixup_rt_mutex_waiters(struct rt_mutex_base *lock)  in fixup_rt_mutex_waiters()  argument
109 	unsigned long owner, *p = (unsigned long *) &lock->owner;  in fixup_rt_mutex_waiters()
111 	if (rt_mutex_has_waiters(lock))  in fixup_rt_mutex_waiters()
182 static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_acquire()  argument
186 	return try_cmpxchg_acquire(&lock->owner, &old, new);  in rt_mutex_cmpxchg_acquire()
189 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_release()  argument
193 	return try_cmpxchg_release(&lock->owner, &old, new);  in rt_mutex_cmpxchg_release()
201 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)  in mark_rt_mutex_waiters()  argument
203 	unsigned long owner, *p = (unsigned long *) &lock->owner;  in mark_rt_mutex_waiters()
217 static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock,  in unlock_rt_mutex_safe()  argument
219 	__releases(lock->wait_lock)  in unlock_rt_mutex_safe()
221 	struct task_struct *owner = rt_mutex_owner(lock);  in unlock_rt_mutex_safe()
223 	clear_rt_mutex_waiters(lock);  in unlock_rt_mutex_safe()
224 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in unlock_rt_mutex_safe()
249 	return rt_mutex_cmpxchg_release(lock, owner, NULL);  in unlock_rt_mutex_safe()
253 static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_acquire()  argument
261 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_release()  argument
268 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)  in mark_rt_mutex_waiters()  argument
270 	lock->owner = (struct task_struct *)  in mark_rt_mutex_waiters()
271 			((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);  in mark_rt_mutex_waiters()
277 static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock,  in unlock_rt_mutex_safe()  argument
279 	__releases(lock->wait_lock)  in unlock_rt_mutex_safe()
281 	lock->owner = NULL;  in unlock_rt_mutex_safe()
282 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in unlock_rt_mutex_safe()
396 rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue()  argument
398 	rb_add_cached(&waiter->tree_entry, &lock->waiters, __waiter_less);  in rt_mutex_enqueue()
402 rt_mutex_dequeue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue()  argument
407 	rb_erase_cached(&waiter->tree_entry, &lock->waiters);  in rt_mutex_dequeue()
501 	return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;  in task_blocked_on_lock()
577 	struct rt_mutex_base *lock;  in rt_mutex_adjust_prio_chain()  local
656 	if (next_lock != waiter->lock)  in rt_mutex_adjust_prio_chain()
723 	lock = waiter->lock;  in rt_mutex_adjust_prio_chain()
729 	if (!raw_spin_trylock(&lock->wait_lock)) {  in rt_mutex_adjust_prio_chain()
744 	if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {  in rt_mutex_adjust_prio_chain()
759 		raw_spin_unlock(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
780 		if (!rt_mutex_owner(lock)) {  in rt_mutex_adjust_prio_chain()
781 			raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
786 		task = get_task_struct(rt_mutex_owner(lock));  in rt_mutex_adjust_prio_chain()
799 		top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
803 		raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
816 	prerequeue_top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
819 	rt_mutex_dequeue(lock, waiter);  in rt_mutex_adjust_prio_chain()
839 	rt_mutex_enqueue(lock, waiter);  in rt_mutex_adjust_prio_chain()
852 	if (!rt_mutex_owner(lock)) {  in rt_mutex_adjust_prio_chain()
858 		if (prerequeue_top_waiter != rt_mutex_top_waiter(lock))  in rt_mutex_adjust_prio_chain()
860 		raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
865 	task = get_task_struct(rt_mutex_owner(lock));  in rt_mutex_adjust_prio_chain()
869 	if (waiter == rt_mutex_top_waiter(lock)) {  in rt_mutex_adjust_prio_chain()
892 		waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
917 	top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
921 	raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
962 try_to_take_rt_mutex(struct rt_mutex_base *lock, struct task_struct *task,  in try_to_take_rt_mutex()  argument
965 	lockdep_assert_held(&lock->wait_lock);  in try_to_take_rt_mutex()
984 	mark_rt_mutex_waiters(lock);  in try_to_take_rt_mutex()
989 	if (rt_mutex_owner(lock))  in try_to_take_rt_mutex()
998 		struct rt_mutex_waiter *top_waiter = rt_mutex_top_waiter(lock);  in try_to_take_rt_mutex()
1009 			rt_mutex_dequeue(lock, waiter);  in try_to_take_rt_mutex()
1022 		if (rt_mutex_has_waiters(lock)) {  in try_to_take_rt_mutex()
1025 					    rt_mutex_top_waiter(lock)))  in try_to_take_rt_mutex()
1057 	if (rt_mutex_has_waiters(lock))  in try_to_take_rt_mutex()
1058 		rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));  in try_to_take_rt_mutex()
1066 	rt_mutex_set_owner(lock, task);  in try_to_take_rt_mutex()
1078 static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock,  in task_blocks_on_rt_mutex()  argument
1084 	struct task_struct *owner = rt_mutex_owner(lock);  in task_blocks_on_rt_mutex()
1089 	lockdep_assert_held(&lock->wait_lock);  in task_blocks_on_rt_mutex()
1105 	waiter->lock = lock;  in task_blocks_on_rt_mutex()
1109 	if (rt_mutex_has_waiters(lock))  in task_blocks_on_rt_mutex()
1110 		top_waiter = rt_mutex_top_waiter(lock);  in task_blocks_on_rt_mutex()
1111 	rt_mutex_enqueue(lock, waiter);  in task_blocks_on_rt_mutex()
1121 		rtm = container_of(lock, struct rt_mutex, rtmutex);  in task_blocks_on_rt_mutex()
1125 			rt_mutex_dequeue(lock, waiter);  in task_blocks_on_rt_mutex()
1136 	if (waiter == rt_mutex_top_waiter(lock)) {  in task_blocks_on_rt_mutex()
1166 	raw_spin_unlock_irq(&lock->wait_lock);  in task_blocks_on_rt_mutex()
1168 	res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,  in task_blocks_on_rt_mutex()
1171 	raw_spin_lock_irq(&lock->wait_lock);  in task_blocks_on_rt_mutex()
1183 					    struct rt_mutex_base *lock)  in mark_wakeup_next_waiter()  argument
1189 	waiter = rt_mutex_top_waiter(lock);  in mark_wakeup_next_waiter()
1209 	lock->owner = (void *) RT_MUTEX_HAS_WAITERS;  in mark_wakeup_next_waiter()
1226 static int __sched __rt_mutex_slowtrylock(struct rt_mutex_base *lock)  in __rt_mutex_slowtrylock()  argument
1228 	int ret = try_to_take_rt_mutex(lock, current, NULL);  in __rt_mutex_slowtrylock()
1234 	fixup_rt_mutex_waiters(lock);  in __rt_mutex_slowtrylock()
1242 static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock)  in rt_mutex_slowtrylock()  argument
1252 	if (rt_mutex_owner(lock))  in rt_mutex_slowtrylock()
1259 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowtrylock()
1261 	ret = __rt_mutex_slowtrylock(lock);  in rt_mutex_slowtrylock()
1263 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowtrylock()
1268 static __always_inline int __rt_mutex_trylock(struct rt_mutex_base *lock)  in __rt_mutex_trylock()  argument
1270 	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))  in __rt_mutex_trylock()
1273 	return rt_mutex_slowtrylock(lock);  in __rt_mutex_trylock()
1279 static void __sched rt_mutex_slowunlock(struct rt_mutex_base *lock)  in rt_mutex_slowunlock()  argument
1285 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowunlock()
1287 	debug_rt_mutex_unlock(lock);  in rt_mutex_slowunlock()
1320 	while (!rt_mutex_has_waiters(lock)) {  in rt_mutex_slowunlock()
1322 		if (unlock_rt_mutex_safe(lock, flags) == true)  in rt_mutex_slowunlock()
1325 		raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowunlock()
1334 	mark_wakeup_next_waiter(&wqh, lock);  in rt_mutex_slowunlock()
1335 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowunlock()
1340 static __always_inline void __rt_mutex_unlock(struct rt_mutex_base *lock)  in __rt_mutex_unlock()  argument
1342 	if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))  in __rt_mutex_unlock()
1345 	rt_mutex_slowunlock(lock);  in __rt_mutex_unlock()
1349 static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,  in rtmutex_spin_on_owner()  argument
1358 		if (owner != rt_mutex_owner(lock))  in rtmutex_spin_on_owner()
1376 		    rt_mutex_waiter_is_top_waiter(lock, waiter) ||  in rtmutex_spin_on_owner()
1387 static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,  in rtmutex_spin_on_owner()  argument
1408 static void __sched remove_waiter(struct rt_mutex_base *lock,  in remove_waiter()  argument
1411 	bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));  in remove_waiter()
1412 	struct task_struct *owner = rt_mutex_owner(lock);  in remove_waiter()
1415 	lockdep_assert_held(&lock->wait_lock);  in remove_waiter()
1418 	rt_mutex_dequeue(lock, waiter);  in remove_waiter()
1433 	if (rt_mutex_has_waiters(lock))  in remove_waiter()
1434 		rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock));  in remove_waiter()
1453 	raw_spin_unlock_irq(&lock->wait_lock);  in remove_waiter()
1455 	rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,  in remove_waiter()
1458 	raw_spin_lock_irq(&lock->wait_lock);  in remove_waiter()
1472 static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,  in rt_mutex_slowlock_block()  argument
1478 	struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex);  in rt_mutex_slowlock_block()
1484 		if (try_to_take_rt_mutex(lock, current, waiter))  in rt_mutex_slowlock_block()
1502 		if (waiter == rt_mutex_top_waiter(lock))  in rt_mutex_slowlock_block()
1503 			owner = rt_mutex_owner(lock);  in rt_mutex_slowlock_block()
1506 		raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_slowlock_block()
1508 		if (!owner || !rtmutex_spin_on_owner(lock, waiter, owner))  in rt_mutex_slowlock_block()
1511 		raw_spin_lock_irq(&lock->wait_lock);  in rt_mutex_slowlock_block()
1550 static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,  in __rt_mutex_slowlock()  argument
1556 	struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex);  in __rt_mutex_slowlock()
1560 	lockdep_assert_held(&lock->wait_lock);  in __rt_mutex_slowlock()
1563 	if (try_to_take_rt_mutex(lock, current, NULL)) {  in __rt_mutex_slowlock()
1573 	ret = task_blocks_on_rt_mutex(lock, waiter, current, ww_ctx, chwalk);  in __rt_mutex_slowlock()
1575 		ret = rt_mutex_slowlock_block(lock, ww_ctx, state, NULL, waiter);  in __rt_mutex_slowlock()
1586 		remove_waiter(lock, waiter);  in __rt_mutex_slowlock()
1594 	fixup_rt_mutex_waiters(lock);  in __rt_mutex_slowlock()
1598 static inline int __rt_mutex_slowlock_locked(struct rt_mutex_base *lock,  in __rt_mutex_slowlock_locked()  argument
1608 	ret = __rt_mutex_slowlock(lock, ww_ctx, state, RT_MUTEX_MIN_CHAINWALK,  in __rt_mutex_slowlock_locked()
1621 static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock,  in rt_mutex_slowlock()  argument
1636 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowlock()
1637 	ret = __rt_mutex_slowlock_locked(lock, ww_ctx, state);  in rt_mutex_slowlock()
1638 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowlock()
1643 static __always_inline int __rt_mutex_lock(struct rt_mutex_base *lock,  in __rt_mutex_lock()  argument
1646 	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))  in __rt_mutex_lock()
1649 	return rt_mutex_slowlock(lock, NULL, state);  in __rt_mutex_lock()
1662 static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock)  in rtlock_slowlock_locked()  argument
1667 	lockdep_assert_held(&lock->wait_lock);  in rtlock_slowlock_locked()
1669 	if (try_to_take_rt_mutex(lock, current, NULL))  in rtlock_slowlock_locked()
1677 	task_blocks_on_rt_mutex(lock, &waiter, current, NULL, RT_MUTEX_MIN_CHAINWALK);  in rtlock_slowlock_locked()
1681 		if (try_to_take_rt_mutex(lock, current, &waiter))  in rtlock_slowlock_locked()
1684 		if (&waiter == rt_mutex_top_waiter(lock))  in rtlock_slowlock_locked()
1685 			owner = rt_mutex_owner(lock);  in rtlock_slowlock_locked()
1688 		raw_spin_unlock_irq(&lock->wait_lock);  in rtlock_slowlock_locked()
1690 		if (!owner || !rtmutex_spin_on_owner(lock, &waiter, owner))  in rtlock_slowlock_locked()
1693 		raw_spin_lock_irq(&lock->wait_lock);  in rtlock_slowlock_locked()
1704 	fixup_rt_mutex_waiters(lock);  in rtlock_slowlock_locked()
1708 static __always_inline void __sched rtlock_slowlock(struct rt_mutex_base *lock)  in rtlock_slowlock()  argument
1712 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rtlock_slowlock()
1713 	rtlock_slowlock_locked(lock);  in rtlock_slowlock()
1714 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rtlock_slowlock()