36 					struct rt_mutex *lock,  in __ww_mutex_add_waiter()  argument
42 static inline void __ww_mutex_check_waiters(struct rt_mutex *lock,  in __ww_mutex_check_waiters()  argument
47 static inline void ww_mutex_lock_acquired(struct ww_mutex *lock,  in ww_mutex_lock_acquired()  argument
52 static inline int __ww_mutex_check_kill(struct rt_mutex *lock,  in __ww_mutex_check_kill()  argument
93 rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner)  in rt_mutex_set_owner()  argument
97 	if (rt_mutex_has_waiters(lock))  in rt_mutex_set_owner()
100 	WRITE_ONCE(lock->owner, (struct task_struct *)val);  in rt_mutex_set_owner()
103 static __always_inline void clear_rt_mutex_waiters(struct rt_mutex_base *lock)  in clear_rt_mutex_waiters()  argument
105 	lock->owner = (struct task_struct *)  in clear_rt_mutex_waiters()
106 			((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);  in clear_rt_mutex_waiters()
109 static __always_inline void fixup_rt_mutex_waiters(struct rt_mutex_base *lock)  in fixup_rt_mutex_waiters()  argument
111 	unsigned long owner, *p = (unsigned long *) &lock->owner;  in fixup_rt_mutex_waiters()
113 	if (rt_mutex_has_waiters(lock))  in fixup_rt_mutex_waiters()
184 static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_acquire()  argument
188 	return try_cmpxchg_acquire(&lock->owner, &old, new);  in rt_mutex_cmpxchg_acquire()
191 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_release()  argument
195 	return try_cmpxchg_release(&lock->owner, &old, new);  in rt_mutex_cmpxchg_release()
203 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)  in mark_rt_mutex_waiters()  argument
205 	unsigned long owner, *p = (unsigned long *) &lock->owner;  in mark_rt_mutex_waiters()
219 static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock,  in unlock_rt_mutex_safe()  argument
221 	__releases(lock->wait_lock)  in unlock_rt_mutex_safe()
223 	struct task_struct *owner = rt_mutex_owner(lock);  in unlock_rt_mutex_safe()
225 	clear_rt_mutex_waiters(lock);  in unlock_rt_mutex_safe()
226 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in unlock_rt_mutex_safe()
251 	return rt_mutex_cmpxchg_release(lock, owner, NULL);  in unlock_rt_mutex_safe()
255 static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_acquire()  argument
263 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,  in rt_mutex_cmpxchg_release()  argument
270 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)  in mark_rt_mutex_waiters()  argument
272 	lock->owner = (struct task_struct *)  in mark_rt_mutex_waiters()
273 			((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);  in mark_rt_mutex_waiters()
279 static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock,  in unlock_rt_mutex_safe()  argument
281 	__releases(lock->wait_lock)  in unlock_rt_mutex_safe()
283 	lock->owner = NULL;  in unlock_rt_mutex_safe()
284 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in unlock_rt_mutex_safe()
398 rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue()  argument
400 	rb_add_cached(&waiter->tree_entry, &lock->waiters, __waiter_less);  in rt_mutex_enqueue()
404 rt_mutex_dequeue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue()  argument
409 	rb_erase_cached(&waiter->tree_entry, &lock->waiters);  in rt_mutex_dequeue()
510 	return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;  in task_blocked_on_lock()
586 	struct rt_mutex_base *lock;  in rt_mutex_adjust_prio_chain()  local
665 	if (next_lock != waiter->lock)  in rt_mutex_adjust_prio_chain()
732 	lock = waiter->lock;  in rt_mutex_adjust_prio_chain()
738 	if (!raw_spin_trylock(&lock->wait_lock)) {  in rt_mutex_adjust_prio_chain()
753 	if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {  in rt_mutex_adjust_prio_chain()
768 		raw_spin_unlock(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
789 		if (!rt_mutex_owner(lock)) {  in rt_mutex_adjust_prio_chain()
790 			raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
795 		task = get_task_struct(rt_mutex_owner(lock));  in rt_mutex_adjust_prio_chain()
808 		top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
812 		raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
825 	prerequeue_top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
828 	rt_mutex_dequeue(lock, waiter);  in rt_mutex_adjust_prio_chain()
848 	rt_mutex_enqueue(lock, waiter);  in rt_mutex_adjust_prio_chain()
861 	if (!rt_mutex_owner(lock)) {  in rt_mutex_adjust_prio_chain()
867 		if (prerequeue_top_waiter != rt_mutex_top_waiter(lock))  in rt_mutex_adjust_prio_chain()
869 		raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
874 	task = get_task_struct(rt_mutex_owner(lock));  in rt_mutex_adjust_prio_chain()
878 	if (waiter == rt_mutex_top_waiter(lock)) {  in rt_mutex_adjust_prio_chain()
901 		waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
926 	top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
930 	raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
971 try_to_take_rt_mutex(struct rt_mutex_base *lock, struct task_struct *task,  in try_to_take_rt_mutex()  argument
974 	lockdep_assert_held(&lock->wait_lock);  in try_to_take_rt_mutex()
993 	mark_rt_mutex_waiters(lock);  in try_to_take_rt_mutex()
998 	if (rt_mutex_owner(lock))  in try_to_take_rt_mutex()
1007 		struct rt_mutex_waiter *top_waiter = rt_mutex_top_waiter(lock);  in try_to_take_rt_mutex()
1018 			rt_mutex_dequeue(lock, waiter);  in try_to_take_rt_mutex()
1031 		if (rt_mutex_has_waiters(lock)) {  in try_to_take_rt_mutex()
1034 					    rt_mutex_top_waiter(lock)))  in try_to_take_rt_mutex()
1066 	if (rt_mutex_has_waiters(lock))  in try_to_take_rt_mutex()
1067 		rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));  in try_to_take_rt_mutex()
1075 	rt_mutex_set_owner(lock, task);  in try_to_take_rt_mutex()
1087 static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock,  in task_blocks_on_rt_mutex()  argument
1093 	struct task_struct *owner = rt_mutex_owner(lock);  in task_blocks_on_rt_mutex()
1098 	lockdep_assert_held(&lock->wait_lock);  in task_blocks_on_rt_mutex()
1117 	waiter->lock = lock;  in task_blocks_on_rt_mutex()
1121 	if (rt_mutex_has_waiters(lock))  in task_blocks_on_rt_mutex()
1122 		top_waiter = rt_mutex_top_waiter(lock);  in task_blocks_on_rt_mutex()
1123 	rt_mutex_enqueue(lock, waiter);  in task_blocks_on_rt_mutex()
1133 		rtm = container_of(lock, struct rt_mutex, rtmutex);  in task_blocks_on_rt_mutex()
1137 			rt_mutex_dequeue(lock, waiter);  in task_blocks_on_rt_mutex()
1148 	if (waiter == rt_mutex_top_waiter(lock)) {  in task_blocks_on_rt_mutex()
1178 	raw_spin_unlock_irq(&lock->wait_lock);  in task_blocks_on_rt_mutex()
1180 	res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,  in task_blocks_on_rt_mutex()
1183 	raw_spin_lock_irq(&lock->wait_lock);  in task_blocks_on_rt_mutex()
1195 					    struct rt_mutex_base *lock)  in mark_wakeup_next_waiter()  argument
1201 	waiter = rt_mutex_top_waiter(lock);  in mark_wakeup_next_waiter()
1221 	lock->owner = (void *) RT_MUTEX_HAS_WAITERS;  in mark_wakeup_next_waiter()
1238 static int __sched __rt_mutex_slowtrylock(struct rt_mutex_base *lock)  in __rt_mutex_slowtrylock()  argument
1240 	int ret = try_to_take_rt_mutex(lock, current, NULL);  in __rt_mutex_slowtrylock()
1246 	fixup_rt_mutex_waiters(lock);  in __rt_mutex_slowtrylock()
1254 static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock)  in rt_mutex_slowtrylock()  argument
1264 	if (rt_mutex_owner(lock))  in rt_mutex_slowtrylock()
1271 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowtrylock()
1273 	ret = __rt_mutex_slowtrylock(lock);  in rt_mutex_slowtrylock()
1275 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowtrylock()
1280 static __always_inline int __rt_mutex_trylock(struct rt_mutex_base *lock)  in __rt_mutex_trylock()  argument
1282 	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))  in __rt_mutex_trylock()
1285 	return rt_mutex_slowtrylock(lock);  in __rt_mutex_trylock()
1291 static void __sched rt_mutex_slowunlock(struct rt_mutex_base *lock)  in rt_mutex_slowunlock()  argument
1297 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowunlock()
1299 	debug_rt_mutex_unlock(lock);  in rt_mutex_slowunlock()
1332 	while (!rt_mutex_has_waiters(lock)) {  in rt_mutex_slowunlock()
1334 		if (unlock_rt_mutex_safe(lock, flags) == true)  in rt_mutex_slowunlock()
1337 		raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowunlock()
1346 	mark_wakeup_next_waiter(&wqh, lock);  in rt_mutex_slowunlock()
1347 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowunlock()
1352 static __always_inline void __rt_mutex_unlock(struct rt_mutex_base *lock)  in __rt_mutex_unlock()  argument
1354 	if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))  in __rt_mutex_unlock()
1357 	rt_mutex_slowunlock(lock);  in __rt_mutex_unlock()
1361 static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,  in rtmutex_spin_on_owner()  argument
1370 		if (owner != rt_mutex_owner(lock))  in rtmutex_spin_on_owner()
1388 		    !rt_mutex_waiter_is_top_waiter(lock, waiter)) {  in rtmutex_spin_on_owner()
1398 static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock,  in rtmutex_spin_on_owner()  argument
1419 static void __sched remove_waiter(struct rt_mutex_base *lock,  in remove_waiter()  argument
1422 	bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));  in remove_waiter()
1423 	struct task_struct *owner = rt_mutex_owner(lock);  in remove_waiter()
1426 	lockdep_assert_held(&lock->wait_lock);  in remove_waiter()
1429 	rt_mutex_dequeue(lock, waiter);  in remove_waiter()
1444 	if (rt_mutex_has_waiters(lock))  in remove_waiter()
1445 		rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock));  in remove_waiter()
1464 	raw_spin_unlock_irq(&lock->wait_lock);  in remove_waiter()
1466 	rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,  in remove_waiter()
1469 	raw_spin_lock_irq(&lock->wait_lock);  in remove_waiter()
1483 static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,  in rt_mutex_slowlock_block()  argument
1489 	struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex);  in rt_mutex_slowlock_block()
1495 		if (try_to_take_rt_mutex(lock, current, waiter))  in rt_mutex_slowlock_block()
1513 		if (waiter == rt_mutex_top_waiter(lock))  in rt_mutex_slowlock_block()
1514 			owner = rt_mutex_owner(lock);  in rt_mutex_slowlock_block()
1517 		raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_slowlock_block()
1519 		if (!owner || !rtmutex_spin_on_owner(lock, waiter, owner))  in rt_mutex_slowlock_block()
1522 		raw_spin_lock_irq(&lock->wait_lock);  in rt_mutex_slowlock_block()
1561 static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,  in __rt_mutex_slowlock()  argument
1567 	struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex);  in __rt_mutex_slowlock()
1571 	lockdep_assert_held(&lock->wait_lock);  in __rt_mutex_slowlock()
1574 	if (try_to_take_rt_mutex(lock, current, NULL)) {  in __rt_mutex_slowlock()
1584 	trace_contention_begin(lock, LCB_F_RT);  in __rt_mutex_slowlock()
1586 	ret = task_blocks_on_rt_mutex(lock, waiter, current, ww_ctx, chwalk);  in __rt_mutex_slowlock()
1588 		ret = rt_mutex_slowlock_block(lock, ww_ctx, state, NULL, waiter);  in __rt_mutex_slowlock()
1599 		remove_waiter(lock, waiter);  in __rt_mutex_slowlock()
1607 	fixup_rt_mutex_waiters(lock);  in __rt_mutex_slowlock()
1609 	trace_contention_end(lock, ret);  in __rt_mutex_slowlock()
1614 static inline int __rt_mutex_slowlock_locked(struct rt_mutex_base *lock,  in __rt_mutex_slowlock_locked()  argument
1624 	ret = __rt_mutex_slowlock(lock, ww_ctx, state, RT_MUTEX_MIN_CHAINWALK,  in __rt_mutex_slowlock_locked()
1637 static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock,  in rt_mutex_slowlock()  argument
1652 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowlock()
1653 	ret = __rt_mutex_slowlock_locked(lock, ww_ctx, state);  in rt_mutex_slowlock()
1654 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowlock()
1659 static __always_inline int __rt_mutex_lock(struct rt_mutex_base *lock,  in __rt_mutex_lock()  argument
1662 	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))  in __rt_mutex_lock()
1665 	return rt_mutex_slowlock(lock, NULL, state);  in __rt_mutex_lock()
1678 static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock)  in rtlock_slowlock_locked()  argument
1683 	lockdep_assert_held(&lock->wait_lock);  in rtlock_slowlock_locked()
1685 	if (try_to_take_rt_mutex(lock, current, NULL))  in rtlock_slowlock_locked()
1693 	trace_contention_begin(lock, LCB_F_RT);  in rtlock_slowlock_locked()
1695 	task_blocks_on_rt_mutex(lock, &waiter, current, NULL, RT_MUTEX_MIN_CHAINWALK);  in rtlock_slowlock_locked()
1699 		if (try_to_take_rt_mutex(lock, current, &waiter))  in rtlock_slowlock_locked()
1702 		if (&waiter == rt_mutex_top_waiter(lock))  in rtlock_slowlock_locked()
1703 			owner = rt_mutex_owner(lock);  in rtlock_slowlock_locked()
1706 		raw_spin_unlock_irq(&lock->wait_lock);  in rtlock_slowlock_locked()
1708 		if (!owner || !rtmutex_spin_on_owner(lock, &waiter, owner))  in rtlock_slowlock_locked()
1711 		raw_spin_lock_irq(&lock->wait_lock);  in rtlock_slowlock_locked()
1722 	fixup_rt_mutex_waiters(lock);  in rtlock_slowlock_locked()
1725 	trace_contention_end(lock, 0);  in rtlock_slowlock_locked()
1728 static __always_inline void __sched rtlock_slowlock(struct rt_mutex_base *lock)  in rtlock_slowlock()  argument
1732 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rtlock_slowlock()
1733 	rtlock_slowlock_locked(lock);  in rtlock_slowlock()
1734 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rtlock_slowlock()