52 rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)  in rt_mutex_set_owner()  argument
56 	if (rt_mutex_has_waiters(lock))  in rt_mutex_set_owner()
59 	lock->owner = (struct task_struct *)val;  in rt_mutex_set_owner()
62 static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)  in clear_rt_mutex_waiters()  argument
64 	lock->owner = (struct task_struct *)  in clear_rt_mutex_waiters()
65 			((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);  in clear_rt_mutex_waiters()
68 static void fixup_rt_mutex_waiters(struct rt_mutex *lock)  in fixup_rt_mutex_waiters()  argument
70 	unsigned long owner, *p = (unsigned long *) &lock->owner;  in fixup_rt_mutex_waiters()
72 	if (rt_mutex_has_waiters(lock))  in fixup_rt_mutex_waiters()
152 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)  in mark_rt_mutex_waiters()  argument
154 	unsigned long owner, *p = (unsigned long *) &lock->owner;  in mark_rt_mutex_waiters()
168 static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,  in unlock_rt_mutex_safe()  argument
170 	__releases(lock->wait_lock)  in unlock_rt_mutex_safe()
172 	struct task_struct *owner = rt_mutex_owner(lock);  in unlock_rt_mutex_safe()
174 	clear_rt_mutex_waiters(lock);  in unlock_rt_mutex_safe()
175 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in unlock_rt_mutex_safe()
200 	return rt_mutex_cmpxchg_release(lock, owner, NULL);  in unlock_rt_mutex_safe()
208 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)  in mark_rt_mutex_waiters()  argument
210 	lock->owner = (struct task_struct *)  in mark_rt_mutex_waiters()
211 			((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);  in mark_rt_mutex_waiters()
217 static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,  in unlock_rt_mutex_safe()  argument
219 	__releases(lock->wait_lock)  in unlock_rt_mutex_safe()
221 	lock->owner = NULL;  in unlock_rt_mutex_safe()
222 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in unlock_rt_mutex_safe()
272 rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_enqueue()  argument
274 	struct rb_node **link = &lock->waiters.rb_root.rb_node;  in rt_mutex_enqueue()
291 	rb_insert_color_cached(&waiter->tree_entry, &lock->waiters, leftmost);  in rt_mutex_enqueue()
295 rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)  in rt_mutex_dequeue()  argument
300 	rb_erase_cached(&waiter->tree_entry, &lock->waiters);  in rt_mutex_dequeue()
382 	return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;  in task_blocked_on_lock()
458 	struct rt_mutex *lock;  in rt_mutex_adjust_prio_chain()  local
537 	if (next_lock != waiter->lock)  in rt_mutex_adjust_prio_chain()
579 	lock = waiter->lock;  in rt_mutex_adjust_prio_chain()
585 	if (!raw_spin_trylock(&lock->wait_lock)) {  in rt_mutex_adjust_prio_chain()
600 	if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {  in rt_mutex_adjust_prio_chain()
601 		debug_rt_mutex_deadlock(chwalk, orig_waiter, lock);  in rt_mutex_adjust_prio_chain()
602 		raw_spin_unlock(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
624 		if (!rt_mutex_owner(lock)) {  in rt_mutex_adjust_prio_chain()
625 			raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
630 		task = rt_mutex_owner(lock);  in rt_mutex_adjust_prio_chain()
644 		top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
648 		raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
661 	prerequeue_top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
664 	rt_mutex_dequeue(lock, waiter);  in rt_mutex_adjust_prio_chain()
685 	rt_mutex_enqueue(lock, waiter);  in rt_mutex_adjust_prio_chain()
698 	if (!rt_mutex_owner(lock)) {  in rt_mutex_adjust_prio_chain()
704 		if (prerequeue_top_waiter != rt_mutex_top_waiter(lock))  in rt_mutex_adjust_prio_chain()
705 			wake_up_process(rt_mutex_top_waiter(lock)->task);  in rt_mutex_adjust_prio_chain()
706 		raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
711 	task = rt_mutex_owner(lock);  in rt_mutex_adjust_prio_chain()
716 	if (waiter == rt_mutex_top_waiter(lock)) {  in rt_mutex_adjust_prio_chain()
739 		waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
764 	top_waiter = rt_mutex_top_waiter(lock);  in rt_mutex_adjust_prio_chain()
768 	raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_adjust_prio_chain()
808 static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,  in try_to_take_rt_mutex()  argument
811 	lockdep_assert_held(&lock->wait_lock);  in try_to_take_rt_mutex()
830 	mark_rt_mutex_waiters(lock);  in try_to_take_rt_mutex()
835 	if (rt_mutex_owner(lock))  in try_to_take_rt_mutex()
848 		if (waiter != rt_mutex_top_waiter(lock))  in try_to_take_rt_mutex()
855 		rt_mutex_dequeue(lock, waiter);  in try_to_take_rt_mutex()
866 		if (rt_mutex_has_waiters(lock)) {  in try_to_take_rt_mutex()
873 						  rt_mutex_top_waiter(lock)))  in try_to_take_rt_mutex()
905 	if (rt_mutex_has_waiters(lock))  in try_to_take_rt_mutex()
906 		rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));  in try_to_take_rt_mutex()
911 	debug_rt_mutex_lock(lock);  in try_to_take_rt_mutex()
917 	rt_mutex_set_owner(lock, task);  in try_to_take_rt_mutex()
929 static int task_blocks_on_rt_mutex(struct rt_mutex *lock,  in task_blocks_on_rt_mutex()  argument
934 	struct task_struct *owner = rt_mutex_owner(lock);  in task_blocks_on_rt_mutex()
939 	lockdep_assert_held(&lock->wait_lock);  in task_blocks_on_rt_mutex()
955 	waiter->lock = lock;  in task_blocks_on_rt_mutex()
960 	if (rt_mutex_has_waiters(lock))  in task_blocks_on_rt_mutex()
961 		top_waiter = rt_mutex_top_waiter(lock);  in task_blocks_on_rt_mutex()
962 	rt_mutex_enqueue(lock, waiter);  in task_blocks_on_rt_mutex()
972 	if (waiter == rt_mutex_top_waiter(lock)) {  in task_blocks_on_rt_mutex()
1002 	raw_spin_unlock_irq(&lock->wait_lock);  in task_blocks_on_rt_mutex()
1004 	res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,  in task_blocks_on_rt_mutex()
1007 	raw_spin_lock_irq(&lock->wait_lock);  in task_blocks_on_rt_mutex()
1019 				    struct rt_mutex *lock)  in mark_wakeup_next_waiter()  argument
1025 	waiter = rt_mutex_top_waiter(lock);  in mark_wakeup_next_waiter()
1045 	lock->owner = (void *) RT_MUTEX_HAS_WAITERS;  in mark_wakeup_next_waiter()
1068 static void remove_waiter(struct rt_mutex *lock,  in remove_waiter()  argument
1071 	bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));  in remove_waiter()
1072 	struct task_struct *owner = rt_mutex_owner(lock);  in remove_waiter()
1075 	lockdep_assert_held(&lock->wait_lock);  in remove_waiter()
1078 	rt_mutex_dequeue(lock, waiter);  in remove_waiter()
1093 	if (rt_mutex_has_waiters(lock))  in remove_waiter()
1094 		rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock));  in remove_waiter()
1113 	raw_spin_unlock_irq(&lock->wait_lock);  in remove_waiter()
1115 	rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,  in remove_waiter()
1118 	raw_spin_lock_irq(&lock->wait_lock);  in remove_waiter()
1139 	next_lock = waiter->lock;  in rt_mutex_adjust_pi()
1168 __rt_mutex_slowlock(struct rt_mutex *lock, int state,  in __rt_mutex_slowlock()  argument
1176 		if (try_to_take_rt_mutex(lock, current, waiter))  in __rt_mutex_slowlock()
1193 		raw_spin_unlock_irq(&lock->wait_lock);  in __rt_mutex_slowlock()
1199 		raw_spin_lock_irq(&lock->wait_lock);  in __rt_mutex_slowlock()
1231 rt_mutex_slowlock(struct rt_mutex *lock, int state,  in rt_mutex_slowlock()  argument
1249 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowlock()
1252 	if (try_to_take_rt_mutex(lock, current, NULL)) {  in rt_mutex_slowlock()
1253 		raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowlock()
1263 	ret = task_blocks_on_rt_mutex(lock, &waiter, current, chwalk);  in rt_mutex_slowlock()
1267 		ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);  in rt_mutex_slowlock()
1271 		remove_waiter(lock, &waiter);  in rt_mutex_slowlock()
1279 	fixup_rt_mutex_waiters(lock);  in rt_mutex_slowlock()
1281 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowlock()
1292 static inline int __rt_mutex_slowtrylock(struct rt_mutex *lock)  in __rt_mutex_slowtrylock()  argument
1294 	int ret = try_to_take_rt_mutex(lock, current, NULL);  in __rt_mutex_slowtrylock()
1300 	fixup_rt_mutex_waiters(lock);  in __rt_mutex_slowtrylock()
1308 static inline int rt_mutex_slowtrylock(struct rt_mutex *lock)  in rt_mutex_slowtrylock()  argument
1318 	if (rt_mutex_owner(lock))  in rt_mutex_slowtrylock()
1325 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowtrylock()
1327 	ret = __rt_mutex_slowtrylock(lock);  in rt_mutex_slowtrylock()
1329 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowtrylock()
1339 static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock,  in rt_mutex_slowunlock()  argument
1345 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowunlock()
1347 	debug_rt_mutex_unlock(lock);  in rt_mutex_slowunlock()
1380 	while (!rt_mutex_has_waiters(lock)) {  in rt_mutex_slowunlock()
1382 		if (unlock_rt_mutex_safe(lock, flags) == true)  in rt_mutex_slowunlock()
1385 		raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_slowunlock()
1394 	mark_wakeup_next_waiter(wake_q, lock);  in rt_mutex_slowunlock()
1395 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_slowunlock()
1407 rt_mutex_fastlock(struct rt_mutex *lock, int state,  in rt_mutex_fastlock()  argument
1408 		  int (*slowfn)(struct rt_mutex *lock, int state,  in rt_mutex_fastlock()  argument
1412 	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))  in rt_mutex_fastlock()
1415 	return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK);  in rt_mutex_fastlock()
1419 rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,  in rt_mutex_timed_fastlock()  argument
1422 			int (*slowfn)(struct rt_mutex *lock, int state,  in rt_mutex_timed_fastlock()  argument
1427 	    likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))  in rt_mutex_timed_fastlock()
1430 	return slowfn(lock, state, timeout, chwalk);  in rt_mutex_timed_fastlock()
1434 rt_mutex_fasttrylock(struct rt_mutex *lock,  in rt_mutex_fasttrylock()  argument
1435 		     int (*slowfn)(struct rt_mutex *lock))  in rt_mutex_fasttrylock()  argument
1437 	if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))  in rt_mutex_fasttrylock()
1440 	return slowfn(lock);  in rt_mutex_fasttrylock()
1455 rt_mutex_fastunlock(struct rt_mutex *lock,  in rt_mutex_fastunlock()  argument
1456 		    bool (*slowfn)(struct rt_mutex *lock,  in rt_mutex_fastunlock()  argument
1461 	if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))  in rt_mutex_fastunlock()
1464 	if (slowfn(lock, &wake_q))  in rt_mutex_fastunlock()
1468 static inline void __rt_mutex_lock(struct rt_mutex *lock, unsigned int subclass)  in __rt_mutex_lock()  argument
1472 	mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_);  in __rt_mutex_lock()
1473 	rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock);  in __rt_mutex_lock()
1483 void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass)  in rt_mutex_lock_nested()  argument
1485 	__rt_mutex_lock(lock, subclass);  in rt_mutex_lock_nested()
1496 void __sched rt_mutex_lock(struct rt_mutex *lock)  in rt_mutex_lock()  argument
1498 	__rt_mutex_lock(lock, 0);  in rt_mutex_lock()
1512 int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock)  in rt_mutex_lock_interruptible()  argument
1518 	mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);  in rt_mutex_lock_interruptible()
1519 	ret = rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock);  in rt_mutex_lock_interruptible()
1521 		mutex_release(&lock->dep_map, 1, _RET_IP_);  in rt_mutex_lock_interruptible()
1530 int __sched rt_mutex_futex_trylock(struct rt_mutex *lock)  in rt_mutex_futex_trylock()  argument
1532 	return rt_mutex_slowtrylock(lock);  in rt_mutex_futex_trylock()
1535 int __sched __rt_mutex_futex_trylock(struct rt_mutex *lock)  in __rt_mutex_futex_trylock()  argument
1537 	return __rt_mutex_slowtrylock(lock);  in __rt_mutex_futex_trylock()
1554 rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout)  in rt_mutex_timed_lock()  argument
1560 	mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);  in rt_mutex_timed_lock()
1561 	ret = rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,  in rt_mutex_timed_lock()
1565 		mutex_release(&lock->dep_map, 1, _RET_IP_);  in rt_mutex_timed_lock()
1582 int __sched rt_mutex_trylock(struct rt_mutex *lock)  in rt_mutex_trylock()  argument
1589 	ret = rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);  in rt_mutex_trylock()
1591 		mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);  in rt_mutex_trylock()
1602 void __sched rt_mutex_unlock(struct rt_mutex *lock)  in rt_mutex_unlock()  argument
1604 	mutex_release(&lock->dep_map, 1, _RET_IP_);  in rt_mutex_unlock()
1605 	rt_mutex_fastunlock(lock, rt_mutex_slowunlock);  in rt_mutex_unlock()
1613 bool __sched __rt_mutex_futex_unlock(struct rt_mutex *lock,  in __rt_mutex_futex_unlock()  argument
1616 	lockdep_assert_held(&lock->wait_lock);  in __rt_mutex_futex_unlock()
1618 	debug_rt_mutex_unlock(lock);  in __rt_mutex_futex_unlock()
1620 	if (!rt_mutex_has_waiters(lock)) {  in __rt_mutex_futex_unlock()
1621 		lock->owner = NULL;  in __rt_mutex_futex_unlock()
1631 	mark_wakeup_next_waiter(wake_q, lock);  in __rt_mutex_futex_unlock()
1636 void __sched rt_mutex_futex_unlock(struct rt_mutex *lock)  in rt_mutex_futex_unlock()  argument
1642 	raw_spin_lock_irqsave(&lock->wait_lock, flags);  in rt_mutex_futex_unlock()
1643 	postunlock = __rt_mutex_futex_unlock(lock, &wake_q);  in rt_mutex_futex_unlock()
1644 	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);  in rt_mutex_futex_unlock()
1658 void rt_mutex_destroy(struct rt_mutex *lock)  in rt_mutex_destroy()  argument
1660 	WARN_ON(rt_mutex_is_locked(lock));  in rt_mutex_destroy()
1662 	lock->magic = NULL;  in rt_mutex_destroy()
1676 void __rt_mutex_init(struct rt_mutex *lock, const char *name,  in __rt_mutex_init()  argument
1679 	lock->owner = NULL;  in __rt_mutex_init()
1680 	raw_spin_lock_init(&lock->wait_lock);  in __rt_mutex_init()
1681 	lock->waiters = RB_ROOT_CACHED;  in __rt_mutex_init()
1684 		debug_rt_mutex_init(lock, name, key);  in __rt_mutex_init()
1702 void rt_mutex_init_proxy_locked(struct rt_mutex *lock,  in rt_mutex_init_proxy_locked()  argument
1705 	__rt_mutex_init(lock, NULL, NULL);  in rt_mutex_init_proxy_locked()
1706 	debug_rt_mutex_proxy_lock(lock, proxy_owner);  in rt_mutex_init_proxy_locked()
1707 	rt_mutex_set_owner(lock, proxy_owner);  in rt_mutex_init_proxy_locked()
1722 void rt_mutex_proxy_unlock(struct rt_mutex *lock,  in rt_mutex_proxy_unlock()  argument
1725 	debug_rt_mutex_proxy_unlock(lock);  in rt_mutex_proxy_unlock()
1726 	rt_mutex_set_owner(lock, NULL);  in rt_mutex_proxy_unlock()
1729 int __rt_mutex_start_proxy_lock(struct rt_mutex *lock,  in __rt_mutex_start_proxy_lock()  argument
1735 	if (try_to_take_rt_mutex(lock, task, NULL))  in __rt_mutex_start_proxy_lock()
1739 	ret = task_blocks_on_rt_mutex(lock, waiter, task,  in __rt_mutex_start_proxy_lock()
1742 	if (ret && !rt_mutex_owner(lock)) {  in __rt_mutex_start_proxy_lock()
1753 		remove_waiter(lock, waiter);  in __rt_mutex_start_proxy_lock()
1773 int rt_mutex_start_proxy_lock(struct rt_mutex *lock,  in rt_mutex_start_proxy_lock()  argument
1779 	raw_spin_lock_irq(&lock->wait_lock);  in rt_mutex_start_proxy_lock()
1780 	ret = __rt_mutex_start_proxy_lock(lock, waiter, task);  in rt_mutex_start_proxy_lock()
1781 	raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_start_proxy_lock()
1798 struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)  in rt_mutex_next_owner()  argument
1800 	if (!rt_mutex_has_waiters(lock))  in rt_mutex_next_owner()
1803 	return rt_mutex_top_waiter(lock)->task;  in rt_mutex_next_owner()
1823 int rt_mutex_wait_proxy_lock(struct rt_mutex *lock,  in rt_mutex_wait_proxy_lock()  argument
1829 	raw_spin_lock_irq(&lock->wait_lock);  in rt_mutex_wait_proxy_lock()
1832 	ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);  in rt_mutex_wait_proxy_lock()
1837 	fixup_rt_mutex_waiters(lock);  in rt_mutex_wait_proxy_lock()
1838 	raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_wait_proxy_lock()
1862 bool rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock,  in rt_mutex_cleanup_proxy_lock()  argument
1867 	raw_spin_lock_irq(&lock->wait_lock);  in rt_mutex_cleanup_proxy_lock()
1879 	try_to_take_rt_mutex(lock, current, waiter);  in rt_mutex_cleanup_proxy_lock()
1884 	if (rt_mutex_owner(lock) != current) {  in rt_mutex_cleanup_proxy_lock()
1885 		remove_waiter(lock, waiter);  in rt_mutex_cleanup_proxy_lock()
1892 	fixup_rt_mutex_waiters(lock);  in rt_mutex_cleanup_proxy_lock()
1894 	raw_spin_unlock_irq(&lock->wait_lock);  in rt_mutex_cleanup_proxy_lock()