Lines Matching full:lock
26 * lock->owner state tracking:
28 * lock->owner holds the task_struct pointer of the owner. Bit 0
29 * is used to keep track of the "lock has waiters" state.
32 * NULL 0 lock is free (fast acquire possible)
33 * NULL 1 lock is free and has waiters and the top waiter
34 * is going to take the lock*
35 * taskpointer 0 lock is held (fast release possible)
36 * taskpointer 1 lock is held and has waiters**
39 * possible when bit 0 of lock->owner is 0.
41 * (*) It also can be a transitional state when grabbing the lock
42 * with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
43 * we need to set the bit0 before looking at the lock, and the owner may be
47 * waiters. This can happen when grabbing the lock in the slow path.
48 * To prevent a cmpxchg of the owner releasing the lock, we need to
49 * set this bit before looking at the lock.
53 rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner) in rt_mutex_set_owner() argument
57 if (rt_mutex_has_waiters(lock)) in rt_mutex_set_owner()
60 lock->owner = (struct task_struct *)val; in rt_mutex_set_owner()
63 static inline void clear_rt_mutex_waiters(struct rt_mutex *lock) in clear_rt_mutex_waiters() argument
65 lock->owner = (struct task_struct *) in clear_rt_mutex_waiters()
66 ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS); in clear_rt_mutex_waiters()
69 static void fixup_rt_mutex_waiters(struct rt_mutex *lock) in fixup_rt_mutex_waiters() argument
71 unsigned long owner, *p = (unsigned long *) &lock->owner; in fixup_rt_mutex_waiters()
73 if (rt_mutex_has_waiters(lock)) in fixup_rt_mutex_waiters()
78 * lock->owner still has the waiters bit set, otherwise the in fixup_rt_mutex_waiters()
84 * lock(l->lock) in fixup_rt_mutex_waiters()
88 * unlock(l->lock) in fixup_rt_mutex_waiters()
92 * lock(l->lock) in fixup_rt_mutex_waiters()
96 * unlock(l->lock) in fixup_rt_mutex_waiters()
99 * lock(l->lock) in fixup_rt_mutex_waiters()
102 * unlock(l->lock) in fixup_rt_mutex_waiters()
103 * lock(l->lock) in fixup_rt_mutex_waiters()
107 * unlock(l->lock) in fixup_rt_mutex_waiters()
108 * lock(l->lock) in fixup_rt_mutex_waiters()
115 * lock(l->lock) in fixup_rt_mutex_waiters()
128 * serialized by l->lock, so nothing else can modify the waiters in fixup_rt_mutex_waiters()
150 * all future threads that attempt to [Rmw] the lock to the slowpath. As such
153 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) in mark_rt_mutex_waiters() argument
155 unsigned long owner, *p = (unsigned long *) &lock->owner; in mark_rt_mutex_waiters()
166 * 2) Drop lock->wait_lock
167 * 3) Try to unlock the lock with cmpxchg
169 static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock, in unlock_rt_mutex_safe() argument
171 __releases(lock->wait_lock) in unlock_rt_mutex_safe()
173 struct task_struct *owner = rt_mutex_owner(lock); in unlock_rt_mutex_safe()
175 clear_rt_mutex_waiters(lock); in unlock_rt_mutex_safe()
176 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in unlock_rt_mutex_safe()
182 * lock(wait_lock); in unlock_rt_mutex_safe()
184 * mark_rt_mutex_waiters(lock); in unlock_rt_mutex_safe()
185 * acquire(lock); in unlock_rt_mutex_safe()
189 * lock(wait_lock); in unlock_rt_mutex_safe()
190 * mark_rt_mutex_waiters(lock); in unlock_rt_mutex_safe()
195 * lock(wait_lock); in unlock_rt_mutex_safe()
198 * lock(wait_lock); in unlock_rt_mutex_safe()
199 * acquire(lock); in unlock_rt_mutex_safe()
201 return rt_mutex_cmpxchg_release(lock, owner, NULL); in unlock_rt_mutex_safe()
209 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) in mark_rt_mutex_waiters() argument
211 lock->owner = (struct task_struct *) in mark_rt_mutex_waiters()
212 ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS); in mark_rt_mutex_waiters()
216 * Simple slow path only version: lock->owner is protected by lock->wait_lock.
218 static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock, in unlock_rt_mutex_safe() argument
220 __releases(lock->wait_lock) in unlock_rt_mutex_safe()
222 lock->owner = NULL; in unlock_rt_mutex_safe()
223 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in unlock_rt_mutex_safe()
273 rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter) in rt_mutex_enqueue() argument
275 struct rb_node **link = &lock->waiters.rb_root.rb_node; in rt_mutex_enqueue()
292 rb_insert_color_cached(&waiter->tree_entry, &lock->waiters, leftmost); in rt_mutex_enqueue()
296 rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter) in rt_mutex_dequeue() argument
301 rb_erase_cached(&waiter->tree_entry, &lock->waiters); in rt_mutex_dequeue()
383 return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL; in task_blocked_on_lock()
398 * comparison to detect lock chain changes.
427 * [1] lock(task->pi_lock); [R] acquire [P]
430 * [4] lock = waiter->lock; [P]
431 * [5] if (!try_lock(lock->wait_lock)) { [P] try to acquire [L]
436 * [7] requeue_lock_waiter(lock, waiter); [P] + [L]
440 * [10] task = owner(lock); [L]
442 * lock(task->pi_lock); [L] acquire [P]
443 * [11] requeue_pi_waiter(tsk, waiters(lock));[P] + [L]
446 * unlock(lock->wait_lock); release [L]
459 struct rt_mutex *lock; in rt_mutex_adjust_prio_chain() local
473 * We limit the lock chain length for each invocation. in rt_mutex_adjust_prio_chain()
484 printk(KERN_WARNING "Maximum lock depth %d reached " in rt_mutex_adjust_prio_chain()
524 * the previous owner of the lock might have released the lock. in rt_mutex_adjust_prio_chain()
531 * the task might have moved on in the lock chain or even left in rt_mutex_adjust_prio_chain()
532 * the chain completely and blocks now on an unrelated lock or in rt_mutex_adjust_prio_chain()
535 * We stored the lock on which @task was blocked in @next_lock, in rt_mutex_adjust_prio_chain()
538 if (next_lock != waiter->lock) in rt_mutex_adjust_prio_chain()
578 * [4] Get the next lock in rt_mutex_adjust_prio_chain()
580 lock = waiter->lock; in rt_mutex_adjust_prio_chain()
583 * which is the reverse lock order versus the other rtmutex in rt_mutex_adjust_prio_chain()
586 if (!raw_spin_trylock(&lock->wait_lock)) { in rt_mutex_adjust_prio_chain()
594 * lock->wait_lock. in rt_mutex_adjust_prio_chain()
596 * Deadlock detection. If the lock is the same as the original in rt_mutex_adjust_prio_chain()
597 * lock which caused us to walk the lock chain or if the in rt_mutex_adjust_prio_chain()
598 * current lock is owned by the task which initiated the chain in rt_mutex_adjust_prio_chain()
601 if (lock == orig_lock || rt_mutex_owner(lock) == top_task) { in rt_mutex_adjust_prio_chain()
602 debug_rt_mutex_deadlock(chwalk, orig_waiter, lock); in rt_mutex_adjust_prio_chain()
603 raw_spin_unlock(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
609 * If we just follow the lock chain for deadlock detection, no in rt_mutex_adjust_prio_chain()
622 * [9] check_exit_conditions_3 protected by lock->wait_lock. in rt_mutex_adjust_prio_chain()
623 * If there is no owner of the lock, end of chain. in rt_mutex_adjust_prio_chain()
625 if (!rt_mutex_owner(lock)) { in rt_mutex_adjust_prio_chain()
626 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
630 /* [10] Grab the next task, i.e. owner of @lock */ in rt_mutex_adjust_prio_chain()
631 task = get_task_struct(rt_mutex_owner(lock)); in rt_mutex_adjust_prio_chain()
644 top_waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
648 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
658 * operation on @lock. We need it for the boost/deboost in rt_mutex_adjust_prio_chain()
661 prerequeue_top_waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
663 /* [7] Requeue the waiter in the lock waiter tree. */ in rt_mutex_adjust_prio_chain()
664 rt_mutex_dequeue(lock, waiter); in rt_mutex_adjust_prio_chain()
685 rt_mutex_enqueue(lock, waiter); in rt_mutex_adjust_prio_chain()
692 * [9] check_exit_conditions_3 protected by lock->wait_lock. in rt_mutex_adjust_prio_chain()
694 * We must abort the chain walk if there is no lock owner even in rt_mutex_adjust_prio_chain()
695 * in the dead lock detection case, as we have nothing to in rt_mutex_adjust_prio_chain()
698 if (!rt_mutex_owner(lock)) { in rt_mutex_adjust_prio_chain()
702 * to get the lock. in rt_mutex_adjust_prio_chain()
704 if (prerequeue_top_waiter != rt_mutex_top_waiter(lock)) in rt_mutex_adjust_prio_chain()
705 wake_up_process(rt_mutex_top_waiter(lock)->task); in rt_mutex_adjust_prio_chain()
706 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
710 /* [10] Grab the next task, i.e. the owner of @lock */ in rt_mutex_adjust_prio_chain()
711 task = get_task_struct(rt_mutex_owner(lock)); in rt_mutex_adjust_prio_chain()
715 if (waiter == rt_mutex_top_waiter(lock)) { in rt_mutex_adjust_prio_chain()
718 * waiter on the lock. Replace the previous top waiter in rt_mutex_adjust_prio_chain()
728 * The waiter was the top waiter on the lock, but is in rt_mutex_adjust_prio_chain()
738 waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
750 * and lock->wait_lock. The actual decisions are made after we in rt_mutex_adjust_prio_chain()
753 * Check whether the task which owns the current lock is pi in rt_mutex_adjust_prio_chain()
754 * blocked itself. If yes we store a pointer to the lock for in rt_mutex_adjust_prio_chain()
755 * the lock chain change detection above. After we dropped in rt_mutex_adjust_prio_chain()
760 * Store the top waiter of @lock for the end of chain walk in rt_mutex_adjust_prio_chain()
763 top_waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
767 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
773 * We reached the end of the lock chain. Stop right here. No in rt_mutex_adjust_prio_chain()
780 * If the current waiter is not the top waiter on the lock, in rt_mutex_adjust_prio_chain()
800 * Must be called with lock->wait_lock held and interrupts disabled
802 * @lock: The lock to be acquired.
803 * @task: The task which wants to acquire the lock
804 * @waiter: The waiter that is queued to the lock's wait tree if the
807 static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, in try_to_take_rt_mutex() argument
810 lockdep_assert_held(&lock->wait_lock); in try_to_take_rt_mutex()
813 * Before testing whether we can acquire @lock, we set the in try_to_take_rt_mutex()
814 * RT_MUTEX_HAS_WAITERS bit in @lock->owner. This forces all in try_to_take_rt_mutex()
815 * other tasks which try to modify @lock into the slow path in try_to_take_rt_mutex()
816 * and they serialize on @lock->wait_lock. in try_to_take_rt_mutex()
821 * - There is a lock owner. The caller must fixup the in try_to_take_rt_mutex()
822 * transient state if it does a trylock or leaves the lock in try_to_take_rt_mutex()
825 * - @task acquires the lock and there are no other in try_to_take_rt_mutex()
829 mark_rt_mutex_waiters(lock); in try_to_take_rt_mutex()
832 * If @lock has an owner, give up. in try_to_take_rt_mutex()
834 if (rt_mutex_owner(lock)) in try_to_take_rt_mutex()
839 * into @lock waiter tree. If @waiter == NULL then this is a in try_to_take_rt_mutex()
845 * @lock, give up. in try_to_take_rt_mutex()
847 if (waiter != rt_mutex_top_waiter(lock)) in try_to_take_rt_mutex()
851 * We can acquire the lock. Remove the waiter from the in try_to_take_rt_mutex()
852 * lock waiters tree. in try_to_take_rt_mutex()
854 rt_mutex_dequeue(lock, waiter); in try_to_take_rt_mutex()
858 * If the lock has waiters already we check whether @task is in try_to_take_rt_mutex()
859 * eligible to take over the lock. in try_to_take_rt_mutex()
862 * the lock. @task->pi_blocked_on is NULL, so it does in try_to_take_rt_mutex()
865 if (rt_mutex_has_waiters(lock)) { in try_to_take_rt_mutex()
872 rt_mutex_top_waiter(lock))) in try_to_take_rt_mutex()
877 * don't have to change anything in the lock in try_to_take_rt_mutex()
882 * No waiters. Take the lock without the in try_to_take_rt_mutex()
900 * Finish the lock acquisition. @task is the new owner. If in try_to_take_rt_mutex()
904 if (rt_mutex_has_waiters(lock)) in try_to_take_rt_mutex()
905 rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock)); in try_to_take_rt_mutex()
909 /* We got the lock. */ in try_to_take_rt_mutex()
910 debug_rt_mutex_lock(lock); in try_to_take_rt_mutex()
916 rt_mutex_set_owner(lock, task); in try_to_take_rt_mutex()
922 * Task blocks on lock.
926 * This must be called with lock->wait_lock held and interrupts disabled
928 static int task_blocks_on_rt_mutex(struct rt_mutex *lock, in task_blocks_on_rt_mutex() argument
933 struct task_struct *owner = rt_mutex_owner(lock); in task_blocks_on_rt_mutex()
938 lockdep_assert_held(&lock->wait_lock); in task_blocks_on_rt_mutex()
954 waiter->lock = lock; in task_blocks_on_rt_mutex()
958 /* Get the top priority waiter on the lock */ in task_blocks_on_rt_mutex()
959 if (rt_mutex_has_waiters(lock)) in task_blocks_on_rt_mutex()
960 top_waiter = rt_mutex_top_waiter(lock); in task_blocks_on_rt_mutex()
961 rt_mutex_enqueue(lock, waiter); in task_blocks_on_rt_mutex()
971 if (waiter == rt_mutex_top_waiter(lock)) { in task_blocks_on_rt_mutex()
982 /* Store the lock on which owner is blocked or NULL */ in task_blocks_on_rt_mutex()
995 * The owner can't disappear while holding a lock, in task_blocks_on_rt_mutex()
1001 raw_spin_unlock_irq(&lock->wait_lock); in task_blocks_on_rt_mutex()
1003 res = rt_mutex_adjust_prio_chain(owner, chwalk, lock, in task_blocks_on_rt_mutex()
1006 raw_spin_lock_irq(&lock->wait_lock); in task_blocks_on_rt_mutex()
1015 * Called with lock->wait_lock held and interrupts disabled.
1018 struct rt_mutex *lock) in mark_wakeup_next_waiter() argument
1024 waiter = rt_mutex_top_waiter(lock); in mark_wakeup_next_waiter()
1038 * queued on the lock until it gets the lock, this lock in mark_wakeup_next_waiter()
1042 * the top waiter can steal this lock. in mark_wakeup_next_waiter()
1044 lock->owner = (void *) RT_MUTEX_HAS_WAITERS; in mark_wakeup_next_waiter()
1062 * Remove a waiter from a lock and give up
1064 * Must be called with lock->wait_lock held and interrupts disabled. I must
1067 static void remove_waiter(struct rt_mutex *lock, in remove_waiter() argument
1070 bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock)); in remove_waiter()
1071 struct task_struct *owner = rt_mutex_owner(lock); in remove_waiter()
1074 lockdep_assert_held(&lock->wait_lock); in remove_waiter()
1077 rt_mutex_dequeue(lock, waiter); in remove_waiter()
1083 * waiter of the lock and there is an owner to update. in remove_waiter()
1092 if (rt_mutex_has_waiters(lock)) in remove_waiter()
1093 rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock)); in remove_waiter()
1097 /* Store the lock on which owner is blocked or NULL */ in remove_waiter()
1112 raw_spin_unlock_irq(&lock->wait_lock); in remove_waiter()
1114 rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock, in remove_waiter()
1117 raw_spin_lock_irq(&lock->wait_lock); in remove_waiter()
1138 next_lock = waiter->lock; in rt_mutex_adjust_pi()
1158 * @lock: the rt_mutex to take
1164 * Must be called with lock->wait_lock held and interrupts disabled
1167 __rt_mutex_slowlock(struct rt_mutex *lock, int state, in __rt_mutex_slowlock() argument
1174 /* Try to acquire the lock: */ in __rt_mutex_slowlock()
1175 if (try_to_take_rt_mutex(lock, current, waiter)) in __rt_mutex_slowlock()
1192 raw_spin_unlock_irq(&lock->wait_lock); in __rt_mutex_slowlock()
1198 raw_spin_lock_irq(&lock->wait_lock); in __rt_mutex_slowlock()
1227 * Slow path lock function:
1230 rt_mutex_slowlock(struct rt_mutex *lock, int state, in rt_mutex_slowlock() argument
1244 * rtmutex with lock->wait_lock held. But we cannot unconditionally in rt_mutex_slowlock()
1248 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowlock()
1250 /* Try to acquire the lock again: */ in rt_mutex_slowlock()
1251 if (try_to_take_rt_mutex(lock, current, NULL)) { in rt_mutex_slowlock()
1252 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_slowlock()
1262 ret = task_blocks_on_rt_mutex(lock, &waiter, current, chwalk); in rt_mutex_slowlock()
1266 ret = __rt_mutex_slowlock(lock, state, timeout, &waiter); in rt_mutex_slowlock()
1270 remove_waiter(lock, &waiter); in rt_mutex_slowlock()
1278 fixup_rt_mutex_waiters(lock); in rt_mutex_slowlock()
1280 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_slowlock()
1291 static inline int __rt_mutex_slowtrylock(struct rt_mutex *lock) in __rt_mutex_slowtrylock() argument
1293 int ret = try_to_take_rt_mutex(lock, current, NULL); in __rt_mutex_slowtrylock()
1296 * try_to_take_rt_mutex() sets the lock waiters bit in __rt_mutex_slowtrylock()
1299 fixup_rt_mutex_waiters(lock); in __rt_mutex_slowtrylock()
1305 * Slow path try-lock function:
1307 static inline int rt_mutex_slowtrylock(struct rt_mutex *lock) in rt_mutex_slowtrylock() argument
1313 * If the lock already has an owner we fail to get the lock. in rt_mutex_slowtrylock()
1314 * This can be done without taking the @lock->wait_lock as in rt_mutex_slowtrylock()
1317 if (rt_mutex_owner(lock)) in rt_mutex_slowtrylock()
1321 * The mutex has currently no owner. Lock the wait lock and try to in rt_mutex_slowtrylock()
1322 * acquire the lock. We use irqsave here to support early boot calls. in rt_mutex_slowtrylock()
1324 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowtrylock()
1326 ret = __rt_mutex_slowtrylock(lock); in rt_mutex_slowtrylock()
1328 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_slowtrylock()
1338 static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock, in rt_mutex_slowunlock() argument
1344 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowunlock()
1346 debug_rt_mutex_unlock(lock); in rt_mutex_slowunlock()
1353 * foo->lock->owner = NULL; in rt_mutex_slowunlock()
1354 * rtmutex_lock(foo->lock); <- fast path in rt_mutex_slowunlock()
1356 * rtmutex_unlock(foo->lock); <- fast path in rt_mutex_slowunlock()
1359 * raw_spin_unlock(foo->lock->wait_lock); in rt_mutex_slowunlock()
1364 * lock->wait_lock. So we do the following sequence: in rt_mutex_slowunlock()
1366 * owner = rt_mutex_owner(lock); in rt_mutex_slowunlock()
1367 * clear_rt_mutex_waiters(lock); in rt_mutex_slowunlock()
1368 * raw_spin_unlock(&lock->wait_lock); in rt_mutex_slowunlock()
1369 * if (cmpxchg(&lock->owner, owner, 0) == owner) in rt_mutex_slowunlock()
1374 * lock->owner is serialized by lock->wait_lock: in rt_mutex_slowunlock()
1376 * lock->owner = NULL; in rt_mutex_slowunlock()
1377 * raw_spin_unlock(&lock->wait_lock); in rt_mutex_slowunlock()
1379 while (!rt_mutex_has_waiters(lock)) { in rt_mutex_slowunlock()
1380 /* Drops lock->wait_lock ! */ in rt_mutex_slowunlock()
1381 if (unlock_rt_mutex_safe(lock, flags) == true) in rt_mutex_slowunlock()
1384 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowunlock()
1393 mark_wakeup_next_waiter(wake_q, lock); in rt_mutex_slowunlock()
1394 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_slowunlock()
1400 * debug aware fast / slowpath lock,trylock,unlock
1406 rt_mutex_fastlock(struct rt_mutex *lock, int state, in rt_mutex_fastlock() argument
1407 int (*slowfn)(struct rt_mutex *lock, int state, in rt_mutex_fastlock() argument
1411 if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) in rt_mutex_fastlock()
1414 return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK); in rt_mutex_fastlock()
1418 rt_mutex_timed_fastlock(struct rt_mutex *lock, int state, in rt_mutex_timed_fastlock() argument
1421 int (*slowfn)(struct rt_mutex *lock, int state, in rt_mutex_timed_fastlock() argument
1426 likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) in rt_mutex_timed_fastlock()
1429 return slowfn(lock, state, timeout, chwalk); in rt_mutex_timed_fastlock()
1433 rt_mutex_fasttrylock(struct rt_mutex *lock, in rt_mutex_fasttrylock() argument
1434 int (*slowfn)(struct rt_mutex *lock)) in rt_mutex_fasttrylock() argument
1436 if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) in rt_mutex_fasttrylock()
1439 return slowfn(lock); in rt_mutex_fasttrylock()
1454 rt_mutex_fastunlock(struct rt_mutex *lock, in rt_mutex_fastunlock() argument
1455 bool (*slowfn)(struct rt_mutex *lock, in rt_mutex_fastunlock() argument
1460 if (likely(rt_mutex_cmpxchg_release(lock, current, NULL))) in rt_mutex_fastunlock()
1463 if (slowfn(lock, &wake_q)) in rt_mutex_fastunlock()
1467 static inline void __rt_mutex_lock(struct rt_mutex *lock, unsigned int subclass) in __rt_mutex_lock() argument
1471 mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_); in __rt_mutex_lock()
1472 rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock); in __rt_mutex_lock()
1477 * rt_mutex_lock_nested - lock a rt_mutex
1479 * @lock: the rt_mutex to be locked
1482 void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass) in rt_mutex_lock_nested() argument
1484 __rt_mutex_lock(lock, subclass); in rt_mutex_lock_nested()
1491 * rt_mutex_lock - lock a rt_mutex
1493 * @lock: the rt_mutex to be locked
1495 void __sched rt_mutex_lock(struct rt_mutex *lock) in rt_mutex_lock() argument
1497 __rt_mutex_lock(lock, 0); in rt_mutex_lock()
1503 * rt_mutex_lock_interruptible - lock a rt_mutex interruptible
1505 * @lock: the rt_mutex to be locked
1511 int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock) in rt_mutex_lock_interruptible() argument
1517 mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); in rt_mutex_lock_interruptible()
1518 ret = rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock); in rt_mutex_lock_interruptible()
1520 mutex_release(&lock->dep_map, 1, _RET_IP_); in rt_mutex_lock_interruptible()
1529 int __sched rt_mutex_futex_trylock(struct rt_mutex *lock) in rt_mutex_futex_trylock() argument
1531 return rt_mutex_slowtrylock(lock); in rt_mutex_futex_trylock()
1534 int __sched __rt_mutex_futex_trylock(struct rt_mutex *lock) in __rt_mutex_futex_trylock() argument
1536 return __rt_mutex_slowtrylock(lock); in __rt_mutex_futex_trylock()
1540 * rt_mutex_timed_lock - lock a rt_mutex interruptible
1544 * @lock: the rt_mutex to be locked
1553 rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout) in rt_mutex_timed_lock() argument
1559 mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); in rt_mutex_timed_lock()
1560 ret = rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, in rt_mutex_timed_lock()
1564 mutex_release(&lock->dep_map, 1, _RET_IP_); in rt_mutex_timed_lock()
1571 * rt_mutex_trylock - try to lock a rt_mutex
1573 * @lock: the rt_mutex to be locked
1581 int __sched rt_mutex_trylock(struct rt_mutex *lock) in rt_mutex_trylock() argument
1588 ret = rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock); in rt_mutex_trylock()
1590 mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); in rt_mutex_trylock()
1599 * @lock: the rt_mutex to be unlocked
1601 void __sched rt_mutex_unlock(struct rt_mutex *lock) in rt_mutex_unlock() argument
1603 mutex_release(&lock->dep_map, 1, _RET_IP_); in rt_mutex_unlock()
1604 rt_mutex_fastunlock(lock, rt_mutex_slowunlock); in rt_mutex_unlock()
1612 bool __sched __rt_mutex_futex_unlock(struct rt_mutex *lock, in __rt_mutex_futex_unlock() argument
1615 lockdep_assert_held(&lock->wait_lock); in __rt_mutex_futex_unlock()
1617 debug_rt_mutex_unlock(lock); in __rt_mutex_futex_unlock()
1619 if (!rt_mutex_has_waiters(lock)) { in __rt_mutex_futex_unlock()
1620 lock->owner = NULL; in __rt_mutex_futex_unlock()
1630 mark_wakeup_next_waiter(wake_q, lock); in __rt_mutex_futex_unlock()
1635 void __sched rt_mutex_futex_unlock(struct rt_mutex *lock) in rt_mutex_futex_unlock() argument
1641 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_futex_unlock()
1642 postunlock = __rt_mutex_futex_unlock(lock, &wake_q); in rt_mutex_futex_unlock()
1643 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_futex_unlock()
1651 * @lock: the mutex to be destroyed
1657 void rt_mutex_destroy(struct rt_mutex *lock) in rt_mutex_destroy() argument
1659 WARN_ON(rt_mutex_is_locked(lock)); in rt_mutex_destroy()
1661 lock->magic = NULL; in rt_mutex_destroy()
1667 * __rt_mutex_init - initialize the rt lock
1669 * @lock: the rt lock to be initialized
1671 * Initialize the rt lock to unlocked state.
1673 * Initializing of a locked rt lock is not allowed
1675 void __rt_mutex_init(struct rt_mutex *lock, const char *name, in __rt_mutex_init() argument
1678 lock->owner = NULL; in __rt_mutex_init()
1679 raw_spin_lock_init(&lock->wait_lock); in __rt_mutex_init()
1680 lock->waiters = RB_ROOT_CACHED; in __rt_mutex_init()
1683 debug_rt_mutex_init(lock, name, key); in __rt_mutex_init()
1688 * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a
1691 * @lock: the rt_mutex to be locked
1701 void rt_mutex_init_proxy_locked(struct rt_mutex *lock, in rt_mutex_init_proxy_locked() argument
1704 __rt_mutex_init(lock, NULL, NULL); in rt_mutex_init_proxy_locked()
1705 debug_rt_mutex_proxy_lock(lock, proxy_owner); in rt_mutex_init_proxy_locked()
1706 rt_mutex_set_owner(lock, proxy_owner); in rt_mutex_init_proxy_locked()
1710 * rt_mutex_proxy_unlock - release a lock on behalf of owner
1712 * @lock: the rt_mutex to be locked
1721 void rt_mutex_proxy_unlock(struct rt_mutex *lock, in rt_mutex_proxy_unlock() argument
1724 debug_rt_mutex_proxy_unlock(lock); in rt_mutex_proxy_unlock()
1725 rt_mutex_set_owner(lock, NULL); in rt_mutex_proxy_unlock()
1729 * __rt_mutex_start_proxy_lock() - Start lock acquisition for another task
1730 * @lock: the rt_mutex to take
1741 * 0 - task blocked on lock
1742 * 1 - acquired the lock for task, caller should wake it up
1747 int __rt_mutex_start_proxy_lock(struct rt_mutex *lock, in __rt_mutex_start_proxy_lock() argument
1753 lockdep_assert_held(&lock->wait_lock); in __rt_mutex_start_proxy_lock()
1755 if (try_to_take_rt_mutex(lock, task, NULL)) in __rt_mutex_start_proxy_lock()
1759 ret = task_blocks_on_rt_mutex(lock, waiter, task, in __rt_mutex_start_proxy_lock()
1762 if (ret && !rt_mutex_owner(lock)) { in __rt_mutex_start_proxy_lock()
1766 * released the lock while we were walking the in __rt_mutex_start_proxy_lock()
1778 * rt_mutex_start_proxy_lock() - Start lock acquisition for another task
1779 * @lock: the rt_mutex to take
1790 * 0 - task blocked on lock
1791 * 1 - acquired the lock for task, caller should wake it up
1796 int rt_mutex_start_proxy_lock(struct rt_mutex *lock, in rt_mutex_start_proxy_lock() argument
1802 raw_spin_lock_irq(&lock->wait_lock); in rt_mutex_start_proxy_lock()
1803 ret = __rt_mutex_start_proxy_lock(lock, waiter, task); in rt_mutex_start_proxy_lock()
1805 remove_waiter(lock, waiter); in rt_mutex_start_proxy_lock()
1806 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_start_proxy_lock()
1812 * rt_mutex_next_owner - return the next owner of the lock
1814 * @lock: the rt lock query
1816 * Returns the next owner of the lock or NULL
1818 * Caller has to serialize against other accessors to the lock
1823 struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock) in rt_mutex_next_owner() argument
1825 if (!rt_mutex_has_waiters(lock)) in rt_mutex_next_owner()
1828 return rt_mutex_top_waiter(lock)->task; in rt_mutex_next_owner()
1832 * rt_mutex_wait_proxy_lock() - Wait for lock acquisition
1833 * @lock: the rt_mutex we were woken on
1838 * Wait for the the lock acquisition started on our behalf by
1848 int rt_mutex_wait_proxy_lock(struct rt_mutex *lock, in rt_mutex_wait_proxy_lock() argument
1854 raw_spin_lock_irq(&lock->wait_lock); in rt_mutex_wait_proxy_lock()
1857 ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter); in rt_mutex_wait_proxy_lock()
1862 fixup_rt_mutex_waiters(lock); in rt_mutex_wait_proxy_lock()
1863 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_wait_proxy_lock()
1869 * rt_mutex_cleanup_proxy_lock() - Cleanup failed lock acquisition
1870 * @lock: the rt_mutex we were woken on
1876 * Unless we acquired the lock; we're still enqueued on the wait-list and can
1883 * false - we acquired the lock after rt_mutex_wait_proxy_lock() returned,
1888 bool rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock, in rt_mutex_cleanup_proxy_lock() argument
1893 raw_spin_lock_irq(&lock->wait_lock); in rt_mutex_cleanup_proxy_lock()
1895 * Do an unconditional try-lock, this deals with the lock stealing in rt_mutex_cleanup_proxy_lock()
1901 * we will own the lock and it will have removed the waiter. If we in rt_mutex_cleanup_proxy_lock()
1905 try_to_take_rt_mutex(lock, current, waiter); in rt_mutex_cleanup_proxy_lock()
1910 if (rt_mutex_owner(lock) != current) { in rt_mutex_cleanup_proxy_lock()
1911 remove_waiter(lock, waiter); in rt_mutex_cleanup_proxy_lock()
1918 fixup_rt_mutex_waiters(lock); in rt_mutex_cleanup_proxy_lock()
1920 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_cleanup_proxy_lock()