43 __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)  in __mutex_init()  argument
45 	atomic_long_set(&lock->owner, 0);  in __mutex_init()
46 	raw_spin_lock_init(&lock->wait_lock);  in __mutex_init()
47 	INIT_LIST_HEAD(&lock->wait_list);  in __mutex_init()
49 	osq_lock_init(&lock->osq);  in __mutex_init()
52 	debug_mutex_init(lock, name, key);  in __mutex_init()
57  * @owner: contains: 'struct task_struct *' to the current lock owner,
62  * Bit1 indicates unlock needs to hand the lock to the top-waiter
76 static inline struct task_struct *__mutex_owner(struct mutex *lock)  in __mutex_owner()  argument
78 	return (struct task_struct *)(atomic_long_read(&lock->owner) & ~MUTEX_FLAGS);  in __mutex_owner()
86 bool mutex_is_locked(struct mutex *lock)  in mutex_is_locked()  argument
88 	return __mutex_owner(lock) != NULL;  in mutex_is_locked()
97 static inline struct task_struct *__mutex_trylock_common(struct mutex *lock, bool handoff)  in __mutex_trylock_common()  argument
101 	owner = atomic_long_read(&lock->owner);  in __mutex_trylock_common()
123 		if (atomic_long_try_cmpxchg_acquire(&lock->owner, &owner, task | flags)) {  in __mutex_trylock_common()
136 static inline bool __mutex_trylock_or_handoff(struct mutex *lock, bool handoff)  in __mutex_trylock_or_handoff()  argument
138 	return !__mutex_trylock_common(lock, handoff);  in __mutex_trylock_or_handoff()
144 static inline bool __mutex_trylock(struct mutex *lock)  in __mutex_trylock()  argument
146 	return !__mutex_trylock_common(lock, false);  in __mutex_trylock()
160 static __always_inline bool __mutex_trylock_fast(struct mutex *lock)  in __mutex_trylock_fast()  argument
165 	if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr))  in __mutex_trylock_fast()
171 static __always_inline bool __mutex_unlock_fast(struct mutex *lock)  in __mutex_unlock_fast()  argument
175 	return atomic_long_try_cmpxchg_release(&lock->owner, &curr, 0UL);  in __mutex_unlock_fast()
179 static inline void __mutex_set_flag(struct mutex *lock, unsigned long flag)  in __mutex_set_flag()  argument
181 	atomic_long_or(flag, &lock->owner);  in __mutex_set_flag()
184 static inline void __mutex_clear_flag(struct mutex *lock, unsigned long flag)  in __mutex_clear_flag()  argument
186 	atomic_long_andnot(flag, &lock->owner);  in __mutex_clear_flag()
189 static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_waiter *waiter)  in __mutex_waiter_is_first()  argument
191 	return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter;  in __mutex_waiter_is_first()
195  * Add @waiter to a given location in the lock wait_list and set the
199 __mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,  in __mutex_add_waiter()  argument
202 	debug_mutex_add_waiter(lock, waiter, current);  in __mutex_add_waiter()
205 	if (__mutex_waiter_is_first(lock, waiter))  in __mutex_add_waiter()
206 		__mutex_set_flag(lock, MUTEX_FLAG_WAITERS);  in __mutex_add_waiter()
210 __mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter)  in __mutex_remove_waiter()  argument
213 	if (likely(list_empty(&lock->wait_list)))  in __mutex_remove_waiter()
214 		__mutex_clear_flag(lock, MUTEX_FLAGS);  in __mutex_remove_waiter()
216 	debug_mutex_remove_waiter(lock, waiter, current);  in __mutex_remove_waiter()
225 static void __mutex_handoff(struct mutex *lock, struct task_struct *task)  in __mutex_handoff()  argument
227 	unsigned long owner = atomic_long_read(&lock->owner);  in __mutex_handoff()
240 		if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, new))  in __mutex_handoff()
247  * We split the mutex lock/unlock logic into separate fastpath and
252 static void __sched __mutex_lock_slowpath(struct mutex *lock);
256  * @lock: the mutex to be acquired
258  * Lock the mutex exclusively for this task. If the mutex is not
275 void __sched mutex_lock(struct mutex *lock)  in mutex_lock()  argument
279 	if (!__mutex_trylock_fast(lock))  in mutex_lock()
280 		__mutex_lock_slowpath(lock);  in mutex_lock()
292 static inline struct task_struct *__mutex_trylock_or_owner(struct mutex *lock)  in __mutex_trylock_or_owner()  argument
294 	return __mutex_trylock_common(lock, false);  in __mutex_trylock_or_owner()
298 bool ww_mutex_spin_on_owner(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,  in ww_mutex_spin_on_owner()  argument
303 	ww = container_of(lock, struct ww_mutex, base);  in ww_mutex_spin_on_owner()
322 	 * lock from a waiter with an earlier stamp, since the  in ww_mutex_spin_on_owner()
323 	 * other thread may already own a lock that we also  in ww_mutex_spin_on_owner()
326 	if (!waiter && (atomic_long_read(&lock->owner) & MUTEX_FLAG_WAITERS))  in ww_mutex_spin_on_owner()
333 	if (waiter && !__mutex_waiter_is_first(lock, waiter))  in ww_mutex_spin_on_owner()
346 bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner,  in mutex_spin_on_owner()  argument
352 	while (__mutex_owner(lock) == owner) {  in mutex_spin_on_owner()
355 		 * checking lock->owner still matches owner. If that fails,  in mutex_spin_on_owner()
362 		 * Use vcpu_is_preempted to detect lock holder preemption issue.  in mutex_spin_on_owner()
370 		if (ww_ctx && !ww_mutex_spin_on_owner(lock, ww_ctx, waiter)) {  in mutex_spin_on_owner()
385 static inline int mutex_can_spin_on_owner(struct mutex *lock)  in mutex_can_spin_on_owner()  argument
394 	owner = __mutex_owner(lock);  in mutex_can_spin_on_owner()
397 	 * As lock holder preemption issue, we both skip spinning if task is not  in mutex_can_spin_on_owner()
405 	 * If lock->owner is not set, the mutex has been released. Return true  in mutex_can_spin_on_owner()
415  * We try to spin for acquisition when we find that the lock owner
417  * need to reschedule. The rationale is that if the lock owner is
418  * running, it is likely to release the lock soon.
420  * The mutex spinners are queued up using MCS lock so that only one
422  * going to happen, there is no point in going through the lock/unlock
425  * Returns true when the lock was taken, otherwise false, indicating
429  * queue. The waiter-spinner will spin on the lock directly and concurrently
434 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,  in mutex_optimistic_spin()  argument
442 		 * is not going to take OSQ lock anyway, there is no need  in mutex_optimistic_spin()
445 		if (!mutex_can_spin_on_owner(lock))  in mutex_optimistic_spin()
451 		 * MCS (queued) lock first before spinning on the owner field.  in mutex_optimistic_spin()
453 		if (!osq_lock(&lock->osq))  in mutex_optimistic_spin()
461 		owner = __mutex_trylock_or_owner(lock);  in mutex_optimistic_spin()
467 		 * release the lock or go to sleep.  in mutex_optimistic_spin()
469 		if (!mutex_spin_on_owner(lock, owner, ww_ctx, waiter))  in mutex_optimistic_spin()
482 		osq_unlock(&lock->osq);  in mutex_optimistic_spin()
489 		osq_unlock(&lock->osq);  in mutex_optimistic_spin()
494 	 * reschedule now, before we try-lock the mutex. This avoids getting  in mutex_optimistic_spin()
510 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,  in mutex_optimistic_spin()  argument
517 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip);
521  * @lock: the mutex to be released
530 void __sched mutex_unlock(struct mutex *lock)  in mutex_unlock()  argument
533 	if (__mutex_unlock_fast(lock))  in mutex_unlock()
536 	__mutex_unlock_slowpath(lock, _RET_IP_);  in mutex_unlock()
542  * @lock: the mutex to be released
551 void __sched ww_mutex_unlock(struct ww_mutex *lock)  in ww_mutex_unlock()  argument
553 	__ww_mutex_unlock(lock);  in ww_mutex_unlock()
554 	mutex_unlock(&lock->base);  in ww_mutex_unlock()
559  * Lock a mutex (possibly interruptible), slowpath:
562 __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclass,  in __mutex_lock_common()  argument
575 	MUTEX_WARN_ON(lock->magic != lock);  in __mutex_lock_common()
577 	ww = container_of(lock, struct ww_mutex, base);  in __mutex_lock_common()
596 	mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);  in __mutex_lock_common()
598 	if (__mutex_trylock(lock) ||  in __mutex_lock_common()
599 	    mutex_optimistic_spin(lock, ww_ctx, NULL)) {  in __mutex_lock_common()
600 		/* got the lock, yay! */  in __mutex_lock_common()
601 		lock_acquired(&lock->dep_map, ip);  in __mutex_lock_common()
608 	raw_spin_lock(&lock->wait_lock);  in __mutex_lock_common()
612 	if (__mutex_trylock(lock)) {  in __mutex_lock_common()
614 			__ww_mutex_check_waiters(lock, ww_ctx);  in __mutex_lock_common()
619 	debug_mutex_lock_common(lock, &waiter);  in __mutex_lock_common()
624 	lock_contended(&lock->dep_map, ip);  in __mutex_lock_common()
628 		__mutex_add_waiter(lock, &waiter, &lock->wait_list);  in __mutex_lock_common()
634 		ret = __ww_mutex_add_waiter(&waiter, lock, ww_ctx);  in __mutex_lock_common()
645 		 * mutex_unlock() handing the lock off to us, do a trylock  in __mutex_lock_common()
649 		if (__mutex_trylock(lock))  in __mutex_lock_common()
654 		 * wait_lock. This ensures the lock cancellation is ordered  in __mutex_lock_common()
663 			ret = __ww_mutex_check_kill(lock, &waiter, ww_ctx);  in __mutex_lock_common()
668 		raw_spin_unlock(&lock->wait_lock);  in __mutex_lock_common()
671 		first = __mutex_waiter_is_first(lock, &waiter);  in __mutex_lock_common()
679 		if (__mutex_trylock_or_handoff(lock, first) ||  in __mutex_lock_common()
680 		    (first && mutex_optimistic_spin(lock, ww_ctx, &waiter)))  in __mutex_lock_common()
683 		raw_spin_lock(&lock->wait_lock);  in __mutex_lock_common()
685 	raw_spin_lock(&lock->wait_lock);  in __mutex_lock_common()
691 		 * Wound-Wait; we stole the lock (!first_waiter), check the  in __mutex_lock_common()
695 		    !__mutex_waiter_is_first(lock, &waiter))  in __mutex_lock_common()
696 			__ww_mutex_check_waiters(lock, ww_ctx);  in __mutex_lock_common()
699 	__mutex_remove_waiter(lock, &waiter);  in __mutex_lock_common()
704 	/* got the lock - cleanup and rejoice! */  in __mutex_lock_common()
705 	lock_acquired(&lock->dep_map, ip);  in __mutex_lock_common()
710 	raw_spin_unlock(&lock->wait_lock);  in __mutex_lock_common()
716 	__mutex_remove_waiter(lock, &waiter);  in __mutex_lock_common()
718 	raw_spin_unlock(&lock->wait_lock);  in __mutex_lock_common()
720 	mutex_release(&lock->dep_map, ip);  in __mutex_lock_common()
726 __mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,  in __mutex_lock()  argument
729 	return __mutex_lock_common(lock, state, subclass, nest_lock, ip, NULL, false);  in __mutex_lock()
733 __ww_mutex_lock(struct mutex *lock, unsigned int state, unsigned int subclass,  in __ww_mutex_lock()  argument
736 	return __mutex_lock_common(lock, state, subclass, NULL, ip, ww_ctx, true);  in __ww_mutex_lock()
741 mutex_lock_nested(struct mutex *lock, unsigned int subclass)  in mutex_lock_nested()  argument
743 	__mutex_lock(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);  in mutex_lock_nested()
749 _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)  in _mutex_lock_nest_lock()  argument
751 	__mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, nest, _RET_IP_);  in _mutex_lock_nest_lock()
756 mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass)  in mutex_lock_killable_nested()  argument
758 	return __mutex_lock(lock, TASK_KILLABLE, subclass, NULL, _RET_IP_);  in mutex_lock_killable_nested()
763 mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)  in mutex_lock_interruptible_nested()  argument
765 	return __mutex_lock(lock, TASK_INTERRUPTIBLE, subclass, NULL, _RET_IP_);  in mutex_lock_interruptible_nested()
770 mutex_lock_io_nested(struct mutex *lock, unsigned int subclass)  in mutex_lock_io_nested()  argument
777 	__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,  in mutex_lock_io_nested()
784 ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)  in ww_mutex_deadlock_injection()  argument
798 		ctx->contending_lock = lock;  in ww_mutex_deadlock_injection()
800 		ww_mutex_unlock(lock);  in ww_mutex_deadlock_injection()
810 ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)  in ww_mutex_lock()  argument
815 	ret =  __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE,  in ww_mutex_lock()
818 		return ww_mutex_deadlock_injection(lock, ctx);  in ww_mutex_lock()
825 ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)  in ww_mutex_lock_interruptible()  argument
830 	ret = __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE,  in ww_mutex_lock_interruptible()
834 		return ww_mutex_deadlock_injection(lock, ctx);  in ww_mutex_lock_interruptible()
843  * Release the lock, slowpath:
845 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)  in __mutex_unlock_slowpath()  argument
851 	mutex_release(&lock->dep_map, ip);  in __mutex_unlock_slowpath()
854 	 * Release the lock before (potentially) taking the spinlock such that  in __mutex_unlock_slowpath()
860 	owner = atomic_long_read(&lock->owner);  in __mutex_unlock_slowpath()
868 		if (atomic_long_try_cmpxchg_release(&lock->owner, &owner, __owner_flags(owner))) {  in __mutex_unlock_slowpath()
876 	raw_spin_lock(&lock->wait_lock);  in __mutex_unlock_slowpath()
877 	debug_mutex_unlock(lock);  in __mutex_unlock_slowpath()
878 	if (!list_empty(&lock->wait_list)) {  in __mutex_unlock_slowpath()
881 			list_first_entry(&lock->wait_list,  in __mutex_unlock_slowpath()
886 		debug_mutex_wake_waiter(lock, waiter);  in __mutex_unlock_slowpath()
891 		__mutex_handoff(lock, next);  in __mutex_unlock_slowpath()
893 	raw_spin_unlock(&lock->wait_lock);  in __mutex_unlock_slowpath()
904 __mutex_lock_killable_slowpath(struct mutex *lock);
907 __mutex_lock_interruptible_slowpath(struct mutex *lock);
911  * @lock: The mutex to be acquired.
913  * Lock the mutex like mutex_lock().  If a signal is delivered while the
918  * Return: 0 if the lock was successfully acquired or %-EINTR if a
921 int __sched mutex_lock_interruptible(struct mutex *lock)  in mutex_lock_interruptible()  argument
925 	if (__mutex_trylock_fast(lock))  in mutex_lock_interruptible()
928 	return __mutex_lock_interruptible_slowpath(lock);  in mutex_lock_interruptible()
935  * @lock: The mutex to be acquired.
937  * Lock the mutex like mutex_lock().  If a signal which will be fatal to
942  * Return: 0 if the lock was successfully acquired or %-EINTR if a
945 int __sched mutex_lock_killable(struct mutex *lock)  in mutex_lock_killable()  argument
949 	if (__mutex_trylock_fast(lock))  in mutex_lock_killable()
952 	return __mutex_lock_killable_slowpath(lock);  in mutex_lock_killable()
958  * @lock: The mutex to be acquired.
960  * Lock the mutex like mutex_lock().  While the task is waiting for this
966 void __sched mutex_lock_io(struct mutex *lock)  in mutex_lock_io()  argument
971 	mutex_lock(lock);  in mutex_lock_io()
977 __mutex_lock_slowpath(struct mutex *lock)  in __mutex_lock_slowpath()  argument
979 	__mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);  in __mutex_lock_slowpath()
983 __mutex_lock_killable_slowpath(struct mutex *lock)  in __mutex_lock_killable_slowpath()  argument
985 	return __mutex_lock(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);  in __mutex_lock_killable_slowpath()
989 __mutex_lock_interruptible_slowpath(struct mutex *lock)  in __mutex_lock_interruptible_slowpath()  argument
991 	return __mutex_lock(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_);  in __mutex_lock_interruptible_slowpath()
995 __ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)  in __ww_mutex_lock_slowpath()  argument
997 	return __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE, 0,  in __ww_mutex_lock_slowpath()
1002 __ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock,  in __ww_mutex_lock_interruptible_slowpath()  argument
1005 	return __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE, 0,  in __ww_mutex_lock_interruptible_slowpath()
1013  * @lock: the mutex to be acquired
1025 int __sched mutex_trylock(struct mutex *lock)  in mutex_trylock()  argument
1029 	MUTEX_WARN_ON(lock->magic != lock);  in mutex_trylock()
1031 	locked = __mutex_trylock(lock);  in mutex_trylock()
1033 		mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);  in mutex_trylock()
1041 ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)  in ww_mutex_lock()  argument
1045 	if (__mutex_trylock_fast(&lock->base)) {  in ww_mutex_lock()
1047 			ww_mutex_set_context_fastpath(lock, ctx);  in ww_mutex_lock()
1051 	return __ww_mutex_lock_slowpath(lock, ctx);  in ww_mutex_lock()
1056 ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)  in ww_mutex_lock_interruptible()  argument
1060 	if (__mutex_trylock_fast(&lock->base)) {  in ww_mutex_lock_interruptible()
1062 			ww_mutex_set_context_fastpath(lock, ctx);  in ww_mutex_lock_interruptible()
1066 	return __ww_mutex_lock_interruptible_slowpath(lock, ctx);  in ww_mutex_lock_interruptible()
1076  * @lock: the mutex to return holding if we dec to 0
1078  * return true and hold lock if we dec to 0, return false otherwise
1080 int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)  in atomic_dec_and_mutex_lock()  argument
1085 	/* we might hit 0, so take the lock */  in atomic_dec_and_mutex_lock()
1086 	mutex_lock(lock);  in atomic_dec_and_mutex_lock()
1089 		mutex_unlock(lock);  in atomic_dec_and_mutex_lock()
1092 	/* we hit 0, and we hold the lock */  in atomic_dec_and_mutex_lock()