279 ww_mutex_lock_acquired(struct ww_mutex *ww, struct ww_acquire_ctx *ww_ctx)  in ww_mutex_lock_acquired()  argument
293 	DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);  in ww_mutex_lock_acquired()
295 	if (ww_ctx->contending_lock) {  in ww_mutex_lock_acquired()
300 		DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);  in ww_mutex_lock_acquired()
306 		DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);  in ww_mutex_lock_acquired()
307 		ww_ctx->contending_lock = NULL;  in ww_mutex_lock_acquired()
313 	DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);  in ww_mutex_lock_acquired()
315 	ww_ctx->acquired++;  in ww_mutex_lock_acquired()
316 	ww->ctx = ww_ctx;  in ww_mutex_lock_acquired()
341 	       struct ww_acquire_ctx *ww_ctx)  in __ww_mutex_die()  argument
343 	if (!ww_ctx->is_wait_die)  in __ww_mutex_die()
346 	if (waiter->ww_ctx->acquired > 0 &&  in __ww_mutex_die()
347 			__ww_ctx_stamp_after(waiter->ww_ctx, ww_ctx)) {  in __ww_mutex_die()
363 			     struct ww_acquire_ctx *ww_ctx,  in __ww_mutex_wound()  argument
386 	if (ww_ctx->acquired > 0 && __ww_ctx_stamp_after(hold_ctx, ww_ctx)) {  in __ww_mutex_wound()
417 __ww_mutex_check_waiters(struct mutex *lock, struct ww_acquire_ctx *ww_ctx)  in __ww_mutex_check_waiters()  argument
424 		if (!cur->ww_ctx)  in __ww_mutex_check_waiters()
427 		if (__ww_mutex_die(lock, cur, ww_ctx) ||  in __ww_mutex_check_waiters()
428 		    __ww_mutex_wound(lock, cur->ww_ctx, ww_ctx))  in __ww_mutex_check_waiters()
475 bool ww_mutex_spin_on_owner(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,  in ww_mutex_spin_on_owner()  argument
493 	if (ww_ctx->acquired > 0 && READ_ONCE(ww->ctx))  in ww_mutex_spin_on_owner()
524 			 struct ww_acquire_ctx *ww_ctx, struct mutex_waiter *waiter)  in mutex_spin_on_owner()  argument
547 		if (ww_ctx && !ww_mutex_spin_on_owner(lock, ww_ctx, waiter)) {  in mutex_spin_on_owner()
611 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,  in mutex_optimistic_spin()  argument
646 		if (!mutex_spin_on_owner(lock, owner, ww_ctx, waiter))  in mutex_optimistic_spin()
687 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,  in mutex_optimistic_spin()  argument
749 __ww_mutex_kill(struct mutex *lock, struct ww_acquire_ctx *ww_ctx)  in __ww_mutex_kill()  argument
751 	if (ww_ctx->acquired > 0) {  in __ww_mutex_kill()
756 		DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock);  in __ww_mutex_kill()
757 		ww_ctx->contending_lock = ww;  in __ww_mutex_kill()
804 		if (!cur->ww_ctx)  in __ww_mutex_check_kill()
827 		      struct ww_acquire_ctx *ww_ctx)  in __ww_mutex_add_waiter()  argument
833 	if (!ww_ctx) {  in __ww_mutex_add_waiter()
838 	is_wait_die = ww_ctx->is_wait_die;  in __ww_mutex_add_waiter()
849 		if (!cur->ww_ctx)  in __ww_mutex_add_waiter()
852 		if (__ww_ctx_stamp_after(ww_ctx, cur->ww_ctx)) {  in __ww_mutex_add_waiter()
859 				int ret = __ww_mutex_kill(lock, ww_ctx);  in __ww_mutex_add_waiter()
871 		__ww_mutex_die(lock, cur, ww_ctx);  in __ww_mutex_add_waiter()
889 		__ww_mutex_wound(lock, ww_ctx, ww->ctx);  in __ww_mutex_add_waiter()
901 		    struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)  in __mutex_lock_common()  argument
911 	if (use_ww_ctx && ww_ctx) {  in __mutex_lock_common()
912 		if (unlikely(ww_ctx == READ_ONCE(ww->ctx)))  in __mutex_lock_common()
920 		if (ww_ctx->acquired == 0)  in __mutex_lock_common()
921 			ww_ctx->wounded = 0;  in __mutex_lock_common()
928 	    mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx, NULL)) {  in __mutex_lock_common()
931 		if (use_ww_ctx && ww_ctx)  in __mutex_lock_common()
932 			ww_mutex_set_context_fastpath(ww, ww_ctx);  in __mutex_lock_common()
942 		if (use_ww_ctx && ww_ctx)  in __mutex_lock_common()
943 			__ww_mutex_check_waiters(lock, ww_ctx);  in __mutex_lock_common()
958 		waiter.ww_ctx = MUTEX_POISON_WW_CTX;  in __mutex_lock_common()
965 		ret = __ww_mutex_add_waiter(&waiter, lock, ww_ctx);  in __mutex_lock_common()
969 		waiter.ww_ctx = ww_ctx;  in __mutex_lock_common()
995 		if (use_ww_ctx && ww_ctx) {  in __mutex_lock_common()
996 			ret = __ww_mutex_check_kill(lock, &waiter, ww_ctx);  in __mutex_lock_common()
1008 		if ((use_ww_ctx && ww_ctx) || !first) {  in __mutex_lock_common()
1021 		    (first && mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx, &waiter)))  in __mutex_lock_common()
1030 	if (use_ww_ctx && ww_ctx) {  in __mutex_lock_common()
1035 		if (!ww_ctx->is_wait_die &&  in __mutex_lock_common()
1037 			__ww_mutex_check_waiters(lock, ww_ctx);  in __mutex_lock_common()
1050 	if (use_ww_ctx && ww_ctx)  in __mutex_lock_common()
1051 		ww_mutex_lock_acquired(ww, ww_ctx);  in __mutex_lock_common()
1078 		struct ww_acquire_ctx *ww_ctx)  in __ww_mutex_lock()  argument
1080 	return __mutex_lock_common(lock, state, subclass, nest_lock, ip, ww_ctx, true);  in __ww_mutex_lock()