306 ww_mutex_lock_acquired(struct ww_mutex *ww, struct ww_acquire_ctx *ww_ctx)  in ww_mutex_lock_acquired()  argument
320 	DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);  in ww_mutex_lock_acquired()
322 	if (ww_ctx->contending_lock) {  in ww_mutex_lock_acquired()
327 		DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);  in ww_mutex_lock_acquired()
333 		DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);  in ww_mutex_lock_acquired()
334 		ww_ctx->contending_lock = NULL;  in ww_mutex_lock_acquired()
340 	DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);  in ww_mutex_lock_acquired()
342 	ww_ctx->acquired++;  in ww_mutex_lock_acquired()
343 	ww->ctx = ww_ctx;  in ww_mutex_lock_acquired()
368 	       struct ww_acquire_ctx *ww_ctx)  in __ww_mutex_die()  argument
370 	if (!ww_ctx->is_wait_die)  in __ww_mutex_die()
373 	if (waiter->ww_ctx->acquired > 0 &&  in __ww_mutex_die()
374 			__ww_ctx_stamp_after(waiter->ww_ctx, ww_ctx)) {  in __ww_mutex_die()
390 			     struct ww_acquire_ctx *ww_ctx,  in __ww_mutex_wound()  argument
413 	if (ww_ctx->acquired > 0 && __ww_ctx_stamp_after(hold_ctx, ww_ctx)) {  in __ww_mutex_wound()
444 __ww_mutex_check_waiters(struct mutex *lock, struct ww_acquire_ctx *ww_ctx)  in __ww_mutex_check_waiters()  argument
451 		if (!cur->ww_ctx)  in __ww_mutex_check_waiters()
454 		if (__ww_mutex_die(lock, cur, ww_ctx) ||  in __ww_mutex_check_waiters()
455 		    __ww_mutex_wound(lock, cur->ww_ctx, ww_ctx))  in __ww_mutex_check_waiters()
502 bool ww_mutex_spin_on_owner(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,  in ww_mutex_spin_on_owner()  argument
520 	if (ww_ctx->acquired > 0 && READ_ONCE(ww->ctx))  in ww_mutex_spin_on_owner()
551 			 struct ww_acquire_ctx *ww_ctx, struct mutex_waiter *waiter)  in mutex_spin_on_owner()  argument
574 		if (ww_ctx && !ww_mutex_spin_on_owner(lock, ww_ctx, waiter)) {  in mutex_spin_on_owner()
638 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,  in mutex_optimistic_spin()  argument
673 		if (!mutex_spin_on_owner(lock, owner, ww_ctx, waiter))  in mutex_optimistic_spin()
714 mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,  in mutex_optimistic_spin()  argument
776 __ww_mutex_kill(struct mutex *lock, struct ww_acquire_ctx *ww_ctx)  in __ww_mutex_kill()  argument
778 	if (ww_ctx->acquired > 0) {  in __ww_mutex_kill()
783 		DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock);  in __ww_mutex_kill()
784 		ww_ctx->contending_lock = ww;  in __ww_mutex_kill()
831 		if (!cur->ww_ctx)  in __ww_mutex_check_kill()
854 		      struct ww_acquire_ctx *ww_ctx)  in __ww_mutex_add_waiter()  argument
860 	if (!ww_ctx) {  in __ww_mutex_add_waiter()
865 	is_wait_die = ww_ctx->is_wait_die;  in __ww_mutex_add_waiter()
876 		if (!cur->ww_ctx)  in __ww_mutex_add_waiter()
879 		if (__ww_ctx_stamp_after(ww_ctx, cur->ww_ctx)) {  in __ww_mutex_add_waiter()
886 				int ret = __ww_mutex_kill(lock, ww_ctx);  in __ww_mutex_add_waiter()
898 		__ww_mutex_die(lock, cur, ww_ctx);  in __ww_mutex_add_waiter()
916 		__ww_mutex_wound(lock, ww_ctx, ww->ctx);  in __ww_mutex_add_waiter()
928 		    struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)  in __mutex_lock_common()  argument
942 	if (use_ww_ctx && ww_ctx) {  in __mutex_lock_common()
943 		if (unlikely(ww_ctx == READ_ONCE(ww->ctx)))  in __mutex_lock_common()
951 		if (ww_ctx->acquired == 0)  in __mutex_lock_common()
952 			ww_ctx->wounded = 0;  in __mutex_lock_common()
959 	    mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx, NULL)) {  in __mutex_lock_common()
962 		if (use_ww_ctx && ww_ctx)  in __mutex_lock_common()
963 			ww_mutex_set_context_fastpath(ww, ww_ctx);  in __mutex_lock_common()
973 		if (use_ww_ctx && ww_ctx)  in __mutex_lock_common()
974 			__ww_mutex_check_waiters(lock, ww_ctx);  in __mutex_lock_common()
989 		waiter.ww_ctx = MUTEX_POISON_WW_CTX;  in __mutex_lock_common()
996 		ret = __ww_mutex_add_waiter(&waiter, lock, ww_ctx);  in __mutex_lock_common()
1000 		waiter.ww_ctx = ww_ctx;  in __mutex_lock_common()
1026 		if (use_ww_ctx && ww_ctx) {  in __mutex_lock_common()
1027 			ret = __ww_mutex_check_kill(lock, &waiter, ww_ctx);  in __mutex_lock_common()
1039 		if ((use_ww_ctx && ww_ctx) || !first) {  in __mutex_lock_common()
1052 		    (first && mutex_optimistic_spin(lock, ww_ctx, use_ww_ctx, &waiter)))  in __mutex_lock_common()
1061 	if (use_ww_ctx && ww_ctx) {  in __mutex_lock_common()
1066 		if (!ww_ctx->is_wait_die &&  in __mutex_lock_common()
1068 			__ww_mutex_check_waiters(lock, ww_ctx);  in __mutex_lock_common()
1081 	if (use_ww_ctx && ww_ctx)  in __mutex_lock_common()
1082 		ww_mutex_lock_acquired(ww, ww_ctx);  in __mutex_lock_common()
1109 		struct ww_acquire_ctx *ww_ctx)  in __ww_mutex_lock()  argument
1111 	return __mutex_lock_common(lock, state, subclass, nest_lock, ip, ww_ctx, true);  in __ww_mutex_lock()