 @@ -1,14 +1,14 @@
-/*	$NetBSD: linux_ww_mutex.c,v 1.7.2.1 2023/08/01 16:56:55 martin Exp $	*/
+/*	$NetBSD: linux_ww_mutex.c,v 1.7.2.2 2023/08/01 17:16:26 martin Exp $	*/
 /*-
  * Copyright (c) 2014 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Taylor R. Campbell.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -20,27 +20,27 @@
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: linux_ww_mutex.c,v 1.7.2.1 2023/08/01 16:56:55 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: linux_ww_mutex.c,v 1.7.2.2 2023/08/01 17:16:26 martin Exp $");
 #include <sys/types.h>
 #include <sys/atomic.h>
 #include <sys/condvar.h>
 #include <sys/lockdebug.h>
 #include <sys/lwp.h>
 #include <sys/mutex.h>
 #include <sys/rbtree.h>
 #include <linux/ww_mutex.h>
 #include <linux/errno.h>
 #define	WW_WANTLOCK(WW)							      \
 @@ -158,118 +158,183 @@ ww_dump(const volatile void *cookie, loc
 	default:
 		pr("unknown\n");
 		break;
+	}
+}
 static lockops_t ww_lockops = {
 	.lo_name = "Wait/wound mutex",
 	.lo_type = LOCKOPS_SLEEP,
 	.lo_dump = ww_dump,
 };
 #endif
 /*
  * ww_mutex_init(mutex, class)
+ *
  *	Initialize mutex in the given class.  Must precede any other
  *	ww_mutex_* operations.  After done, mutex must be destroyed
  *	with ww_mutex_destroy.
  */
 void
 ww_mutex_init(struct ww_mutex *mutex, struct ww_class *class)
+{
 	/*
 	 * XXX Apparently Linux takes these with spin locks held.  That
 	 * strikes me as a bad idea, but so it is...
 	 */
 	mutex_init(&mutex->wwm_lock, MUTEX_DEFAULT, IPL_VM);
 	mutex->wwm_state = WW_UNLOCKED;
 	mutex->wwm_class = class;
 	rb_tree_init(&mutex->wwm_waiters, &ww_acquire_ctx_rb_ops);
 	cv_init(&mutex->wwm_cv, "linuxwwm");
 #ifdef LOCKDEBUG
 	mutex->wwm_debug = LOCKDEBUG_ALLOC(mutex, &ww_lockops,
 	    (uintptr_t)__builtin_return_address(0));
 #endif
+}
 /*
  * ww_mutex_destroy(mutex)
+ *
  *	Destroy mutex initialized by ww_mutex_init.  Caller must not be
  *	with any other ww_mutex_* operations except after
  *	reinitializing with ww_mutex_init.
  */
 void
 ww_mutex_destroy(struct ww_mutex *mutex)
+{
 	KASSERT(mutex->wwm_state == WW_UNLOCKED);
 #ifdef LOCKDEBUG
 	LOCKDEBUG_FREE(mutex->wwm_debug, mutex);
 #endif
 	cv_destroy(&mutex->wwm_cv);
 #if 0
 	rb_tree_destroy(&mutex->wwm_waiters, &ww_acquire_ctx_rb_ops);
 #endif
 	KASSERT(mutex->wwm_state == WW_UNLOCKED);
 	mutex_destroy(&mutex->wwm_lock);
+}
 /*
- * XXX WARNING: This returns true if it is locked by ANYONE.  Does not
+ * ww_mutex_is_locked(mutex)
  * mean `Do I hold this lock?' (answering which really requires an
- * acquire context).
+ *	True if anyone holds mutex locked at the moment, false if not.
  *	Answer is stale as soon returned unless mutex is held by
  *	caller.
+ *
  *	XXX WARNING: This returns true if it is locked by ANYONE.  Does
  *	not mean `Do I hold this lock?' (answering which really
  *	requires an acquire context).
  */
 bool
 ww_mutex_is_locked(struct ww_mutex *mutex)
+{
 	int locked;
 	mutex_enter(&mutex->wwm_lock);
 	switch (mutex->wwm_state) {
 	case WW_UNLOCKED:
 		locked = false;
 		break;
 	case WW_OWNED:
 	case WW_CTX:
 	case WW_WANTOWN:
 		locked = true;
 		break;
 	default:
 		panic("wait/wound mutex %p in bad state: %d", mutex,
 		    (int)mutex->wwm_state);
+	}
 	mutex_exit(&mutex->wwm_lock);
 	return locked;
+}
 /*
  * ww_mutex_state_wait(mutex, state)
+ *
  *	Wait for mutex, which must be in the given state, to transition
  *	to another state.  Uninterruptible; never fails.
+ *
  *	Caller must hold mutex's internal lock.
+ *
  *	May sleep.
+ *
  *	Internal subroutine.
  */
 static void
 ww_mutex_state_wait(struct ww_mutex *mutex, enum ww_mutex_state state)
+{
 	KASSERT(mutex_owned(&mutex->wwm_lock));
 	KASSERT(mutex->wwm_state == state);
 	do cv_wait(&mutex->wwm_cv, &mutex->wwm_lock);
 	while (mutex->wwm_state == state);
+}
 /*
  * ww_mutex_state_wait_sig(mutex, state)
+ *
  *	Wait for mutex, which must be in the given state, to transition
  *	to another state, or fail if interrupted by a signal.  Return 0
  *	on success, -EINTR if interrupted by a signal.
+ *
  *	Caller must hold mutex's internal lock.
+ *
  *	May sleep.
+ *
  *	Internal subroutine.
  */
 static int
 ww_mutex_state_wait_sig(struct ww_mutex *mutex, enum ww_mutex_state state)
+{
 	int ret;
 	KASSERT(mutex_owned(&mutex->wwm_lock));
 	KASSERT(mutex->wwm_state == state);
 	do {
 		/* XXX errno NetBSD->Linux */
 		ret = -cv_wait_sig(&mutex->wwm_cv, &mutex->wwm_lock);
-		if (ret == -ERESTART)
+		if (ret) {
-			ret = -ERESTARTSYS;
+			KASSERTMSG((ret == -EINTR || ret == -ERESTART),
-		if (ret)
+			    "ret=%d", ret);
 			ret = -EINTR;
 			break;
+		}
 	} while (mutex->wwm_state == state);
 	KASSERTMSG((ret == 0 || ret == -EINTR), "ret=%d", ret);
 	return ret;
+}
 /*
  * ww_mutex_lock_wait(mutex, ctx)
+ *
  *	With mutex locked and in the WW_CTX or WW_WANTOWN state, owned
  *	by another thread with an acquire context, wait to acquire
  *	mutex.  While waiting, record ctx in the tree of waiters.  Does
  *	not update the mutex state otherwise.
+ *
  *	Caller must not already hold mutex.  Caller must hold mutex's
  *	internal lock.  Uninterruptible; never fails.
+ *
  *	May sleep.
+ *
  *	Internal subroutine.
  */
 static void
 ww_mutex_lock_wait(struct ww_mutex *mutex, struct ww_acquire_ctx *ctx)
+{
 	struct ww_acquire_ctx *collision __diagused;
 	KASSERT(mutex_owned(&mutex->wwm_lock));
 	KASSERT((mutex->wwm_state == WW_CTX) ||
 	    (mutex->wwm_state == WW_WANTOWN));
 	KASSERT(mutex->wwm_u.ctx != ctx);
 	KASSERTMSG((ctx->wwx_class == mutex->wwm_u.ctx->wwx_class),
 	    "ww mutex class mismatch: %p != %p",
 	    ctx->wwx_class, mutex->wwm_u.ctx->wwx_class);
 @@ -281,26 +346,42 @@ ww_mutex_lock_wait(struct ww_mutex *mute
 	collision = rb_tree_insert_node(&mutex->wwm_waiters, ctx);
 	KASSERTMSG((collision == ctx),
 	    "ticket number reused: %"PRId64" (%p) %"PRId64" (%p)",
 	    ctx->wwx_ticket, ctx, collision->wwx_ticket, collision);
 	do cv_wait(&mutex->wwm_cv, &mutex->wwm_lock);
 	while (!(((mutex->wwm_state == WW_CTX) ||
 		    (mutex->wwm_state == WW_WANTOWN)) &&
 		 (mutex->wwm_u.ctx == ctx)));
 	rb_tree_remove_node(&mutex->wwm_waiters, ctx);
+}
 /*
  * ww_mutex_lock_wait_sig(mutex, ctx)
+ *
  *	With mutex locked and in the WW_CTX or WW_WANTOWN state, owned
  *	by another thread with an acquire context, wait to acquire
  *	mutex and return 0, or return -EINTR if interrupted by a
  *	signal.  While waiting, record ctx in the tree of waiters.
  *	Does not update the mutex state otherwise.
+ *
  *	Caller must not already hold mutex.  Caller must hold mutex's
  *	internal lock.
+ *
  *	May sleep.
+ *
  *	Internal subroutine.
  */
 static int
 ww_mutex_lock_wait_sig(struct ww_mutex *mutex, struct ww_acquire_ctx *ctx)
+{
 	struct ww_acquire_ctx *collision __diagused;
 	int ret;
 	KASSERT(mutex_owned(&mutex->wwm_lock));
 	KASSERT((mutex->wwm_state == WW_CTX) ||
 	    (mutex->wwm_state == WW_WANTOWN));
 	KASSERT(mutex->wwm_u.ctx != ctx);
 	KASSERTMSG((ctx->wwx_class == mutex->wwm_u.ctx->wwx_class),
 	    "ww mutex class mismatch: %p != %p",
 @@ -308,38 +389,51 @@ ww_mutex_lock_wait_sig(struct ww_mutex *
 	KASSERTMSG((mutex->wwm_u.ctx->wwx_ticket != ctx->wwx_ticket),
 	    "ticket number reused: %"PRId64" (%p) %"PRId64" (%p)",
 	    ctx->wwx_ticket, ctx,
 	    mutex->wwm_u.ctx->wwx_ticket, mutex->wwm_u.ctx);
 	collision = rb_tree_insert_node(&mutex->wwm_waiters, ctx);
 	KASSERTMSG((collision == ctx),
 	    "ticket number reused: %"PRId64" (%p) %"PRId64" (%p)",
 	    ctx->wwx_ticket, ctx, collision->wwx_ticket, collision);
 	do {
 		/* XXX errno NetBSD->Linux */
 		ret = -cv_wait_sig(&mutex->wwm_cv, &mutex->wwm_lock);
-		if (ret == -ERESTART)
+		if (ret) {
-			ret = -ERESTARTSYS;
+			KASSERTMSG((ret == -EINTR || ret == -ERESTART),
-		if (ret)
+			    "ret=%d", ret);
 			ret = -EINTR;
 			goto out;
+		}
 	} while (!(((mutex->wwm_state == WW_CTX) ||
 		    (mutex->wwm_state == WW_WANTOWN)) &&
 		(mutex->wwm_u.ctx == ctx)));
 out:	rb_tree_remove_node(&mutex->wwm_waiters, ctx);
 	KASSERTMSG((ret == 0 || ret == -EINTR), "ret=%d", ret);
 	return ret;
+}
 /*
  * ww_mutex_lock_noctx(mutex)
+ *
  *	Acquire mutex without an acquire context.  Caller must not
  *	already hold the mutex.  Uninterruptible; never fails.
+ *
  *	May sleep.
+ *
  *	Internal subroutine, implementing ww_mutex_lock(..., NULL).
  */
 static void
 ww_mutex_lock_noctx(struct ww_mutex *mutex)
+{
 	mutex_enter(&mutex->wwm_lock);
 retry:	switch (mutex->wwm_state) {
 	case WW_UNLOCKED:
 		mutex->wwm_state = WW_OWNED;
 		mutex->wwm_u.owner = curlwp;
 		break;
 	case WW_OWNED:
 		KASSERTMSG((mutex->wwm_u.owner != curlwp),
 		    "locking %p against myself: %p", mutex, curlwp);
 @@ -354,82 +448,116 @@ retry:	switch (mutex->wwm_state) {
 		    "locking %p against myself: %p", mutex, curlwp);
 		ww_mutex_state_wait(mutex, WW_WANTOWN);
 		goto retry;
 	default:
 		panic("wait/wound mutex %p in bad state: %d",
 		    mutex, (int)mutex->wwm_state);
+	}
 	KASSERT(mutex->wwm_state == WW_OWNED);
 	KASSERT(mutex->wwm_u.owner == curlwp);
 	WW_LOCKED(mutex);
 	mutex_exit(&mutex->wwm_lock);
+}
 /*
  * ww_mutex_lock_noctx_sig(mutex)
+ *
  *	Acquire mutex without an acquire context and return 0, or fail
  *	and return -EINTR if interrupted by a signal.  Caller must not
  *	already hold the mutex.
+ *
  *	May sleep.
+ *
  *	Internal subroutine, implementing
  *	ww_mutex_lock_interruptible(..., NULL).
  */
 static int
 ww_mutex_lock_noctx_sig(struct ww_mutex *mutex)
+{
 	int ret;
 	mutex_enter(&mutex->wwm_lock);
 retry:	switch (mutex->wwm_state) {
 	case WW_UNLOCKED:
 		mutex->wwm_state = WW_OWNED;
 		mutex->wwm_u.owner = curlwp;
 		break;
 	case WW_OWNED:
 		KASSERTMSG((mutex->wwm_u.owner != curlwp),
 		    "locking %p against myself: %p", mutex, curlwp);
 		ret = ww_mutex_state_wait_sig(mutex, WW_OWNED);
-		if (ret)
+		if (ret) {
 			KASSERTMSG(ret == -EINTR, "ret=%d", ret);
 			goto out;
+		}
 		goto retry;
 	case WW_CTX:
 		KASSERT(mutex->wwm_u.ctx != NULL);
 		mutex->wwm_state = WW_WANTOWN;
 		/* FALLTHROUGH */
 	case WW_WANTOWN:
 		KASSERTMSG((mutex->wwm_u.ctx->wwx_owner != curlwp),
 		    "locking %p against myself: %p", mutex, curlwp);
 		ret = ww_mutex_state_wait_sig(mutex, WW_WANTOWN);
-		if (ret)
+		if (ret) {
 			KASSERTMSG(ret == -EINTR, "ret=%d", ret);
 			goto out;
+		}
 		goto retry;
 	default:
 		panic("wait/wound mutex %p in bad state: %d",
 		    mutex, (int)mutex->wwm_state);
+	}
 	KASSERT(mutex->wwm_state == WW_OWNED);
 	KASSERT(mutex->wwm_u.owner == curlwp);
 	WW_LOCKED(mutex);
 	ret = 0;
 out:	mutex_exit(&mutex->wwm_lock);
 	KASSERTMSG((ret == 0 || ret == -EINTR), "ret=%d", ret);
 	return ret;
+}
 /*
  * ww_mutex_lock(mutex, ctx)
+ *
  *	Lock the mutex and return 0, or fail if impossible.
+ *
  *	- If ctx is null, caller must not hold mutex, and ww_mutex_lock
  *	  always succeeds and returns 0.
+ *
  *	- If ctx is nonnull, then:
  *	  . Fail with -EALREADY if caller already holds mutex.
  *	  . Fail with -EDEADLK if someone else holds mutex but there is
  *	    a cycle.
+ *
  *	May sleep.
  */
 int
 ww_mutex_lock(struct ww_mutex *mutex, struct ww_acquire_ctx *ctx)
+{
 	int ret;
 	/*
 	 * We do not WW_WANTLOCK at the beginning because we may
 	 * correctly already hold it, if we have a context, in which
 	 * case we must return EALREADY to the caller.
 	 */
 	ASSERT_SLEEPABLE();
 	if (ctx == NULL) {
 		WW_WANTLOCK(mutex);
 		ww_mutex_lock_noctx(mutex);
-		return 0;
+		ret = 0;
 		goto out;
+	}
 	KASSERTMSG((ctx->wwx_owner == curlwp),
 	    "ctx %p owned by %p, not self (%p)", ctx, ctx->wwx_owner, curlwp);
 	KASSERTMSG(!ctx->wwx_acquire_done,
 	    "ctx %p done acquiring locks, can't acquire more", ctx);
 	KASSERTMSG((ctx->wwx_acquired != ~0U),
 	    "ctx %p finished, can't be used any more", ctx);
 	KASSERTMSG((ctx->wwx_class == mutex->wwm_class),
 	    "ctx %p in class %p, mutex %p in class %p",
 	    ctx, ctx->wwx_class, mutex, mutex->wwm_class);
 	mutex_enter(&mutex->wwm_lock);
 @@ -456,168 +584,212 @@ retry:	switch (mutex->wwm_state) {
+	}
 	KASSERT(mutex->wwm_state == WW_CTX);
 	KASSERT(mutex->wwm_u.ctx != NULL);
 	KASSERT((mutex->wwm_u.ctx == ctx) ||
 	    (mutex->wwm_u.ctx->wwx_owner != curlwp));
 	if (mutex->wwm_u.ctx == ctx) {
 		/*
 		 * We already own it.  Yes, this can happen correctly
 		 * for objects whose locking order is determined by
 		 * userland.
 		 */
-		mutex_exit(&mutex->wwm_lock);
+		ret = -EALREADY;
-		return -EALREADY;
+		goto out_unlock;
+	}
 	/*
 	 * We do not own it.  We can safely assert to LOCKDEBUG that we
 	 * want it.
 	 */
 	WW_WANTLOCK(mutex);
 	if (mutex->wwm_u.ctx->wwx_ticket < ctx->wwx_ticket) {
 		/*
 		 * Owned by a higher-priority party.  Tell the caller
 		 * to unlock everything and start over.
 		 */
 		KASSERTMSG((ctx->wwx_class == mutex->wwm_u.ctx->wwx_class),
 		    "ww mutex class mismatch: %p != %p",
 		    ctx->wwx_class, mutex->wwm_u.ctx->wwx_class);
-		mutex_exit(&mutex->wwm_lock);
+		ret = -EDEADLK;
-		return -EDEADLK;
+		goto out_unlock;
+	}
 	/*
 	 * Owned by a lower-priority party.  Ask that party to wake us
 	 * when it is done or it realizes it needs to back off.
 	 */
 	ww_mutex_lock_wait(mutex, ctx);
 locked:	KASSERT((mutex->wwm_state == WW_CTX) ||
 	    (mutex->wwm_state == WW_WANTOWN));
 	KASSERT(mutex->wwm_u.ctx == ctx);
 	WW_LOCKED(mutex);
 	ctx->wwx_acquired++;
 	ret = 0;
 out_unlock:
 	mutex_exit(&mutex->wwm_lock);
-	return 0;
+out:	KASSERTMSG((ret == 0 || ret == -EALREADY || ret == -EDEADLK),
 	    "ret=%d", ret);
 	return ret;
+}
 /*
  * ww_mutex_lock_interruptible(mutex, ctx)
+ *
  *	Lock the mutex and return 0, or fail if impossible or
  *	interrupted.
+ *
  *	- If ctx is null, caller must not hold mutex, and ww_mutex_lock
  *	  always succeeds and returns 0.
+ *
  *	- If ctx is nonnull, then:
  *	  . Fail with -EALREADY if caller already holds mutex.
  *	  . Fail with -EDEADLK if someone else holds mutex but there is
  *	    a cycle.
  *	  . Fail with -EINTR if interrupted by a signal.
+ *
  *	May sleep.
  */
 int
 ww_mutex_lock_interruptible(struct ww_mutex *mutex, struct ww_acquire_ctx *ctx)
+{
 	int ret;
 	/*
 	 * We do not WW_WANTLOCK at the beginning because we may
 	 * correctly already hold it, if we have a context, in which
 	 * case we must return EALREADY to the caller.
 	 */
 	ASSERT_SLEEPABLE();
 	if (ctx == NULL) {
 		WW_WANTLOCK(mutex);
-		return ww_mutex_lock_noctx_sig(mutex);
+		ret = ww_mutex_lock_noctx_sig(mutex);
 		KASSERTMSG((ret == 0 || ret == -EINTR), "ret=%d", ret);
 		goto out;
+	}
 	KASSERTMSG((ctx->wwx_owner == curlwp),
 	    "ctx %p owned by %p, not self (%p)", ctx, ctx->wwx_owner, curlwp);
 	KASSERTMSG(!ctx->wwx_acquire_done,
 	    "ctx %p done acquiring locks, can't acquire more", ctx);
 	KASSERTMSG((ctx->wwx_acquired != ~0U),
 	    "ctx %p finished, can't be used any more", ctx);
 	KASSERTMSG((ctx->wwx_class == mutex->wwm_class),
 	    "ctx %p in class %p, mutex %p in class %p",
 	    ctx, ctx->wwx_class, mutex, mutex->wwm_class);
 	mutex_enter(&mutex->wwm_lock);
 retry:	switch (mutex->wwm_state) {
 	case WW_UNLOCKED:
 		WW_WANTLOCK(mutex);
 		mutex->wwm_state = WW_CTX;
 		mutex->wwm_u.ctx = ctx;
 		goto locked;
 	case WW_OWNED:
 		WW_WANTLOCK(mutex);
 		KASSERTMSG((mutex->wwm_u.owner != curlwp),
 		    "locking %p against myself: %p", mutex, curlwp);
 		ret = ww_mutex_state_wait_sig(mutex, WW_OWNED);
-		if (ret)
+		if (ret) {
-			goto out;
+			KASSERTMSG(ret == -EINTR, "ret=%d", ret);
 			goto out_unlock;
+		}
 		goto retry;
 	case WW_CTX:
 		break;
 	case WW_WANTOWN:
 		ret = ww_mutex_state_wait_sig(mutex, WW_WANTOWN);
-		if (ret)
+		if (ret) {
-			goto out;
+			KASSERTMSG(ret == -EINTR, "ret=%d", ret);
 			goto out_unlock;
+		}
 		goto retry;
 	default:
 		panic("wait/wound mutex %p in bad state: %d",
 		    mutex, (int)mutex->wwm_state);
+	}
 	KASSERT(mutex->wwm_state == WW_CTX);
 	KASSERT(mutex->wwm_u.ctx != NULL);
 	KASSERT((mutex->wwm_u.ctx == ctx) ||
 	    (mutex->wwm_u.ctx->wwx_owner != curlwp));
 	if (mutex->wwm_u.ctx == ctx) {
 		/*
 		 * We already own it.  Yes, this can happen correctly
 		 * for objects whose locking order is determined by
 		 * userland.
 		 */
-		mutex_exit(&mutex->wwm_lock);
+		ret = -EALREADY;
-		return -EALREADY;
+		goto out_unlock;
+	}
 	/*
 	 * We do not own it.  We can safely assert to LOCKDEBUG that we
 	 * want it.
 	 */
 	WW_WANTLOCK(mutex);
 	if (mutex->wwm_u.ctx->wwx_ticket < ctx->wwx_ticket) {
 		/*
 		 * Owned by a higher-priority party.  Tell the caller
 		 * to unlock everything and start over.
 		 */
 		KASSERTMSG((ctx->wwx_class == mutex->wwm_u.ctx->wwx_class),
 		    "ww mutex class mismatch: %p != %p",
 		    ctx->wwx_class, mutex->wwm_u.ctx->wwx_class);
-		mutex_exit(&mutex->wwm_lock);
+		ret = -EDEADLK;
-		return -EDEADLK;
+		goto out_unlock;
+	}
 	/*
 	 * Owned by a lower-priority party.  Ask that party to wake us
 	 * when it is done or it realizes it needs to back off.
 	 */
 	ret = ww_mutex_lock_wait_sig(mutex, ctx);
-	if (ret)
+	if (ret) {
-		goto out;
+		KASSERTMSG(ret == -EINTR, "ret=%d", ret);
 		goto out_unlock;
+	}
 locked:	KASSERT((mutex->wwm_state == WW_CTX) ||
 	    (mutex->wwm_state == WW_WANTOWN));
 	KASSERT(mutex->wwm_u.ctx == ctx);
 	WW_LOCKED(mutex);
 	ctx->wwx_acquired++;
 	ret = 0;
-out:	mutex_exit(&mutex->wwm_lock);
+out_unlock:
 	mutex_exit(&mutex->wwm_lock);
 out:	KASSERTMSG((ret == 0 || ret == -EALREADY || ret == -EDEADLK ||
 		ret == -EINTR), "ret=%d", ret);
 	return ret;
+}
 /*
  * ww_mutex_lock_slow(mutex, ctx)
+ *
  *	Slow path: After ww_mutex_lock* has failed with -EDEADLK, and
  *	after the caller has ditched all its locks, wait for the owner
  *	of mutex to relinquish mutex before the caller can start over
  *	acquiring locks again.
+ *
  *	Uninterruptible; never fails.
+ *
  *	May sleep.
  */
 void
 ww_mutex_lock_slow(struct ww_mutex *mutex, struct ww_acquire_ctx *ctx)
+{
 	/* Caller must not try to lock against self here.  */
 	WW_WANTLOCK(mutex);
 	ASSERT_SLEEPABLE();
 	if (ctx == NULL) {
 		ww_mutex_lock_noctx(mutex);
 		return;
+	}
 @@ -664,99 +836,127 @@ retry:	switch (mutex->wwm_state) {
 	 * Owned by another party, of any priority.  Ask that party to
 	 * wake us when it's done.
 	 */
 	ww_mutex_lock_wait(mutex, ctx);
 locked:	KASSERT((mutex->wwm_state == WW_CTX) ||
 	    (mutex->wwm_state == WW_WANTOWN));
 	KASSERT(mutex->wwm_u.ctx == ctx);
 	WW_LOCKED(mutex);
 	ctx->wwx_acquired++;
 	mutex_exit(&mutex->wwm_lock);
+}
 /*
  * ww_mutex_lock_slow(mutex, ctx)
+ *
  *	Slow path: After ww_mutex_lock* has failed with -EDEADLK, and
  *	after the caller has ditched all its locks, wait for the owner
  *	of mutex to relinquish mutex before the caller can start over
  *	acquiring locks again, or fail with -EINTR if interrupted by a
  *	signal.
+ *
  *	May sleep.
  */
 int
 ww_mutex_lock_slow_interruptible(struct ww_mutex *mutex,
     struct ww_acquire_ctx *ctx)
+{
 	int ret;
 	WW_WANTLOCK(mutex);
 	ASSERT_SLEEPABLE();
-	if (ctx == NULL)
+	if (ctx == NULL) {
-		return ww_mutex_lock_noctx_sig(mutex);
+		ret = ww_mutex_lock_noctx_sig(mutex);
 		KASSERTMSG((ret == 0 || ret == -EINTR), "ret=%d", ret);
 		goto out;
+	}
 	KASSERTMSG((ctx->wwx_owner == curlwp),
 	    "ctx %p owned by %p, not self (%p)", ctx, ctx->wwx_owner, curlwp);
 	KASSERTMSG(!ctx->wwx_acquire_done,
 	    "ctx %p done acquiring locks, can't acquire more", ctx);
 	KASSERTMSG((ctx->wwx_acquired != ~0U),
 	    "ctx %p finished, can't be used any more", ctx);
 	KASSERTMSG((ctx->wwx_acquired == 0),
 	    "ctx %p still holds %u locks, not allowed in slow path",
 	    ctx, ctx->wwx_acquired);
 	KASSERTMSG((ctx->wwx_class == mutex->wwm_class),
 	    "ctx %p in class %p, mutex %p in class %p",
 	    ctx, ctx->wwx_class, mutex, mutex->wwm_class);
 	mutex_enter(&mutex->wwm_lock);
 retry:	switch (mutex->wwm_state) {
 	case WW_UNLOCKED:
 		mutex->wwm_state = WW_CTX;
 		mutex->wwm_u.ctx = ctx;
 		goto locked;
 	case WW_OWNED:
 		KASSERTMSG((mutex->wwm_u.owner != curlwp),
 		    "locking %p against myself: %p", mutex, curlwp);
 		ret = ww_mutex_state_wait_sig(mutex, WW_OWNED);
-		if (ret)
+		if (ret) {
-			goto out;
+			KASSERTMSG(ret == -EINTR, "ret=%d", ret);
 			goto out_unlock;
+		}
 		goto retry;
 	case WW_CTX:
 		break;
 	case WW_WANTOWN:
 		ret = ww_mutex_state_wait_sig(mutex, WW_WANTOWN);
-		if (ret)
+		if (ret) {
-			goto out;
+			KASSERTMSG(ret == -EINTR, "ret=%d", ret);
 			goto out_unlock;
+		}
 		goto retry;
 	default:
 		panic("wait/wound mutex %p in bad state: %d",
 		    mutex, (int)mutex->wwm_state);
+	}
 	KASSERT(mutex->wwm_state == WW_CTX);
 	KASSERT(mutex->wwm_u.ctx != NULL);
 	KASSERTMSG((mutex->wwm_u.ctx->wwx_owner != curlwp),
 	    "locking %p against myself: %p", mutex, curlwp);
 	/*
 	 * Owned by another party, of any priority.  Ask that party to
 	 * wake us when it's done.
 	 */
 	ret = ww_mutex_lock_wait_sig(mutex, ctx);
-	if (ret)
+	if (ret) {
-		goto out;
+		KASSERTMSG(ret == -EINTR, "ret=%d", ret);
 		goto out_unlock;
+	}
 locked:	KASSERT((mutex->wwm_state == WW_CTX) ||
 	    (mutex->wwm_state == WW_WANTOWN));
 	KASSERT(mutex->wwm_u.ctx == ctx);
 	WW_LOCKED(mutex);
 	ctx->wwx_acquired++;
 	ret = 0;
-out:	mutex_exit(&mutex->wwm_lock);
+out_unlock:
 	mutex_exit(&mutex->wwm_lock);
 out:	KASSERTMSG((ret == 0 || ret == -EINTR), "ret=%d", ret);
 	return ret;
+}
 /*
  * ww_mutex_trylock(mutex)
+ *
  *	Tro to acquire mutex and return 1, but if it can't be done
  *	immediately, return 0.
  */
 int
 ww_mutex_trylock(struct ww_mutex *mutex)
+{
 	int ret;
 	mutex_enter(&mutex->wwm_lock);
 	if (mutex->wwm_state == WW_UNLOCKED) {
 		mutex->wwm_state = WW_OWNED;
 		mutex->wwm_u.owner = curlwp;
 		WW_WANTLOCK(mutex);
 		WW_LOCKED(mutex);
 		ret = 1;
 	} else {
 @@ -766,43 +966,59 @@ ww_mutex_trylock(struct ww_mutex *mutex)
 		KASSERTMSG(((mutex->wwm_state != WW_CTX) ||
 		    (mutex->wwm_u.ctx->wwx_owner != curlwp)),
 		    "locking %p against myself: %p", mutex, curlwp);
 		KASSERTMSG(((mutex->wwm_state != WW_WANTOWN) ||
 		    (mutex->wwm_u.ctx->wwx_owner != curlwp)),
 		    "locking %p against myself: %p", mutex, curlwp);
 		ret = 0;
+	}
 	mutex_exit(&mutex->wwm_lock);
 	return ret;
+}
 /*
  * ww_mutex_unlock_release(mutex)
+ *
  *	Decrement the number of mutexes acquired in the current locking
  *	context of mutex, which must be held by the caller and in
  *	WW_CTX or WW_WANTOWN state, and clear the mutex's reference.
  *	Caller must hold the internal lock of mutex, and is responsible
  *	for notifying waiters.
+ *
  *	Internal subroutine.
  */
 static void
 ww_mutex_unlock_release(struct ww_mutex *mutex)
+{
 	KASSERT(mutex_owned(&mutex->wwm_lock));
 	KASSERT((mutex->wwm_state == WW_CTX) ||
 	    (mutex->wwm_state == WW_WANTOWN));
 	KASSERT(mutex->wwm_u.ctx != NULL);
 	KASSERTMSG((mutex->wwm_u.ctx->wwx_owner == curlwp),
 	    "ww_mutex %p ctx %p held by %p, not by self (%p)",
 	    mutex, mutex->wwm_u.ctx, mutex->wwm_u.ctx->wwx_owner,
 	    curlwp);
 	KASSERT(mutex->wwm_u.ctx->wwx_acquired != ~0U);
 	mutex->wwm_u.ctx->wwx_acquired--;
 	mutex->wwm_u.ctx = NULL;
+}
 /*
  * ww_mutex_unlock(mutex)
+ *
  *	Release mutex and wake the next caller waiting, if any.
  */
 void
 ww_mutex_unlock(struct ww_mutex *mutex)
+{
 	struct ww_acquire_ctx *ctx;
 	mutex_enter(&mutex->wwm_lock);
 	KASSERT(mutex->wwm_state != WW_UNLOCKED);
 	switch (mutex->wwm_state) {
 	case WW_UNLOCKED:
 		panic("unlocking unlocked wait/wound mutex: %p", mutex);
 	case WW_OWNED:
 		/* Let the context lockers fight over it.  */
 		mutex->wwm_u.owner = NULL;