 @@ -1,14 +1,14 @@
-/*	$NetBSD: subr_lockdebug.c,v 1.59 2018/02/14 03:56:26 ozaki-r Exp $	*/
+/*	$NetBSD: subr_lockdebug.c,v 1.60 2018/02/20 03:34:52 ozaki-r Exp $	*/
 /*-
  * Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Andrew Doran.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -24,27 +24,27 @@
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 /*
  * Basic lock debugging code shared among lock primitives.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_lockdebug.c,v 1.59 2018/02/14 03:56:26 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_lockdebug.c,v 1.60 2018/02/20 03:34:52 ozaki-r Exp $");
 #ifdef _KERNEL_OPT
 #include "opt_ddb.h"
 #endif
 #include <sys/param.h>
 #include <sys/proc.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/lockdebug.h>
 #include <sys/sleepq.h>
 #include <sys/cpu.h>
 @@ -186,27 +186,27 @@ lockdebug_unlock_cpus(void)
 /*
  * lockdebug_lookup:
+ *
  *	Find a lockdebug structure by a pointer to a lock and return it locked.
  */
 static inline lockdebug_t *
 lockdebug_lookup(const char *func, size_t line, const volatile void *lock,
     uintptr_t where)
+{
 	lockdebug_t *ld;
 	ld = lockdebug_lookup1(lock);
-	if (ld == NULL) {
+	if (__predict_false(ld == NULL)) {
 		panic("%s,%zu: uninitialized lock (lock=%p, from=%08"
 		    PRIxPTR ")", func, line, lock, where);
+	}
 	return ld;
+}
 /*
  * lockdebug_init:
+ *
  *	Initialize the lockdebug system.  Allocate an initial pool of
  *	lockdebug structures before the VM system is up and running.
  */
 static void
 @@ -236,34 +236,34 @@ lockdebug_init(void)
  * lockdebug_alloc:
+ *
  *	A lock is being initialized, so allocate an associated debug
  *	structure.
  */
 bool
 lockdebug_alloc(const char *func, size_t line, volatile void *lock,
     lockops_t *lo, uintptr_t initaddr)
+{
 	struct cpu_info *ci;
 	lockdebug_t *ld;
 	int s;
-	if (lo == NULL || panicstr != NULL || ld_panic)
+	if (__predict_false(lo == NULL || panicstr != NULL || ld_panic))
 		return false;
-	if (ld_freeptr == 0)
+	if (__predict_false(ld_freeptr == 0))
 		lockdebug_init();
 	s = splhigh();
 	__cpu_simple_lock(&ld_mod_lk);
-	if ((ld = lockdebug_lookup1(lock)) != NULL) {
+	if (__predict_false((ld = lockdebug_lookup1(lock)) != NULL)) {
 		__cpu_simple_unlock(&ld_mod_lk);
 		lockdebug_abort1(func, line, ld, s, "already initialized",
 		    true);
 		return false;
+	}
 	/*
 	 * Pinch a new debug structure.  We may recurse because we call
 	 * kmem_alloc(), which may need to initialize new locks somewhere
 	 * down the path.  If not recursing, we try to maintain at least
 	 * LD_SLOP structures free, which should hopefully be enough to
 	 * satisfy kmem_alloc().  If we can't provide a structure, not to
 	 * worry: we'll just mark the lock as not having an ID.
 @@ -271,36 +271,36 @@ lockdebug_alloc(const char *func, size_t
 	ci = curcpu();
 	ci->ci_lkdebug_recurse++;
 	if (TAILQ_EMPTY(&ld_free)) {
 		if (ci->ci_lkdebug_recurse > 1 || ld_nomore) {
 			ci->ci_lkdebug_recurse--;
 			__cpu_simple_unlock(&ld_mod_lk);
 			splx(s);
 			return false;
+		}
 		s = lockdebug_more(s);
 	} else if (ci->ci_lkdebug_recurse == 1 && ld_nfree < LD_SLOP) {
 		s = lockdebug_more(s);
+	}
-	if ((ld = TAILQ_FIRST(&ld_free)) == NULL) {
+	if (__predict_false((ld = TAILQ_FIRST(&ld_free)) == NULL)) {
 		__cpu_simple_unlock(&ld_mod_lk);
 		splx(s);
 		return false;
+	}
 	TAILQ_REMOVE(&ld_free, ld, ld_chain);
 	ld_nfree--;
 	ci->ci_lkdebug_recurse--;
-	if (ld->ld_lock != NULL) {
+	if (__predict_false(ld->ld_lock != NULL)) {
 		panic("%s,%zu: corrupt table ld %p", func, line, ld);
+	}
 	/* Initialise the structure. */
 	ld->ld_lock = lock;
 	ld->ld_lockops = lo;
 	ld->ld_locked = 0;
 	ld->ld_unlocked = 0;
 	ld->ld_lwp = NULL;
 	ld->ld_initaddr = initaddr;
 	ld->ld_flags = (lo->lo_type == LOCKOPS_SLEEP ? LD_SLEEPER : 0);
 	lockdebug_lock_cpus();
 	(void)rb_tree_insert_node(&ld_rb_tree, __UNVOLATILE(ld));
 @@ -312,40 +312,41 @@ lockdebug_alloc(const char *func, size_t
+}
 /*
  * lockdebug_free:
+ *
  *	A lock is being destroyed, so release debugging resources.
  */
 void
 lockdebug_free(const char *func, size_t line, volatile void *lock)
+{
 	lockdebug_t *ld;
 	int s;
-	if (panicstr != NULL || ld_panic)
+	if (__predict_false(panicstr != NULL || ld_panic))
 		return;
 	s = splhigh();
 	__cpu_simple_lock(&ld_mod_lk);
 	ld = lockdebug_lookup(func, line, lock,
 	    (uintptr_t) __builtin_return_address(0));
-	if (ld == NULL) {
+	if (__predict_false(ld == NULL)) {
 		__cpu_simple_unlock(&ld_mod_lk);
 		panic("%s,%zu: destroying uninitialized object %p"
 		    "(ld_lock=%p)", func, line, lock, ld->ld_lock);
 		return;
+	}
-	if ((ld->ld_flags & LD_LOCKED) != 0 || ld->ld_shares != 0) {
+	if (__predict_false((ld->ld_flags & LD_LOCKED) != 0 ||
 	    ld->ld_shares != 0)) {
 		__cpu_simple_unlock(&ld_mod_lk);
 		lockdebug_abort1(func, line, ld, s, "is locked or in use",
 		    true);
 		return;
+	}
 	lockdebug_lock_cpus();
 	rb_tree_remove_node(&ld_rb_tree, __UNVOLATILE(ld));
 	lockdebug_unlock_cpus();
 	ld->ld_lock = NULL;
 	TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);
 	ld_nfree++;
 	__cpu_simple_unlock(&ld->ld_spinlock);
 	__cpu_simple_unlock(&ld_mod_lk);
 @@ -420,102 +421,102 @@ lockdebug_more(int s)
  */
 void
 lockdebug_wantlock(const char *func, size_t line,
     const volatile void *lock, uintptr_t where, int shared)
+{
 	struct lwp *l = curlwp;
 	lockdebug_t *ld;
 	bool recurse;
 	int s;
 	(void)shared;
 	recurse = false;
-	if (panicstr != NULL || ld_panic)
+	if (__predict_false(panicstr != NULL || ld_panic))
 		return;
 	s = splhigh();
 	if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) {
 		splx(s);
 		return;
+	}
 	if ((ld->ld_flags & LD_LOCKED) != 0 || ld->ld_shares != 0) {
 		if ((ld->ld_flags & LD_SLEEPER) != 0) {
 			if (ld->ld_lwp == l)
 				recurse = true;
 		} else if (ld->ld_cpu == (uint16_t)cpu_index(curcpu()))
 			recurse = true;
+	}
 	if (cpu_intr_p()) {
-		if ((ld->ld_flags & LD_SLEEPER) != 0) {
+		if (__predict_false((ld->ld_flags & LD_SLEEPER) != 0)) {
 			lockdebug_abort1(func, line, ld, s,
 			    "acquiring sleep lock from interrupt context",
 			    true);
 			return;
+		}
+	}
 	if (shared > 0)
 		ld->ld_shwant++;
 	else if (shared == 0)
 		ld->ld_exwant++;
-	if (recurse) {
+	if (__predict_false(recurse)) {
 		lockdebug_abort1(func, line, ld, s, "locking against myself",
 		    true);
 		return;
+	}
 	__cpu_simple_unlock(&ld->ld_spinlock);
 	splx(s);
+}
 /*
  * lockdebug_locked:
+ *
  *	Process a lock acquire operation.
  */
 void
 lockdebug_locked(const char *func, size_t line,
     volatile void *lock, void *cvlock, uintptr_t where, int shared)
+{
 	struct lwp *l = curlwp;
 	lockdebug_t *ld;
 	int s;
-	if (panicstr != NULL || ld_panic)
+	if (__predict_false(panicstr != NULL || ld_panic))
 		return;
 	s = splhigh();
 	if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) {
 		splx(s);
 		return;
+	}
 	if (cvlock) {
 		KASSERT(ld->ld_lockops->lo_type == LOCKOPS_CV);
 		if (lock == (void *)&lbolt) {
 			/* nothing */
 		} else if (ld->ld_shares++ == 0) {
 			ld->ld_locked = (uintptr_t)cvlock;
-		} else if (cvlock != (void *)ld->ld_locked) {
+		} else if (__predict_false(cvlock != (void *)ld->ld_locked)) {
 			lockdebug_abort1(func, line, ld, s,
 			    "multiple locks used with condition variable",
 			    true);
 			return;
+		}
 	} else if (shared) {
 		l->l_shlocks++;
 		ld->ld_locked = where;
 		ld->ld_shares++;
 		ld->ld_shwant--;
 	} else {
-		if ((ld->ld_flags & LD_LOCKED) != 0) {
+		if (__predict_false((ld->ld_flags & LD_LOCKED) != 0)) {
 			lockdebug_abort1(func, line, ld, s, "already locked",
 			    true);
 			return;
+		}
 		ld->ld_flags |= LD_LOCKED;
 		ld->ld_locked = where;
 		ld->ld_exwant--;
 		if ((ld->ld_flags & LD_SLEEPER) != 0) {
 			TAILQ_INSERT_TAIL(&l->l_ld_locks, ld, ld_chain);
 		} else {
 			TAILQ_INSERT_TAIL(&curcpu()->ci_data.cpu_ld_locks,
 			    ld, ld_chain);
+		}
 @@ -529,74 +530,75 @@ lockdebug_locked(const char *func, size_
 /*
  * lockdebug_unlocked:
+ *
  *	Process a lock release operation.
  */
 void
 lockdebug_unlocked(const char *func, size_t line,
     volatile void *lock, uintptr_t where, int shared)
+{
 	struct lwp *l = curlwp;
 	lockdebug_t *ld;
 	int s;
-	if (panicstr != NULL || ld_panic)
+	if (__predict_false(panicstr != NULL || ld_panic))
 		return;
 	s = splhigh();
 	if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) {
 		splx(s);
 		return;
+	}
 	if (ld->ld_lockops->lo_type == LOCKOPS_CV) {
 		if (lock == (void *)&lbolt) {
 			/* nothing */
 		} else {
 			ld->ld_shares--;
+		}
 	} else if (shared) {
-		if (l->l_shlocks == 0) {
+		if (__predict_false(l->l_shlocks == 0)) {
 			lockdebug_abort1(func, line, ld, s,
 			    "no shared locks held by LWP", true);
 			return;
+		}
-		if (ld->ld_shares == 0) {
+		if (__predict_false(ld->ld_shares == 0)) {
 			lockdebug_abort1(func, line, ld, s,
 			    "no shared holds on this lock", true);
 			return;
+		}
 		l->l_shlocks--;
 		ld->ld_shares--;
 		if (ld->ld_lwp == l) {
 			ld->ld_unlocked = where;
 			ld->ld_lwp = NULL;
+		}
 		if (ld->ld_cpu == (uint16_t)cpu_index(curcpu()))
 			ld->ld_cpu = (uint16_t)-1;
 	} else {
-		if ((ld->ld_flags & LD_LOCKED) == 0) {
+		if (__predict_false((ld->ld_flags & LD_LOCKED) == 0)) {
 			lockdebug_abort1(func, line, ld, s, "not locked", true);
 			return;
+		}
 		if ((ld->ld_flags & LD_SLEEPER) != 0) {
-			if (ld->ld_lwp != curlwp) {
+			if (__predict_false(ld->ld_lwp != curlwp)) {
 				lockdebug_abort1(func, line, ld, s,
 				    "not held by current LWP", true);
 				return;
+			}
 			TAILQ_REMOVE(&l->l_ld_locks, ld, ld_chain);
 		} else {
-			if (ld->ld_cpu != (uint16_t)cpu_index(curcpu())) {
+			uint16_t idx = (uint16_t)cpu_index(curcpu());
 			if (__predict_false(ld->ld_cpu != idx)) {
 				lockdebug_abort1(func, line, ld, s,
 				    "not held by current CPU", true);
 				return;
+			}
 			TAILQ_REMOVE(&curcpu()->ci_data.cpu_ld_locks, ld,
 			    ld_chain);
+		}
 		ld->ld_flags &= ~LD_LOCKED;
 		ld->ld_unlocked = where;
 		ld->ld_lwp = NULL;
+	}
 	__cpu_simple_unlock(&ld->ld_spinlock);
 	splx(s);
 @@ -604,82 +606,84 @@ lockdebug_unlocked(const char *func, siz
 /*
  * lockdebug_wakeup:
+ *
  *	Process a wakeup on a condition variable.
  */
 void
 lockdebug_wakeup(const char *func, size_t line, volatile void *lock,
     uintptr_t where)
+{
 	lockdebug_t *ld;
 	int s;
-	if (panicstr != NULL || ld_panic || lock == (void *)&lbolt)
+	if (__predict_false(panicstr != NULL || ld_panic || lock == (void *)&lbolt))
 		return;
 	s = splhigh();
 	/* Find the CV... */
 	if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) {
 		splx(s);
 		return;
+	}
 	/*
 	 * If it has any waiters, ensure that they are using the
 	 * same interlock.
 	 */
-	if (ld->ld_shares != 0 && !mutex_owned((kmutex_t *)ld->ld_locked)) {
+	if (__predict_false(ld->ld_shares != 0 &&
 	    !mutex_owned((kmutex_t *)ld->ld_locked))) {
 		lockdebug_abort1(func, line, ld, s, "interlocking mutex not "
 		    "held during wakeup", true);
 		return;
+	}
 	__cpu_simple_unlock(&ld->ld_spinlock);
 	splx(s);
+}
 /*
  * lockdebug_barrier:
+ *
  *	Panic if we hold more than one specified spin lock, and optionally,
  *	if we hold sleep locks.
  */
 void
 lockdebug_barrier(const char *func, size_t line, volatile void *spinlock,
     int slplocks)
+{
 	struct lwp *l = curlwp;
 	lockdebug_t *ld;
 	int s;
-	if (panicstr != NULL || ld_panic)
+	if (__predict_false(panicstr != NULL || ld_panic))
 		return;
 	s = splhigh();
 	if ((l->l_pflag & LP_INTR) == 0) {
 		TAILQ_FOREACH(ld, &curcpu()->ci_data.cpu_ld_locks, ld_chain) {
 			if (ld->ld_lock == spinlock) {
 				continue;
+			}
 			__cpu_simple_lock(&ld->ld_spinlock);
 			lockdebug_abort1(func, line, ld, s,
 			    "spin lock held", true);
 			return;
+		}
+	}
 	if (slplocks) {
 		splx(s);
 		return;
+	}
-	if ((ld = TAILQ_FIRST(&l->l_ld_locks)) != NULL) {
+	ld = TAILQ_FIRST(&l->l_ld_locks);
 	if (__predict_false(ld != NULL)) {
 		__cpu_simple_lock(&ld->ld_spinlock);
 		lockdebug_abort1(func, line, ld, s, "sleep lock held", true);
 		return;
+	}
 	splx(s);
 	if (l->l_shlocks != 0) {
 		TAILQ_FOREACH(ld, &ld_all, ld_achain) {
 			if (ld->ld_lockops->lo_type == LOCKOPS_CV)
 				continue;
 			if (ld->ld_lwp == l)
 				lockdebug_dump(ld, printf);
+		}
 		panic("%s,%zu: holding %d shared locks", func, line,
 @@ -690,44 +694,44 @@ lockdebug_barrier(const char *func, size
 /*
  * lockdebug_mem_check:
+ *
  *	Check for in-use locks within a memory region that is
  *	being freed.
  */
 void
 lockdebug_mem_check(const char *func, size_t line, void *base, size_t sz)
+{
 	lockdebug_t *ld;
 	struct cpu_info *ci;
 	int s;
-	if (panicstr != NULL || ld_panic)
+	if (__predict_false(panicstr != NULL || ld_panic))
 		return;
 	s = splhigh();
 	ci = curcpu();
 	__cpu_simple_lock(&ci->ci_data.cpu_ld_lock);
 	ld = (lockdebug_t *)rb_tree_find_node_geq(&ld_rb_tree, base);
 	if (ld != NULL) {
 		const uintptr_t lock = (uintptr_t)ld->ld_lock;
-		if ((uintptr_t)base > lock)
+		if (__predict_false((uintptr_t)base > lock))
 			panic("%s,%zu: corrupt tree ld=%p, base=%p, sz=%zu",
 			    func, line, ld, base, sz);
 		if (lock >= (uintptr_t)base + sz)
 			ld = NULL;
+	}
 	__cpu_simple_unlock(&ci->ci_data.cpu_ld_lock);
-	if (ld != NULL) {
+	if (__predict_false(ld != NULL)) {
 		__cpu_simple_lock(&ld->ld_spinlock);
 		lockdebug_abort1(func, line, ld, s,
 		    "allocation contains active lock", !cold);
 		return;
+	}
 	splx(s);
+}
 /*
  * lockdebug_dump:
+ *
  *	Dump information about a lock on panic, or for DDB.
  */