 @@ -1,14 +1,14 @@
-/*      $NetBSD: scheduler.c,v 1.28 2012/06/22 12:45:43 rmind Exp $	*/
+/*      $NetBSD: scheduler.c,v 1.29 2012/09/15 17:15:01 pooka Exp $	*/
 /*
  * Copyright (c) 2010, 2011 Antti Kantee.  All Rights Reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
 @@ -16,27 +16,27 @@
  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: scheduler.c,v 1.28 2012/06/22 12:45:43 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: scheduler.c,v 1.29 2012/09/15 17:15:01 pooka Exp $");
 #include <sys/param.h>
 #include <sys/atomic.h>
 #include <sys/cpu.h>
 #include <sys/kmem.h>
 #include <sys/mutex.h>
 #include <sys/namei.h>
 #include <sys/queue.h>
 #include <sys/select.h>
 #include <sys/systm.h>
 #include <rump/rumpuser.h>
 @@ -290,44 +290,44 @@ rump_schedule_cpu_interlock(struct lwp *
 	/*
 	 * First, try fastpath: if we were the previous user of the
 	 * CPU, everything is in order cachewise and we can just
 	 * proceed to use it.
+	 *
 	 * If we are a different thread (i.e. CAS fails), we must go
 	 * through a memory barrier to ensure we get a truthful
 	 * view of the world.
 	 */
 	KASSERT(l->l_target_cpu != NULL);
 	rcpu = &rcpu_storage[l->l_target_cpu-&rump_cpus[0]];
 	if (atomic_cas_ptr(&rcpu->rcpu_prevlwp, l, RCPULWP_BUSY) == l) {
-		if (__predict_true(interlock == rcpu->rcpu_mtx))
+		if (interlock == rcpu->rcpu_mtx)
 			rumpuser_mutex_exit(rcpu->rcpu_mtx);
 		SCHED_FASTPATH(rcpu);
 		/* jones, you're the man */
 		goto fastlane;
+	}
 	/*
 	 * Else, it's the slowpath for us.  First, determine if we
 	 * can migrate.
 	 */
 	if (ncpu == 1)
 		domigrate = false;
 	else
 		domigrate = true;
 	/* Take lock.  This acts as a load barrier too. */
-	if (__predict_true(interlock != rcpu->rcpu_mtx))
+	if (interlock != rcpu->rcpu_mtx)
 		rumpuser_mutex_enter_nowrap(rcpu->rcpu_mtx);
 	for (;;) {
 		SCHED_SLOWPATH(rcpu);
 		old = atomic_swap_ptr(&rcpu->rcpu_prevlwp, RCPULWP_WANTED);
 		/* CPU is free? */
 		if (old != RCPULWP_BUSY && old != RCPULWP_WANTED) {
 			if (atomic_cas_ptr(&rcpu->rcpu_prevlwp,
 			    RCPULWP_WANTED, RCPULWP_BUSY) == RCPULWP_WANTED) {
 				break;
+			}
+		}
 @@ -432,53 +432,59 @@ rump_unschedule_cpu1(struct lwp *l, void
 	void *old;
 	ci = l->l_cpu;
 	ci->ci_curlwp = NULL;
 	rcpu = &rcpu_storage[ci-&rump_cpus[0]];
 	KASSERT(rcpu->rcpu_ci == ci);
 	/*
 	 * Make sure all stores are seen before the CPU release.  This
 	 * is relevant only in the non-fastpath scheduling case, but
 	 * we don't know here if that's going to happen, so need to
 	 * expect the worst.
+	 *
 	 * If the scheduler interlock was requested by the caller, we
 	 * need to obtain it before we release the CPU.  Otherwise, we risk a
 	 * race condition where another thread is scheduled onto the
 	 * rump kernel CPU before our current thread can
 	 * grab the interlock.
 	 */
-	membar_exit();
+	if (interlock == rcpu->rcpu_mtx)
 		rumpuser_mutex_enter_nowrap(rcpu->rcpu_mtx);
 	else
 		membar_exit();
 	/* Release the CPU. */
 	old = atomic_swap_ptr(&rcpu->rcpu_prevlwp, l);
 	/* No waiters?  No problems.  We're outta here. */
 	if (old == RCPULWP_BUSY) {
 		/* Was the scheduler interlock requested? */
 		if (__predict_false(interlock == rcpu->rcpu_mtx))
 			rumpuser_mutex_enter_nowrap(rcpu->rcpu_mtx);
 		return;
+	}
 	KASSERT(old == RCPULWP_WANTED);
 	/*
 	 * Ok, things weren't so snappy.
+	 *
 	 * Snailpath: take lock and signal anyone waiting for this CPU.
 	 */
-	rumpuser_mutex_enter_nowrap(rcpu->rcpu_mtx);
+	if (interlock != rcpu->rcpu_mtx)
 		rumpuser_mutex_enter_nowrap(rcpu->rcpu_mtx);
 	if (rcpu->rcpu_wanted)
 		rumpuser_cv_broadcast(rcpu->rcpu_cv);
 	if (interlock != rcpu->rcpu_mtx)
 	if (__predict_true(interlock != rcpu->rcpu_mtx))
 		rumpuser_mutex_exit(rcpu->rcpu_mtx);
+}
 /* Give up and retake CPU (perhaps a different one) */
 void
 yield()
+{
 	struct lwp *l = curlwp;
 	int nlocks;
 	KERNEL_UNLOCK_ALL(l, &nlocks);
 	rump_unschedule_cpu(l);
 	rump_schedule_cpu(l);