 @@ -1,14 +1,14 @@
-/*	$NetBSD: kern_runq.c,v 1.22 2008/10/07 09:48:27 rmind Exp $	*/
+/*	$NetBSD: kern_runq.c,v 1.22.4.1 2009/01/22 20:06:47 snj Exp $	*/
 /*
  * Copyright (c) 2007, 2008 Mindaugas Rasiukevicius <rmind at NetBSD org>
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -17,27 +17,27 @@
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_runq.c,v 1.22 2008/10/07 09:48:27 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_runq.c,v 1.22.4.1 2009/01/22 20:06:47 snj Exp $");
 #include <sys/param.h>
 #include <sys/kernel.h>
 #include <sys/bitops.h>
 #include <sys/cpu.h>
 #include <sys/idle.h>
 #include <sys/intr.h>
 #include <sys/kmem.h>
 #include <sys/lwp.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
 #include <sys/sched.h>
 #include <sys/syscallargs.h>
 @@ -64,45 +64,45 @@ __KERNEL_RCSID(0, "$NetBSD: kern_runq.c,
 #define	BITMAP_MASK	(BITMAP_BITS - 1)
 /*
  * Structures, runqueue.
  */
 const int	schedppq = 1;
 typedef struct {
 	TAILQ_HEAD(, lwp) q_head;
 } queue_t;
 typedef struct {
-	/* Lock and bitmap */
+	/* Bitmap */
 	uint32_t	r_bitmap[PRI_COUNT >> BITMAP_SHIFT];
 	/* Counters */
 	u_int		r_count;	/* Count of the threads */
 	u_int		r_avgcount;	/* Average count of threads */
 	u_int		r_mcount;	/* Count of migratable threads */
 	/* Runqueues */
 	queue_t		r_rt_queue[PRI_RT_COUNT];
 	queue_t		r_ts_queue[PRI_TS_COUNT];
 	/* Event counters */
 	struct evcnt	r_ev_pull;
 	struct evcnt	r_ev_push;
 	struct evcnt	r_ev_stay;
 	struct evcnt	r_ev_localize;
 } runqueue_t;
 static void *	sched_getrq(runqueue_t *, const pri_t);
 #ifdef MULTIPROCESSOR
-static lwp_t	*sched_catchlwp(struct cpu_info *);
+static lwp_t *	sched_catchlwp(struct cpu_info *);
 static void	sched_balance(void *);
 #endif
 /*
  * Preemption control.
  */
 int		sched_upreempt_pri = PRI_KERNEL;
 #if defined(__HAVE_PREEMPTION)
 int		sched_kpreempt_pri = PRI_USER_RT;
 #else
 int		sched_kpreempt_pri = 1000;
 #endif
 @@ -323,27 +323,27 @@ sched_dequeue(struct lwp *l)
+}
 /*
  * Migration and balancing.
  */
 #ifdef MULTIPROCESSOR
 /* Estimate if LWP is cache-hot */
 static inline bool
 lwp_cache_hot(const struct lwp *l)
+{
-	if (l->l_slptime || l->l_rticks == 0)
+	if (__predict_false(l->l_slptime || l->l_rticks == 0))
 		return false;
 	return (hardclock_ticks - l->l_rticks <= cacheht_time);
+}
 /* Check if LWP can migrate to the chosen CPU */
 static inline bool
 sched_migratable(const struct lwp *l, struct cpu_info *ci)
+{
 	const struct schedstate_percpu *spc = &ci->ci_schedstate;
 	KASSERT(lwp_locked(__UNCONST(l), NULL));
 	/* CPU is offline */
 @@ -362,51 +362,57 @@ sched_migratable(const struct lwp *l, st
  * Estimate the migration of LWP to the other CPU.
  * Take and return the CPU, if migration is needed.
  */
 struct cpu_info *
 sched_takecpu(struct lwp *l)
+{
 	struct cpu_info *ci, *tci, *first, *next;
 	struct schedstate_percpu *spc;
 	runqueue_t *ci_rq, *ici_rq;
 	pri_t eprio, lpri, pri;
 	KASSERT(lwp_locked(l, NULL));
 	/* If thread is strictly bound, do not estimate other CPUs */
 	ci = l->l_cpu;
 	if (l->l_pflag & LP_BOUND)
 		return ci;
 	spc = &ci->ci_schedstate;
 	ci_rq = spc->spc_sched_info;
-	/*
+	/* Make sure that thread is in appropriate processor-set */
-	 * If thread is strictly bound, do not estimate other CPUs.
+	if (__predict_true(spc->spc_psid == l->l_psid)) {
-	 * If CPU of this thread is idling - run there.
+		/* If CPU of this thread is idling - run there */
-	 */
+		if (ci_rq->r_count == 0) {
-	if ((l->l_pflag & LP_BOUND) != 0 || ci_rq->r_count == 0) {
+			ci_rq->r_ev_stay.ev_count++;
-		ci_rq->r_ev_stay.ev_count++;
+			return ci;
 		return ci;
 		/* Stay if thread is cache-hot */
 		eprio = lwp_eprio(l);
-	/* Stay if thread is cache-hot. */
+		if (__predict_true(l->l_stat != LSIDL) &&
-	eprio = lwp_eprio(l);
+		    lwp_cache_hot(l) && eprio >= spc->spc_curpriority) {
-	if (__predict_true(l->l_stat != LSIDL) &&
+			ci_rq->r_ev_stay.ev_count++;
-	    lwp_cache_hot(l) && eprio >= spc->spc_curpriority) {
+			return ci;
 		ci_rq->r_ev_stay.ev_count++;
-		return ci;
+	} else {
 		eprio = lwp_eprio(l);
+	}
 	/* Run on current CPU if priority of thread is higher */
 	ci = curcpu();
 	spc = &ci->ci_schedstate;
 	if (eprio > spc->spc_curpriority && sched_migratable(l, ci)) {
 		ci_rq = spc->spc_sched_info;
 		ci_rq->r_ev_localize.ev_count++;
 		return ci;
+	}
 	/*
 	 * Look for the CPU with the lowest priority thread.  In case of
 	 * equal priority, choose the CPU with the fewest of threads.
 	 */
 	first = l->l_cpu;
 	ci = first;
 	tci = first;
 	lpri = PRI_COUNT;
 	do {
 @@ -431,32 +437,35 @@ sched_takecpu(struct lwp *l)
 	ci_rq = tci->ci_schedstate.spc_sched_info;
 	ci_rq->r_ev_push.ev_count++;
 	return tci;
+}
 /*
  * Tries to catch an LWP from the runqueue of other CPU.
  */
 static struct lwp *
 sched_catchlwp(struct cpu_info *ci)
+{
 	struct cpu_info *curci = curcpu();
-	struct schedstate_percpu *spc;
+	struct schedstate_percpu *spc, *curspc;
 	TAILQ_HEAD(, lwp) *q_head;
 	runqueue_t *ci_rq;
 	struct lwp *l;
 	curspc = &curci->ci_schedstate;
 	spc = &ci->ci_schedstate;
 	KASSERT(curspc->spc_psid == spc->spc_psid);
 	ci_rq = spc->spc_sched_info;
 	if (ci_rq->r_mcount < min_catch) {
 		spc_unlock(ci);
 		return NULL;
+	}
 	/* Take the highest priority thread */
 	q_head = sched_getrq(ci_rq, spc->spc_maxpriority);
 	l = TAILQ_FIRST(q_head);
 	for (;;) {
 		/* Check the first and next result from the queue */
 		if (l == NULL)
 @@ -465,27 +474,27 @@ sched_catchlwp(struct cpu_info *ci)
 		KASSERT(l->l_flag & LW_INMEM);
 		/* Look for threads, whose are allowed to migrate */
 		if ((l->l_pflag & LP_BOUND) || lwp_cache_hot(l) ||
 		    !sched_migratable(l, curci)) {
 			l = TAILQ_NEXT(l, l_runq);
 			continue;
+		}
 		/* Grab the thread, and move to the local run queue */
 		sched_dequeue(l);
 		l->l_cpu = curci;
 		ci_rq->r_ev_pull.ev_count++;
-		lwp_unlock_to(l, curci->ci_schedstate.spc_mutex);
+		lwp_unlock_to(l, curspc->spc_mutex);
 		sched_enqueue(l, false);
 		return l;
+	}
 	spc_unlock(ci);
 	return l;
+}
 /*
  * Periodical calculations for balancing.
  */
 static void
 sched_balance(void *nocallout)
 @@ -598,27 +607,28 @@ sched_idle(void)
+	}
 	spc_unlock(ci);
 no_migration:
 	ci_rq = spc->spc_sched_info;
 	if ((spc->spc_flags & SPCF_OFFLINE) != 0 || ci_rq->r_count != 0) {
 		return;
+	}
 	/* Reset the counter, and call the balancer */
 	ci_rq->r_avgcount = 0;
 	sched_balance(ci);
 	tci = worker_ci;
-	if (ci == tci)
+	tspc = &tci->ci_schedstate;
 	if (ci == tci || spc->spc_psid != tspc->spc_psid)
 		return;
 	spc_dlock(ci, tci);
 	(void)sched_catchlwp(tci);
 	spc_unlock(ci);
+}
 #else
 struct cpu_info *
 sched_takecpu(struct lwp *l)
+{
 	return l->l_cpu;
 @@ -695,37 +705,40 @@ sched_nextlwp(void)
 	TAILQ_HEAD(, lwp) *q_head;
 	runqueue_t *ci_rq;
 	struct lwp *l;
 	/* Return to idle LWP if there is a migrating thread */
 	spc = &ci->ci_schedstate;
 	if (__predict_false(spc->spc_migrating != NULL))
 		return NULL;
 	ci_rq = spc->spc_sched_info;
 #ifdef MULTIPROCESSOR
 	/* If runqueue is empty, try to catch some thread from other CPU */
 	if (__predict_false(ci_rq->r_count == 0)) {
 		struct schedstate_percpu *cspc;
 		struct cpu_info *cci;
 		/* Offline CPUs should not perform this, however */
 		if (__predict_false(spc->spc_flags & SPCF_OFFLINE))
 			return NULL;
 		/* Reset the counter, and call the balancer */
 		ci_rq->r_avgcount = 0;
 		sched_balance(ci);
 		cci = worker_ci;
-		if (ci == cci || !mutex_tryenter(cci->ci_schedstate.spc_mutex))
+		cspc = &cci->ci_schedstate;
 		if (ci == cci || spc->spc_psid != cspc->spc_psid ||
 		    !mutex_tryenter(cci->ci_schedstate.spc_mutex))
 			return NULL;
 		return sched_catchlwp(cci);
+	}
 #else
 	if (__predict_false(ci_rq->r_count == 0))
 		return NULL;
 #endif
 	/* Take the highest priority thread */
 	KASSERT(ci_rq->r_bitmap[spc->spc_maxpriority >> BITMAP_SHIFT]);
 	q_head = sched_getrq(ci_rq, spc->spc_maxpriority);
 	l = TAILQ_FIRST(q_head);
 	KASSERT(l != NULL);