 @@ -1,14 +1,14 @@
-/*	$NetBSD: cpu.c,v 1.214 2009/05/18 01:36:11 mrg Exp $ */
+/*	$NetBSD: cpu.c,v 1.215 2009/05/27 02:19:49 mrg Exp $ */
 /*
  * Copyright (c) 1996
  *	The President and Fellows of Harvard College. All rights reserved.
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.  All rights reserved.
+ *
  * This software was developed by the Computer Systems Engineering group
  * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
  * contributed to Berkeley.
+ *
  * All advertising materials mentioning features or use of this software
  * must display the following acknowledgement:
 @@ -42,27 +42,27 @@
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  *	@(#)cpu.c	8.5 (Berkeley) 11/23/93
+ *
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.214 2009/05/18 01:36:11 mrg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.215 2009/05/27 02:19:49 mrg Exp $");
 #include "opt_multiprocessor.h"
 #include "opt_lockdebug.h"
 #include "opt_ddb.h"
 #include "opt_sparc_arch.h"
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/device.h>
 #include <sys/malloc.h>
 #include <sys/simplelock.h>
 #include <sys/kernel.h>
 @@ -92,30 +92,27 @@ __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.21
 struct cpu_softc {
 	struct device	sc_dev;		/* generic device info */
 	struct cpu_info	*sc_cpuinfo;
 };
 /* The following are used externally (sysctl_hw). */
 char	machine[] = MACHINE;		/* from <machine/param.h> */
 char	machine_arch[] = MACHINE_ARCH;	/* from <machine/param.h> */
 int	cpu_arch;			/* sparc architecture version */
 char	cpu_model[100];			/* machine model (primary CPU) */
 extern char machine_model[];
 int	sparc_ncpus;			/* # of CPUs detected by PROM */
-#ifdef MULTIPROCESSOR
+struct cpu_info *cpus[_MAXNCPU];	/* we only support 4 CPUs. */
 struct cpu_info *cpus[4];		/* we only support 4 CPUs. */
 u_int	cpu_ready_mask;			/* the set of CPUs marked as READY */
 #endif
 /* The CPU configuration driver. */
 static void cpu_mainbus_attach(struct device *, struct device *, void *);
 int  cpu_mainbus_match(struct device *, struct cfdata *, void *);
 CFATTACH_DECL(cpu_mainbus, sizeof(struct cpu_softc),
     cpu_mainbus_match, cpu_mainbus_attach, NULL, NULL);
 #if defined(SUN4D)
 static int cpu_cpuunit_match(struct device *, struct cfdata *, void *);
 static void cpu_cpuunit_attach(struct device *, struct device *, void *);
 CFATTACH_DECL(cpu_cpuunit, sizeof(struct cpu_softc),
 @@ -126,89 +123,67 @@ static void cpu_attach(struct cpu_softc
 static const char *fsrtoname(int, int, int);
 void cache_print(struct cpu_softc *);
 void cpu_setup(void);
 void fpu_init(struct cpu_info *);
 #define	IU_IMPL(psr)	((u_int)(psr) >> 28)
 #define	IU_VERS(psr)	(((psr) >> 24) & 0xf)
 #define SRMMU_IMPL(mmusr)	((u_int)(mmusr) >> 28)
 #define SRMMU_VERS(mmusr)	(((mmusr) >> 24) & 0xf)
 int bootmid;		/* Module ID of boot CPU */
 #if defined(MULTIPROCESSOR)
 void cpu_spinup(struct cpu_info *);
 static void init_cpuinfo(struct cpu_info *, int);
 int go_smp_cpus = 0;	/* non-primary CPUs wait for this to go */
 /* lock this to send IPI's */
 struct simplelock xpmsg_lock = SIMPLELOCK_INITIALIZER;
 static void
 init_cpuinfo(struct cpu_info *cpi, int node)
 	vaddr_t intstack, va;
 	/*
 	 * Finish initialising this cpu_info.
 	 */
 	getcpuinfo(cpi, node);
 	/*
 	 * Arrange interrupt stack.  This cpu will also abuse the bottom
 	 * half of the interrupt stack before it gets to run its idle LWP.
 	 */
 	intstack = uvm_km_alloc(kernel_map, INT_STACK_SIZE, 0, UVM_KMF_WIRED);
 	if (intstack == 0)
 		panic("%s: no uspace/intstack", __func__);
 	cpi->eintstack = (void*)(intstack + INT_STACK_SIZE);
 	/* Allocate virtual space for pmap page_copy/page_zero */
 	va = uvm_km_alloc(kernel_map, 2*PAGE_SIZE, 0, UVM_KMF_VAONLY);
 	if (va == 0)
 		panic("%s: no virtual space", __func__);
 	cpi->vpage[0] = (void *)(va + 0);
 	cpi->vpage[1] = (void *)(va + PAGE_SIZE);
 #endif /* MULTIPROCESSOR */
 #ifdef notdef
 /*
  * IU implementations are parceled out to vendors (with some slight
  * glitches).  Printing these is cute but takes too much space.
  */
 static char *iu_vendor[16] = {
 	"Fujitsu",	/* and also LSI Logic */
 	"ROSS",		/* ROSS (ex-Cypress) */
 	"BIT",
 	"LSIL",		/* LSI Logic finally got their own */
 	"TI",		/* Texas Instruments */
 	"Matsushita",
 	"Philips",
 	"Harvest",	/* Harvest VLSI Design Center */
 	"SPEC",		/* Systems and Processes Engineering Corporation */
 	"Weitek",
 	"vendor#10",
 	"vendor#11",
 	"vendor#12",
 	"vendor#13",
 	"vendor#14",
 	"vendor#15"
 };
 #endif
 #if defined(MULTIPROCESSOR)
 u_int	cpu_ready_mask;			/* the set of CPUs marked as READY */
 void cpu_spinup(struct cpu_info *);
 static void cpu_attach_non_boot(struct cpu_softc *, struct cpu_info *, int);
 int go_smp_cpus = 0;	/* non-primary CPUs wait for this to go */
 /*
  * This must be locked around all message transactions to ensure only
  * one CPU is generating them.
  */
 static kmutex_t xpmsg_mutex;
 #endif /* MULTIPROCESSOR */
 /*
  * 4/110 comment: the 4/110 chops off the top 4 bits of an OBIO address.
  *	this confuses autoconf.  for example, if you try and map
  *	0xfe000000 in obio space on a 4/110 it actually maps 0x0e000000.
  *	this is easy to verify with the PROM.   this causes problems
  *	with devices like "esp0 at obio0 addr 0xfa000000" because the
  *	4/110 treats it as esp0 at obio0 addr 0x0a000000" which is the
  *	address of the 4/110's "sw0" scsi chip.   the same thing happens
  *	between zs1 and zs2.    since the sun4 line is "closed" and
  *	we know all the "obio" devices that will ever be on it we just
  *	put in some special case "if"'s in the match routines of esp,
  *	dma, and zs.
  */
 @@ -316,150 +291,188 @@ cpu_cpuunit_attach(struct device *parent
 	cpu_attach((struct cpu_softc *)self, cpua->cpua_node,
 	    cpua->cpua_device_id);
+}
 #endif /* SUN4D */
 /*
  * Attach the CPU.
  * Discover interesting goop about the virtual address cache
  * (slightly funny place to do it, but this is where it is to be found).
  */
 static void
 cpu_attach(struct cpu_softc *sc, int node, int mid)
+{
 	char buf[100];
 	struct cpu_info *cpi;
 	int idx;
 	static int cpu_attach_count = 0;
 	/*
 	 * The first CPU we're attaching must be the boot CPU.
 	 * (see autoconf.c and cpuunit.c)
 	 */
 	idx = cpu_attach_count++;
 	if (cpu_attach_count == 1) {
 		getcpuinfo(&cpuinfo, node);
 #if defined(MULTIPROCESSOR)
 		cpi = sc->sc_cpuinfo = cpus[idx];
 #else
 		/* The `local' VA is global for uniprocessor. */
 		cpi = sc->sc_cpuinfo = (struct cpu_info *)CPUINFO_VA;
 #endif
 		cpi->master = 1;
 		cpi->eintstack = eintstack;
 		/* Note: `curpcb' is set to `proc0' in locore */
 		/*
 		 * If we haven't been able to determine the Id of the
 		 * boot CPU, set it now. In this case we can only boot
 		 * from CPU #0 (see also the CPU attach code in autoconf.c)
 		 */
 		if (bootmid == 0)
 			bootmid = mid;
 	} else {
 #if defined(MULTIPROCESSOR)
 		int error;
 		/*
 		 * Initialise this cpu's cpu_info.
 		 */
 		cpi = sc->sc_cpuinfo = cpus[idx];
 		init_cpuinfo(cpi, node);
-		/*
+#if !defined(MULTIPROCESSOR)
-		 * Call the MI attach which creates an idle LWP for us.
+	if (cpu_attach_count > 1) {
 		 */
 		error = mi_cpu_attach(cpi);
 		if (error != 0) {
 			aprint_normal("\n");
 			aprint_error("%s: mi_cpu_attach failed with %d\n",
 			    sc->sc_dev.dv_xname, error);
 			return;
 		/*
 		 * Note: `eintstack' is set in init_cpuinfo() above.
 		 * The %wim register will be initialized in cpu_hatch().
 		 */
 		cpi->ci_curlwp = cpi->ci_data.cpu_idlelwp;
 		cpi->curpcb = (struct pcb *)cpi->ci_curlwp->l_addr;
 		cpi->curpcb->pcb_wim = 1;
 #else
 		sc->sc_cpuinfo = NULL;
 		printf(": no SMP support in kernel\n");
 		return;
 #endif
+	}
 #ifdef DEBUG
 	cpi->redzone = (void *)((long)cpi->eintstack + REDSIZE);
 #endif
 	/*
 	 * Initialise this cpu's cpu_info.
 	 */
 	cpi = sc->sc_cpuinfo = cpus[idx];
 	getcpuinfo(cpi, node);
 	cpi->ci_cpuid = idx;
 	cpi->mid = mid;
 	cpi->node = node;
 #ifdef DEBUG
 	cpi->redzone = (void *)((long)cpi->eintstack + REDSIZE);
 #endif
 	if (sparc_ncpus > 1) {
 		printf(": mid %d", mid);
 		if (mid == 0 && !CPU_ISSUN4D)
 			printf(" [WARNING: mid should not be 0]");
+	}
 #if defined(MULTIPROCESSOR)
 	if (cpu_attach_count > 1) {
 		cpu_attach_non_boot(sc, cpi, node);
 		return;
+	}
 #endif /* MULTIPROCESSOR */
 	/* Stuff to only run on the boot CPU */
 	cpu_setup();
 	snprintf(buf, sizeof buf, "%s @ %s MHz, %s FPU",
 		cpi->cpu_name, clockfreq(cpi->hz), cpi->fpu_name);
 	snprintf(cpu_model, sizeof cpu_model, "%s (%s)",
 		machine_model, buf);
 	printf(": %s\n", buf);
 	cache_print(sc);
 	cpi->master = 1;
 	cpi->eintstack = eintstack;
 	/*
 	 * If we haven't been able to determine the Id of the
 	 * boot CPU, set it now. In this case we can only boot
 	 * from CPU #0 (see also the CPU attach code in autoconf.c)
 	 */
 	if (bootmid == 0)
 		bootmid = mid;
+}
 /*
  * Finish CPU attach.
  * Must be run by the CPU which is being attached.
  */
 void
 cpu_setup(void)
+{
  	if (cpuinfo.hotfix)
 		(*cpuinfo.hotfix)(&cpuinfo);
 	/* Initialize FPU */
 	fpu_init(&cpuinfo);
 	/* Enable the cache */
 	cpuinfo.cache_enable();
 	cpuinfo.flags |= CPUFLG_HATCHED;
+}
 #if defined(MULTIPROCESSOR)
 /*
  * Perform most of the tasks needed for a non-boot CPU.
  */
 static void
 cpu_attach_non_boot(struct cpu_softc *sc, struct cpu_info *cpi, int node)
+{
 	vaddr_t intstack, va;
 	int error;
 	/*
 	 * Arrange interrupt stack.  This cpu will also abuse the bottom
 	 * half of the interrupt stack before it gets to run its idle LWP.
 	 */
 	intstack = uvm_km_alloc(kernel_map, INT_STACK_SIZE, 0, UVM_KMF_WIRED);
 	if (intstack == 0)
 		panic("%s: no uspace/intstack", __func__);
 	cpi->eintstack = (void*)(intstack + INT_STACK_SIZE);
-	if (cpi->master) {
+	/* Allocate virtual space for pmap page_copy/page_zero */
-		char buf[100];
+	va = uvm_km_alloc(kernel_map, 2*PAGE_SIZE, 0, UVM_KMF_VAONLY);
 	if (va == 0)
 		panic("%s: no virtual space", __func__);
-		cpu_setup();
+	cpi->vpage[0] = (void *)(va + 0);
-		snprintf(buf, sizeof buf, "%s @ %s MHz, %s FPU",
+	cpi->vpage[1] = (void *)(va + PAGE_SIZE);
 			cpi->cpu_name, clockfreq(cpi->hz), cpi->fpu_name);
-		snprintf(cpu_model, sizeof cpu_model, "%s (%s)",
+	/*
-			machine_model, buf);
+	 * Call the MI attach which creates an idle LWP for us.
-		printf(": %s\n", buf);
+	 */
-		cache_print(sc);
+	error = mi_cpu_attach(cpi);
 	if (error != 0) {
 		aprint_normal("\n");
 		aprint_error("%s: mi_cpu_attach failed with %d\n",
 		    sc->sc_dev.dv_xname, error);
 		return;
+	}
-#if defined(MULTIPROCESSOR)
+	/*
 	 * Note: `eintstack' is set in init_cpuinfo() above.
 	 * The %wim register will be initialized in cpu_hatch().
 	 */
 	cpi->ci_curlwp = cpi->ci_data.cpu_idlelwp;
 	cpi->curpcb = (struct pcb *)cpi->ci_curlwp->l_addr;
 	cpi->curpcb->pcb_wim = 1;
 	/* for now use the fixed virtual addresses setup in autoconf.c */
 	cpi->intreg_4m = (struct icr_pi *)
-		(PI_INTR_VA + (_MAXNBPG * CPU_MID2CPUNO(mid)));
+		(PI_INTR_VA + (_MAXNBPG * CPU_MID2CPUNO(cpi->mid)));
 	/* Now start this CPU */
 	cpu_spinup(cpi);
 	printf(": %s @ %s MHz, %s FPU\n", cpi->cpu_name,
 		clockfreq(cpi->hz), cpi->fpu_name);
 	cache_print(sc);
-	if (sparc_ncpus > 1 && idx == sparc_ncpus-1) {
+	/*
 	 * Now we're on the last CPU to be attaching.
 	 */
 	if (sparc_ncpus > 1 && cpi->ci_cpuid == sparc_ncpus - 1) {
 		CPU_INFO_ITERATOR n;
 		/*
 		 * Install MP cache flush functions, unless the
 		 * single-processor versions are no-ops.
 		 */
 		for (CPU_INFO_FOREACH(n, cpi)) {
 #define SET_CACHE_FUNC(x) \
 	if (cpi->x != __CONCAT(noop_,x)) cpi->x = __CONCAT(smp_,x)
 			SET_CACHE_FUNC(vcache_flush_page);
 			SET_CACHE_FUNC(vcache_flush_segment);
 			SET_CACHE_FUNC(vcache_flush_region);
 			SET_CACHE_FUNC(vcache_flush_context);
+		}
+	}
-#endif /* MULTIPROCESSOR */
+#undef SET_CACHE_FUNC
+}
 #if defined(MULTIPROCESSOR)
 /*
  * Start secondary processors in motion.
  */
 void
 cpu_boot_secondary_processors(void)
+{
 	CPU_INFO_ITERATOR n;
 	struct cpu_info *cpi;
 	printf("cpu0: booting secondary processors:");
 	for (CPU_INFO_FOREACH(n, cpi)) {
 		if (cpuinfo.mid == cpi->mid ||
 		    (cpi->flags & CPUFLG_HATCHED) == 0)
 @@ -469,61 +482,50 @@ cpu_boot_secondary_processors(void)
 		cpi->flags |= CPUFLG_READY;
 		cpu_ready_mask |= (1 << n);
+	}
 	/* Mark the boot CPU as ready */
 	cpuinfo.flags |= CPUFLG_READY;
 	cpu_ready_mask |= (1 << 0);
 	/* Tell the other CPU's to start up.  */
 	go_smp_cpus = 1;
 	printf("\n");
+}
 #endif /* MULTIPROCESSOR */
 /*
- * Finish CPU attach.
+ * Early initialisation, before main().
  * Must be run by the CPU which is being attached.
  */
 void
-cpu_setup(void)
+cpu_init_system(void)
+{
  	if (cpuinfo.hotfix)
 		(*cpuinfo.hotfix)(&cpuinfo);
 	/* Initialize FPU */
 	fpu_init(&cpuinfo);
-	/* Enable the cache */
+	mutex_init(&xpmsg_mutex, MUTEX_SPIN, IPL_VM);
 	cpuinfo.cache_enable();
 	cpuinfo.flags |= CPUFLG_HATCHED;
+}
 #if defined(MULTIPROCESSOR)
 extern void cpu_hatch(void); /* in locore.s */
 /*
  * Allocate per-CPU data, then start up this CPU using PROM.
  */
 void
 cpu_spinup(struct cpu_info *cpi)
+{
 	extern void cpu_hatch(void); /* in locore.s */
 	struct openprom_addr oa;
-	void *pc = (void *)cpu_hatch;
+	void *pc;
 	int n;
 	pc = (void *)cpu_hatch;
 	/* Setup CPU-specific MMU tables */
 	pmap_alloc_cpu(cpi);
 	cpi->flags &= ~CPUFLG_HATCHED;
 	/*
 	 * The physical address of the context table is passed to
 	 * the PROM in a "physical address descriptor".
 	 */
 	oa.oa_space = 0;
 	oa.oa_base = (uint32_t)cpi->ctx_tbl_pa;
 	oa.oa_size = cpi->mmu_ncontext * sizeof(cpi->ctx_tbl[0]); /*???*/
 @@ -546,62 +548,40 @@ cpu_spinup(struct cpu_info *cpi)
+	}
 	printf("CPU did not spin up\n");
+}
 /*
  * Call a function on some CPUs.  `cpuset' can be set to CPUSET_ALL
  * to call every CPU, or `1 << cpi->ci_cpuid' for each CPU to call.
  */
 void
 xcall(xcall_func_t func, xcall_trap_t trap, int arg0, int arg1, int arg2,
       u_int cpuset)
+{
 	struct cpu_info *cpi;
-	int s, n, i, done, callself, mybit;
+	int n, i, done, callself, mybit;
 	volatile struct xpmsg_func *p;
 	int fasttrap;
 	int is_noop = func == (xcall_func_t)sparc_noop;
 	/* XXX - note p->retval is probably no longer useful */
 	mybit = (1 << cpuinfo.ci_cpuid);
 	callself = func && (cpuset & mybit) != 0;
 	cpuset &= ~mybit;
 	/*
 	 * If no cpus are configured yet, just call ourselves.
 	 */
 	if (cpus == NULL) {
 		p = &cpuinfo.msg.u.xpmsg_func;
 		if (callself)
 			p->retval = (*func)(arg0, arg1, arg2);
 		return;
 	/* Mask any CPUs that are not ready */
 	cpuset &= cpu_ready_mask;
 	/* prevent interrupts that grab the kernel lock */
-	s = splsched();
+	mutex_spin_enter(&xpmsg_mutex);
 #ifdef DEBUG
 	if (!cold) {
 		u_int pc, lvl = ((u_int)s & PSR_PIL) >> 8;
 		if (lvl > IPL_SCHED) {
 			__asm("mov %%i7, %0" : "=r" (pc) : );
 			printf_nolog("%d: xcall at lvl %u from 0x%x\n",
 				cpu_number(), lvl, pc);
 #endif
 	LOCK_XPMSG();
 	/*
 	 * Firstly, call each CPU.  We do this so that they might have
 	 * finished by the time we start looking.
 	 */
 	fasttrap = trap != NULL ? 1 : 0;
 	for (CPU_INFO_FOREACH(n, cpi)) {
 		/* Note: n == cpi->ci_cpuid */
 		if ((cpuset & (1 << n)) == 0)
 			continue;
 		cpi->msg.tag = XPMSG_FUNC;
 @@ -611,174 +591,158 @@ xcall(xcall_func_t func, xcall_trap_t tr
 		p->trap = trap;
 		p->arg0 = arg0;
 		p->arg1 = arg1;
 		p->arg2 = arg2;
 		/* Fast cross calls use interrupt level 14 */
 		raise_ipi(cpi,13+fasttrap);/*xcall_cookie->pil*/
+	}
 	/*
 	 * Second, call ourselves.
 	 */
 	p = &cpuinfo.msg.u.xpmsg_func;
 	if (callself)
-		p->retval = (*func)(arg0, arg1, arg2);
+		(*func)(arg0, arg1, arg2);
 	/*
 	 * Lastly, start looping, waiting for all CPUs to register that they
 	 * have completed (bailing if it takes "too long", being loud about
 	 * this in the process).
 	 */
-	done = 0;
+	done = is_noop;
 	i = 100000;	/* time-out, not too long, but still an _AGE_ */
 	while (!done) {
 		if (--i < 0) {
 			printf_nolog("xcall(cpu%d,%p): couldn't ping cpus:",
 			    cpu_number(), func);
+		}
 		done = 1;
 		for (CPU_INFO_FOREACH(n, cpi)) {
 			if ((cpuset & (1 << n)) == 0)
 				continue;
 			if (cpi->msg.complete == 0) {
 				if (i < 0) {
 					printf_nolog(" cpu%d", cpi->ci_cpuid);
 				} else {
 					done = 0;
 					break;
+				}
+			}
+		}
+	}
 	if (i < 0)
 		printf_nolog("\n");
-	UNLOCK_XPMSG();
+	mutex_spin_exit(&xpmsg_mutex);
 	splx(s);
+}
 /*
  * Tell all CPUs other than the current one to enter the PROM idle loop.
  */
 void
 mp_pause_cpus(void)
+{
 	CPU_INFO_ITERATOR n;
 	struct cpu_info *cpi;
 	if (cpus == NULL)
 		return;
 	for (CPU_INFO_FOREACH(n, cpi)) {
 		if (cpuinfo.mid == cpi->mid ||
 		    (cpi->flags & CPUFLG_HATCHED) == 0)
 			continue;
 		/*
 		 * This PROM utility will put the OPENPROM_MBX_ABORT
 		 * message (0xfc) in the target CPU's mailbox and then
 		 * send it a level 15 soft interrupt.
 		 */
 		if (prom_cpuidle(cpi->node) != 0)
 			printf("cpu%d could not be paused\n", cpi->ci_cpuid);
+	}
+}
 /*
  * Resume all idling CPUs.
  */
 void
 mp_resume_cpus(void)
+{
 	CPU_INFO_ITERATOR n;
 	struct cpu_info *cpi;
 	if (cpus == NULL)
 		return;
 	for (CPU_INFO_FOREACH(n, cpi)) {
 		if (cpuinfo.mid == cpi->mid ||
 		    (cpi->flags & CPUFLG_HATCHED) == 0)
 			continue;
 		/*
 		 * This PROM utility makes the target CPU return
 		 * from its prom_cpuidle(0) call (see intr.c:nmi_soft()).
 		 */
 		if (prom_cpuresume(cpi->node) != 0)
 			printf("cpu%d could not be resumed\n", cpi->ci_cpuid);
+	}
+}
 /*
  * Tell all CPUs except the current one to hurry back into the prom
  */
 void
 mp_halt_cpus(void)
+{
 	CPU_INFO_ITERATOR n;
 	struct cpu_info *cpi;
 	if (cpus == NULL)
 		return;
 	for (CPU_INFO_FOREACH(n, cpi)) {
 		int r;
 		if (cpuinfo.mid == cpi->mid)
 			continue;
 		/*
 		 * This PROM utility will put the OPENPROM_MBX_STOP
 		 * message (0xfb) in the target CPU's mailbox and then
 		 * send it a level 15 soft interrupt.
 		 */
 		r = prom_cpustop(cpi->node);
 		printf("cpu%d %shalted\n", cpi->ci_cpuid,
 			r == 0 ? "" : "(boot CPU?) can not be ");
+	}
+}
 #if defined(DDB)
 void
 mp_pause_cpus_ddb(void)
+{
 	CPU_INFO_ITERATOR n;
 	struct cpu_info *cpi;
 	if (cpus == NULL)
 		return;
 	for (CPU_INFO_FOREACH(n, cpi)) {
 		if (cpi == NULL || cpi->mid == cpuinfo.mid ||
 		    (cpi->flags & CPUFLG_HATCHED) == 0)
 			continue;
 		cpi->msg_lev15.tag = XPMSG15_PAUSECPU;
 		raise_ipi(cpi,15);	/* high priority intr */
+	}
+}
 void
 mp_resume_cpus_ddb(void)
+{
 	CPU_INFO_ITERATOR n;
 	struct cpu_info *cpi;
 	if (cpus == NULL)
 		return;
 	for (CPU_INFO_FOREACH(n, cpi)) {
 		if (cpi == NULL || cpuinfo.mid == cpi->mid ||
 		    (cpi->flags & CPUFLG_PAUSED) == 0)
 			continue;
 		/* tell it to continue */
 		cpi->flags &= ~CPUFLG_PAUSED;
+	}
+}
 #endif /* DDB */
 #endif /* MULTIPROCESSOR */
 /*

 @@ -1,14 +1,14 @@
-/*	$NetBSD: cpuvar.h,v 1.77 2009/05/18 01:36:11 mrg Exp $ */
+/*	$NetBSD: cpuvar.h,v 1.78 2009/05/27 02:19:49 mrg Exp $ */
 /*
  *  Copyright (c) 1996 The NetBSD Foundation, Inc.
  *  All rights reserved.
+ *
  *  This code is derived from software contributed to The NetBSD Foundation
  *  by Paul Kranenburg.
+ *
  *  Redistribution and use in source and binary forms, with or without
  *  modification, are permitted provided that the following conditions
  *  are met:
  *  1. Redistributions of source code must retain the above copyright
  *     notice, this list of conditions and the following disclaimer.
 @@ -27,26 +27,27 @@
  *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  *  POSSIBILITY OF SUCH DAMAGE.
  */
 #ifndef _sparc_cpuvar_h
 #define _sparc_cpuvar_h
 #if defined(_KERNEL_OPT)
 #include "opt_multiprocessor.h"
 #include "opt_lockdebug.h"
 #include "opt_ddb.h"
 #include "opt_sparc_arch.h"
 #include "opt_modular.h"
 #endif
 #include <sys/device.h>
 #include <sys/lock.h>
 #include <sys/cpu_data.h>
 #include <sparc/include/reg.h>
 #include <sparc/sparc/cache.h>	/* for cacheinfo */
 /*
  * CPU/MMU module information.
  * There is one of these for each "mainline" CPU module we support.
  * The information contained in the structure is used only during
 @@ -94,48 +95,38 @@ struct xpmsg {
 	volatile int tag;
 #define	XPMSG15_PAUSECPU	1
 #define	XPMSG_FUNC		4
 #define	XPMSG_FTRP		5
 	volatile union {
 		/*
 		 * Cross call: ask to run (*func)(arg0,arg1,arg2)
 		 * or (*trap)(arg0,arg1,arg2). `trap' should be the
 		 * address of a `fast trap' handler that executes in
 		 * the trap window (see locore.s).
 		 */
 		struct xpmsg_func {
-			int	(*func)(int, int, int);
+			void	(*func)(int, int, int);
 			void	(*trap)(int, int, int);
 			int	arg0;
 			int	arg1;
 			int	arg2;
 			int	retval;
 		} xpmsg_func;
 	} u;
 	volatile int	received;
 	volatile int	complete;
 };
 /*
  * This must be locked around all message transactions to ensure only
  * one CPU is generating them.
  */
 extern struct simplelock xpmsg_lock;
 #define LOCK_XPMSG()	simple_lock(&xpmsg_lock);
 #define UNLOCK_XPMSG()	simple_unlock(&xpmsg_lock);
 /*
  * The cpuinfo structure. This structure maintains information about one
  * currently installed CPU (there may be several of these if the machine
  * supports multiple CPUs, as on some Sun4m architectures). The information
  * in this structure supersedes the old "cpumod", "mmumod", and similar
  * fields.
  */
 struct cpu_info {
 	struct cpu_data ci_data;	/* MI per-cpu data */
 	/* Scheduler flags */
 	int	ci_want_ast;
 	int	ci_want_resched;
 @@ -405,49 +396,56 @@ struct cpu_info {
 #define CPUFLG_CACHEPAGETABLES	0x1	/* caching pagetables OK on Sun4m */
 #define CPUFLG_CACHEIOMMUTABLES	0x2	/* caching IOMMU translations OK */
 #define CPUFLG_CACHEDVMA	0x4	/* DVMA goes through cache */
 #define CPUFLG_SUN4CACHEBUG	0x8	/* trap page can't be cached */
 #define CPUFLG_CACHE_MANDATORY	0x10	/* if cache is on, don't use
 					   uncached access */
 #define CPUFLG_HATCHED		0x1000	/* CPU is alive */
 #define CPUFLG_PAUSED		0x2000	/* CPU is paused */
 #define CPUFLG_GOTMSG		0x4000	/* CPU got an lev13 IPI */
 #define CPUFLG_READY		0x8000	/* CPU available for IPI */
 #define CPU_INFO_ITERATOR		int
-#ifdef MULTIPROCESSOR
+/*
-#define CPU_INFO_FOREACH(cii, cp)	cii = 0; cp = cpus[cii], cii < sparc_ncpus; cii++
+ * Provide two forms of CPU_INFO_FOREACH.  One fast one for non-modular
  * non-SMP kernels, and the other for everyone else.  Both work in the
  * non-SMP case, just involving an extra indirection through cpus[0] for
  * the portable version.
  */
 #if defined(MULTIPROCESSOR) || defined(MODULAR) || defined(_MODULE)
 #define	CPU_INFO_FOREACH(cii, cp)	cii = 0; (cp = cpus[cii]) && cp->eintstack && cii < sparc_ncpus; cii++
 #else
-#define	CPU_INFO_FOREACH(cii, cp)	(void)cii, cp = curcpu(); cp != NULL; cp = NULL
+#define CPU_INFO_FOREACH(cii, cp)	(void)cii, cp = curcpu(); cp != NULL; cp = NULL
 #endif
 /*
  * Useful macros.
  */
 #define CPU_NOTREADY(cpi)	((cpi) == NULL || cpuinfo.mid == (cpi)->mid || \
 				    ((cpi)->flags & CPUFLG_READY) == 0)
 /*
  * Related function prototypes
  */
 void getcpuinfo (struct cpu_info *sc, int node);
 void mmu_install_tables (struct cpu_info *);
 void pmap_alloc_cpu (struct cpu_info *);
 #define	CPUSET_ALL	0xffffffffU	/* xcall to all configured CPUs */
 #if defined(MULTIPROCESSOR)
-typedef int (*xcall_func_t)(int, int, int);
+void cpu_init_system(void);
 typedef void (*xcall_func_t)(int, int, int);
 typedef void (*xcall_trap_t)(int, int, int);
 void xcall(xcall_func_t, xcall_trap_t, int, int, int, u_int);
 /* Shorthand */
 #define XCALL0(f,cpuset)		\
 	xcall((xcall_func_t)f, NULL, 0, 0, 0, cpuset)
 #define XCALL1(f,a1,cpuset)		\
 	xcall((xcall_func_t)f, NULL, (int)a1, 0, 0, cpuset)
 #define XCALL2(f,a1,a2,cpuset)		\
 	xcall((xcall_func_t)f, NULL, (int)a1, (int)a2, 0, cpuset)
 #define XCALL3(f,a1,a2,a3,cpuset)	\
 	xcall((xcall_func_t)f, NULL, (int)a1, (int)a2, (int)a3, cpuset)
 #define FXCALL0(f,tf,cpuset)		\
 @@ -462,22 +460,22 @@ void xcall(xcall_func_t, xcall_trap_t, i
 #define XCALL0(f,cpuset)		/**/
 #define XCALL1(f,a1,cpuset)		/**/
 #define XCALL2(f,a1,a2,cpuset)		/**/
 #define XCALL3(f,a1,a2,a3,cpuset)	/**/
 #define FXCALL0(f,tf,cpuset)		/**/
 #define FXCALL1(f,tf,a1,cpuset)		/**/
 #define FXCALL2(f,tf,a1,a2,cpuset)	/**/
 #define FXCALL3(f,tf,a1,a2,a3,cpuset)	/**/
 #endif /* MULTIPROCESSOR */
 extern int bootmid;			/* Module ID of boot CPU */
 #define CPU_MID2CPUNO(mid)		((mid) != 0 ? (mid) - 8 : 0)
 #ifdef MULTIPROCESSOR
 extern struct cpu_info *cpus[];
 #ifdef MULTIPROCESSOR
 extern u_int cpu_ready_mask;		/* the set of CPUs marked as READY */
 #endif
 #define cpuinfo	(*(struct cpu_info *)CPUINFO_VA)
 #endif	/* _sparc_cpuvar_h */

 @@ -1,14 +1,14 @@
-/*	$NetBSD: pmap.c,v 1.328 2009/05/18 02:28:35 mrg Exp $ */
+/*	$NetBSD: pmap.c,v 1.329 2009/05/27 02:19:50 mrg Exp $ */
 /*
  * Copyright (c) 1996
  * 	The President and Fellows of Harvard College. All rights reserved.
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.  All rights reserved.
+ *
  * This software was developed by the Computer Systems Engineering group
  * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
  * contributed to Berkeley.
+ *
  * All advertising materials mentioning features or use of this software
  * must display the following acknowledgement:
 @@ -46,27 +46,27 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  *	@(#)pmap.c	8.4 (Berkeley) 2/5/94
+ *
  */
 /*
  * SPARC physical map management code.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.328 2009/05/18 02:28:35 mrg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.329 2009/05/27 02:19:50 mrg Exp $");
 #include "opt_ddb.h"
 #include "opt_kgdb.h"
 #include "opt_sparc_arch.h"
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/device.h>
 #include <sys/proc.h>
 #include <sys/queue.h>
 #include <sys/pool.h>
 #include <sys/exec.h>
 #include <sys/core.h>
 @@ -3492,26 +3492,43 @@ pmap_bootstrap4m(void *top)
 	pmap_kremove_p	 	=	pmap_kremove4m;
 	pmap_kprotect_p	 	=	pmap_kprotect4m;
 	pmap_page_protect_p	=	pmap_page_protect4m;
 	pmap_protect_p		=	pmap_protect4m;
 	pmap_rmk_p		=	pmap_rmk4m;
 	pmap_rmu_p		=	pmap_rmu4m;
 #endif /* defined SUN4/SUN4C */
 	/*
 	 * p points to top of kernel mem
 	 */
 	p = (vaddr_t)top;
 #if defined(MULTIPROCESSOR)
 	/*
 	 * allocate the rest of the cpu_info{} area.  note we waste the
 	 * first one to get a VA space.
 	 */
 	cpuinfo_len = ((sizeof(struct cpu_info) + NBPG - 1) & ~PGOFSET);
 	if (sparc_ncpus > 1) {
 		p = (p + NBPG - 1) & ~PGOFSET;
 		cpuinfo_data = (uint8_t *)p;
 		p += (cpuinfo_len * sparc_ncpus);
 		/* XXX we waste the first one */
 		memset(cpuinfo_data + cpuinfo_len, 0, cpuinfo_len * (sparc_ncpus - 1));
 	} else
 		cpuinfo_data = (uint8_t *)CPUINFO_VA;
 #endif
 	/*
 	 * Intialize the kernel pmap.
 	 */
 	/* kernel_pmap_store.pm_ctxnum = 0; */
 	kernel_pmap_store.pm_refcount = 1;
 	/*
 	 * Set up pm_regmap for kernel to point NUREG *below* the beginning
 	 * of kernel regmap storage. Since the kernel only uses regions
 	 * above NUREG, we save storage space and can index kernel and
 	 * user regions in the same way.
 	 */
 	kernel_pmap_store.pm_regmap = &kernel_regmap_store[-NUREG];
 @@ -3529,42 +3546,26 @@ pmap_bootstrap4m(void *top)
 	pmap_kernel()->pm_reg_ptps = (int **)(q = p);
 	p += sparc_ncpus * sizeof(int **);
 	memset((void *)q, 0, (u_int)p - (u_int)q);
 	pmap_kernel()->pm_reg_ptps_pa = (int *)(q = p);
 	p += sparc_ncpus * sizeof(int *);
 	memset((void *)q, 0, (u_int)p - (u_int)q);
 	/* Allocate context administration */
 	pmap_kernel()->pm_ctx = ctxinfo = ci = (union ctxinfo *)p;
 	p += ncontext * sizeof *ci;
 	memset((void *)ci, 0, (u_int)p - (u_int)ci);
 #if defined(MULTIPROCESSOR)
 	/*
 	 * allocate the rest of the cpu_info{} area.  note we waste the
 	 * first one to get a VA space.
 	 */
 	p = (p + NBPG - 1) & ~PGOFSET;
 	cpuinfo_data = (uint8_t *)p;
 	cpuinfo_len = ((sizeof(struct cpu_info) + NBPG - 1) & ~PGOFSET);
 	p += (cpuinfo_len * sparc_ncpus);
 	prom_printf("extra cpus: %p, p: %p, gap start: %p, gap end: %p\n",
 	    cpuinfo_data, p, etext_gap_start, etext_gap_end);
 	/* XXX we waste the first one */
 	memset(cpuinfo_data + cpuinfo_len, 0, cpuinfo_len * (sparc_ncpus - 1));
 #endif
 	/*
 	 * Set up the `constants' for the call to vm_init()
 	 * in main().  All pages beginning at p (rounded up to
 	 * the next whole page) and continuing through the number
 	 * of available pages are free.
 	 */
 	p = (p + NBPG - 1) & ~PGOFSET;
 	/*
 	 * Reserve memory for MMU pagetables. Some of these have severe
 	 * alignment restrictions. We allocate in a sequence that
 	 * minimizes alignment gaps.
 	 */
 @@ -3789,47 +3790,56 @@ pmap_bootstrap4m(void *top)
 				pa += NBPG;
 				size -= NBPG;
+			}
+		}
+	}
 	/*
 	 * Now switch to kernel pagetables (finally!)
 	 */
 	mmu_install_tables(&cpuinfo);
 #ifdef MULTIPROCESSOR
 	/*
 	 * Initialise any cpu-specific data now.
 	 */
 	cpu_init_system();
 	/*
 	 * Remap cpu0 from CPUINFO_VA to the new correct value, wasting the
-	 * backing pages we allocated above XXX.
+	 * backing page we allocated above XXX.
 	 */
 	for (off = 0, va = (vaddr_t)cpuinfo_data;
-	     off < sizeof(struct cpu_info);
+	     sparc_ncpus > 1 && off < sizeof(struct cpu_info);
 	     va += NBPG, off += NBPG) {
 		paddr_t pa = PMAP_BOOTSTRAP_VA2PA(CPUINFO_VA + off);
 		prom_printf("going to pmap_kenter_pa(va=%p, pa=%p)\n", va, pa);
 		pmap_kremove(va, NBPG);
 		pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE);
 		cache_flush_page(va, 0);
 		cache_flush_page(CPUINFO_VA, 0);
+	}
 	/*
 	 * Setup the cpus[] array and the ci_self links.
 	 */
 	prom_printf("setting cpus self reference\n");
 	for (i = 0; i < sparc_ncpus; i++) {
 		cpus[i] = (struct cpu_info *)(cpuinfo_data + (cpuinfo_len * i));
 		cpus[i]->ci_self = cpus[i];
 		prom_printf("set cpu%d ci_self address: %p\n", i, cpus[i]);
+	}
 #else
 	cpus[0] = (struct cpu_info *)CPUINFO_VA;
 #endif
 	pmap_update(pmap_kernel());
 	prom_printf("pmap_bootstrap4m done\n");
+}
 static u_long prom_ctxreg;
 void
 mmu_install_tables(struct cpu_info *sc)
+{
 #ifdef DEBUG