 @@ -1,14 +1,14 @@
-/*	$NetBSD: cpu.c,v 1.37 2009/11/21 05:54:04 rmind Exp $	*/
+/*	$NetBSD: cpu.c,v 1.38 2009/11/24 13:04:04 cegger Exp $	*/
 /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp  */
 /*-
  * Copyright (c) 2000 The NetBSD Foundation, Inc.
  * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by RedBack Networks Inc.
+ *
  * Author: Bill Sommerfeld
+ *
  * Redistribution and use in source and binary forms, with or without
 @@ -56,27 +56,27 @@
  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.37 2009/11/21 05:54:04 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.38 2009/11/24 13:04:04 cegger Exp $");
 #include "opt_ddb.h"
 #include "opt_multiprocessor.h"
 #include "opt_mpbios.h"		/* for MPDEBUG */
 #include "opt_mtrr.h"
 #include "opt_xen.h"
 #include "lapic.h"
 #include "ioapic.h"
 #include <sys/param.h>
 #include <sys/proc.h>
 #include <sys/user.h>
 @@ -102,27 +102,29 @@ __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.37
 #include <machine/pio.h>
 #include <xen/vcpuvar.h>
 #if NLAPIC > 0
 #include <machine/apicvar.h>
 #include <machine/i82489reg.h>
 #include <machine/i82489var.h>
 #endif
 #include <dev/ic/mc146818reg.h>
 #include <dev/isa/isareg.h>
-#define	X86_MAXPROCS	32
+#if MAXCPUS > 32
 #error cpu_info contains 32bit bitmasks
 #endif
 int     cpu_match(device_t, cfdata_t, void *);
 void    cpu_attach(device_t, device_t, void *);
 int     vcpu_match(device_t, cfdata_t, void *);
 void    vcpu_attach(device_t, device_t, void *);
 void    cpu_attach_common(device_t, device_t, void *);
 void	cpu_offline_md(void);
 struct cpu_softc {
 	device_t sc_dev;		/* device tree glue */
 	struct cpu_info *sc_info;	/* pointer to CPU info */
 	bool sc_wasonline;
 };
 @@ -136,171 +138,174 @@ CFATTACH_DECL_NEW(cpu, sizeof(struct cpu
     cpu_match, cpu_attach, NULL, NULL);
 CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc),
     vcpu_match, vcpu_attach, NULL, NULL);
 /*
  * Statically-allocated CPU info for the primary CPU (or the only
  * CPU, on uniprocessors).  The CPU info list is initialized to
  * point at it.
  */
 #ifdef TRAPLOG
 #include <machine/tlog.h>
 struct tlog tlog_primary;
 #endif
-struct cpu_info cpu_info_primary = {
+struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = {
 	.ci_dev = 0,
 	.ci_self = &cpu_info_primary,
 	.ci_idepth = -1,
 	.ci_curlwp = &lwp0,
 	.ci_curldt = -1,
 #ifdef TRAPLOG
 	.ci_tlog = &tlog_primary,
 #endif
 };
-struct cpu_info phycpu_info_primary = {
+struct cpu_info phycpu_info_primary __aligned(CACHE_LINE_SIZE) = {
 	.ci_dev = 0,
 	.ci_self = &phycpu_info_primary,
 };
 struct cpu_info *cpu_info_list = &cpu_info_primary;
 struct cpu_info *phycpu_info_list = &phycpu_info_primary;
 static void	cpu_set_tss_gates(struct cpu_info *ci);
 uint32_t cpus_attached = 0;
 uint32_t cpus_running = 0;
 uint32_t phycpus_attached = 0;
 uint32_t phycpus_running = 0;
 bool x86_mp_online;
 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
-struct cpu_info *phycpu_info[X86_MAXPROCS] = { &cpu_info_primary };
+#if defined(MULTIPROCESSOR)
 #ifdef MULTIPROCESSOR
 /*
  * Array of CPU info structures.  Must be statically-allocated because
  * curproc, etc. are used early.
  */
 struct cpu_info *cpu_info[X86_MAXPROCS] = { &cpu_info_primary };
 void    	cpu_hatch(void *);
 static void    	cpu_boot_secondary(struct cpu_info *ci);
 static void    	cpu_start_secondary(struct cpu_info *ci);
 static void	cpu_copy_trampoline(void);
 /*
  * Runs once per boot once multiprocessor goo has been detected and
  * the local APIC on the boot processor has been mapped.
+ *
  * Called from lapic_boot_init() (from mpbios_scan()).
  */
 void
 cpu_init_first(void)
+{
 	int cpunum = lapic_cpu_number();
 	if (cpunum != 0) {
 		cpu_info[0] = NULL;
 		cpu_info[cpunum] = &cpu_info_primary;
 	cpu_info_primary.ci_cpuid = lapic_cpu_number();
 	cpu_copy_trampoline();
+}
-#endif
+#endif	/* MULTIPROCESSOR */
 int
 cpu_match(device_t parent, cfdata_t match, void *aux)
+{
 	return 1;
+}
 void
 cpu_attach(device_t parent, device_t self, void *aux)
+{
 	struct cpu_softc *sc = device_private(self);
 	struct cpu_attach_args *caa = aux;
 	struct cpu_info *ci;
 	uintptr_t ptr;
-	int cpunum = caa->cpu_number;
+	static bool again = false;
 	sc->sc_dev = self;
-	if (cpus_attached == ~0) {
+	if (phycpus_attached == ~0) {
 		aprint_error(": increase MAXCPUS\n");
 		return;
+	}
 	/*
 	 * If we're an Application Processor, allocate a cpu_info
 	 * structure, otherwise use the primary's.
 	 */
 	if (caa->cpu_role == CPU_ROLE_AP) {
 		if ((boothowto & RB_MD1) != 0) {
 			aprint_error(": multiprocessor boot disabled\n");
 			if (!pmf_device_register(self, NULL, NULL))
 				aprint_error_dev(self,
 				   "couldn't establish power handler\n");
 			return;
+		}
 		aprint_naive(": Application Processor\n");
 		ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
 		    KM_SLEEP);
 		ci = (struct cpu_info *)((ptr + CACHE_LINE_SIZE - 1) &
 		    ~(CACHE_LINE_SIZE - 1));
 		ci->ci_curldt = -1;
 		if (phycpu_info[cpunum] != NULL)
 			panic("cpu at apic id %d already attached?", cpunum);
 		phycpu_info[cpunum] = ci;
 	} else {
 		aprint_naive(": %s Processor\n",
 		    caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
 		ci = &phycpu_info_primary;
 		if (cpunum != 0) {
 			phycpu_info[0] = NULL;
 			phycpu_info[cpunum] = ci;
+	}
 	ci->ci_self = ci;
 	sc->sc_info = ci;
 	ci->ci_dev = self;
 	ci->ci_cpuid = caa->cpu_number;
 	ci->ci_vcpu = NULL;
 	/*
 	 * Boot processor may not be attached first, but the below
 	 * must be done to allow booting other processors.
 	 */
 	if (!again) {
 		atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
 		/* Basic init */
 		again = true;
+	}
 	printf(": ");
 	switch (caa->cpu_role) {
 	case CPU_ROLE_SP:
 		printf("(uniprocessor)\n");
-		atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY);
+		atomic_or_32(&ci->ci_flags, CPUF_SP);
 		break;
 	case CPU_ROLE_BP:
 		printf("(boot processor)\n");
-		atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY);
+		atomic_or_32(&ci->ci_flags, CPUF_BSP);
 		break;
 	case CPU_ROLE_AP:
 		/*
 		 * report on an AP
 		 */
 		printf("(application processor)\n");
 		if (ci->ci_flags & CPUF_PRESENT) {
 			struct cpu_info *tmp;
 			tmp = phycpu_info_list;
 			while (tmp->ci_next)
 				tmp = tmp->ci_next;
 			tmp->ci_next = ci;
+		}
 		break;
 	default:
 		panic("unknown processor type??\n");
+	}
-	atomic_or_32(&cpus_attached, ci->ci_cpumask);
+	atomic_or_32(&phycpus_attached, ci->ci_cpumask);
 	return;
+}
 int
 vcpu_match(device_t parent, cfdata_t match, void *aux)
+{
 	struct vcpu_attach_args *vcaa = aux;
 	if (strcmp(vcaa->vcaa_name, match->cf_name) == 0)
 		return 1;
 	return 0;
+}
 @@ -346,185 +351,191 @@ cpu_vm_init(struct cpu_info *ci)
 		return;
 	aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);
 	uvm_page_recolor(ncolors);
+}
 void
 cpu_attach_common(device_t parent, device_t self, void *aux)
+{
 	struct cpu_softc *sc = device_private(self);
 	struct cpu_attach_args *caa = aux;
 	struct cpu_info *ci;
 	uintptr_t ptr;
 	int cpunum = caa->cpu_number;
 	static bool again = false;
 	sc->sc_dev = self;
 	/*
 	 * If we're an Application Processor, allocate a cpu_info
 	 * structure, otherwise use the primary's.
 	 */
 	if (caa->cpu_role == CPU_ROLE_AP) {
 		if (cpunum >= X86_MAXPROCS) {
 			aprint_error(": apic id %d ignored, "
 				"please increase X86_MAXPROCS\n", cpunum);
 		aprint_naive(": Application Processor\n");
 		ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
 		    KM_SLEEP);
 		ci = (struct cpu_info *)((ptr + CACHE_LINE_SIZE - 1) &
 		    ~(CACHE_LINE_SIZE - 1));
 		memset(ci, 0, sizeof(*ci));
 #if defined(MULTIPROCESSOR)
 		if (cpu_info[cpunum] != NULL)
 			panic("cpu at apic id %d already attached?", cpunum);
 		cpu_info[cpunum] = ci;
 #endif
 #ifdef TRAPLOG
 		ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP);
 #endif
 	} else {
 		aprint_naive(": %s Processor\n",
 		    caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
 		ci = &cpu_info_primary;
-#if defined(MULTIPROCESSOR)
+#if NLAPIC > 0
 		if (cpunum != lapic_cpu_number()) {
-			panic("%s: running CPU is at apic %d"
+			/* XXX should be done earlier */
-			    " instead of at expected %d",
+			uint32_t reg;
-			    device_xname(sc->sc_dev), lapic_cpu_number(), cpunum);
+			aprint_verbose("\n");
 			aprint_verbose_dev(self, "running CPU at apic %d"
 			    " instead of at expected %d", lapic_cpu_number(),
 			    cpunum);
 			reg = i82489_readreg(LAPIC_ID);
 			i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) |
 			    (cpunum << LAPIC_ID_SHIFT));
+		}
 		if (cpunum != lapic_cpu_number()) {
 			aprint_error_dev(self, "unable to reset apic id\n");
+		}
 #endif
+	}
 	ci->ci_self = ci;
 	sc->sc_info = ci;
 	ci->ci_dev = self;
 	ci->ci_cpuid = cpunum;
 	KASSERT(HYPERVISOR_shared_info != NULL);
 	ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum];
 	ci->ci_func = caa->cpu_func;
 	/* Must be called before mi_cpu_attach(). */
 	cpu_vm_init(ci);
 	if (caa->cpu_role == CPU_ROLE_AP) {
 #if defined(MULTIPROCESSOR)
 		int error;
 		error = mi_cpu_attach(ci);
 		if (error != 0) {
 			aprint_normal("\n");
-			aprint_error_dev(sc->sc_dev, "mi_cpu_attach failed with %d\n",
+			aprint_error_dev(self,
-			    error);
+			    "mi_cpu_attach failed with %d\n", error);
 			return;
+		}
 #endif
 	} else {
 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);
+	}
 	ci->ci_cpumask = (1 << cpu_index(ci));
 	pmap_reference(pmap_kernel());
 	ci->ci_pmap = pmap_kernel();
 	ci->ci_tlbstate = TLBSTATE_STALE;
-	/* further PCB init done later. */
+	/*
 	 * Boot processor may not be attached first, but the below
-	switch (caa->cpu_role) {
+	 * must be done to allow booting other processors.
-	case CPU_ROLE_SP:
+	 */
-		atomic_or_32(&ci->ci_flags,
+	if (!again) {
-		     CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY);
+		atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
 		/* Basic init. */
 		cpu_intr_init(ci);
 		cpu_get_tsc_freq(ci);
 		cpu_identify(ci);
 		cpu_init(ci);
 		cpu_set_tss_gates(ci);
 		pmap_cpu_init_late(ci);
-		x86_cpu_idle_init();
+#if NLAPIC > 0
 		if (caa->cpu_role != CPU_ROLE_SP) {
 			/* Enable lapic. */
 			lapic_enable();
 			lapic_set_lvt();
 			lapic_calibrate_timer();
+		}
 #endif
 		/* Make sure DELAY() is initialized. */
 		DELAY(1);
 		again = true;
+	}
 	/* further PCB init done later. */
 	switch (caa->cpu_role) {
 	case CPU_ROLE_SP:
 		atomic_or_32(&ci->ci_flags, CPUF_SP);
 		cpu_identify(ci);
 #if 0
 		x86_errata();
 #endif
 		x86_cpu_idle_init();
 		break;
 	case CPU_ROLE_BP:
-		atomic_or_32(&ci->ci_flags,
+		atomic_or_32(&ci->ci_flags, CPUF_BSP);
 		    CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY);
 		cpu_intr_init(ci);
 		cpu_get_tsc_freq(ci);
 		cpu_identify(ci);
 		cpu_init(ci);
 		cpu_set_tss_gates(ci);
 		pmap_cpu_init_late(ci);
 		x86_cpu_idle_init();
 #if NLAPIC > 0
 		/*
 		 * Enable local apic
 		 */
 		lapic_enable();
 		lapic_set_lvt();
 		lapic_calibrate_timer(ci);
 #endif
 #if 0
 		x86_errata();
 #endif
 		x86_cpu_idle_init();
 		break;
 	case CPU_ROLE_AP:
 		/*
 		 * report on an AP
 		 */
 #if defined(MULTIPROCESSOR)
 		cpu_intr_init(ci);
 		gdt_alloc_cpu(ci);
 		cpu_set_tss_gates(ci);
 		pmap_cpu_init_early(ci);
 		pmap_cpu_init_late(ci);
 		cpu_start_secondary(ci);
 		if (ci->ci_flags & CPUF_PRESENT) {
 			struct cpu_info *tmp;
 			identifycpu(ci);
 			tmp = cpu_info_list;
 			while (tmp->ci_next)
 				tmp = tmp->ci_next;
 			tmp->ci_next = ci;
+		}
 #else
-		aprint_normal_dev(sc->sc_dev, "not started\n");
+		aprint_error_dev(self, "not started\n");
 #endif
 		break;
 	default:
 		aprint_normal("\n");
 		panic("unknown processor type??\n");
+	}
 	cpu_vm_init(ci);
 	atomic_or_32(&cpus_attached, ci->ci_cpumask);
 #if 0
 	if (!pmf_device_register(self, cpu_suspend, cpu_resume))
 		aprint_error_dev(self, "couldn't establish power handler\n");
 #endif
 #if defined(MULTIPROCESSOR)
 	if (mp_verbose) {
 		struct lwp *l = ci->ci_data.cpu_idlelwp;
 		struct pcb *pcb = lwp_getpcb(l);
-		aprint_verbose_dev(sc->sc_dev, "idle lwp at %p, idle sp at 0x%p\n",
+		aprint_verbose_dev(self,
 		    "idle lwp at %p, idle sp at 0x%p\n",
 		    l,
 #ifdef i386
 		    (void *)pcb->pcb_esp
 #else
 		    (void *)pcb->pcb_rsp
 #endif
 		);
+	}
 #endif
+}
 /*
 @@ -567,28 +578,28 @@ cpu_init(struct cpu_info *ci)
 	atomic_or_32(&cpus_running, ci->ci_cpumask);
 	atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
+}
 #ifdef MULTIPROCESSOR
 void
 cpu_boot_secondary_processors(void)
+{
 	struct cpu_info *ci;
 	u_long i;
-	for (i = 0; i < X86_MAXPROCS; i++) {
+	for (i = 0; i < maxcpus; i++) {
-		ci = cpu_info[i];
+		ci = cpu_lookup(i);
 		if (ci == NULL)
 			continue;
 		if (ci->ci_data.cpu_idlelwp == NULL)
 			continue;
 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
 			continue;
 		if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
 			continue;
 		cpu_boot_secondary(ci);
+	}
 	x86_mp_online = true;
+}
 @@ -598,28 +609,28 @@ cpu_init_idle_lwp(struct cpu_info *ci)
+{
 	struct lwp *l = ci->ci_data.cpu_idlelwp;
 	struct pcb *pcb = lwp_getpcb(l);
 	pcb->pcb_cr0 = rcr0();
+}
 void
 cpu_init_idle_lwps(void)
+{
 	struct cpu_info *ci;
 	u_long i;
-	for (i = 0; i < X86_MAXPROCS; i++) {
+	for (i = 0; i < maxcpus; i++) {
-		ci = cpu_info[i];
+		ci = cpu_lookup(i);
 		if (ci == NULL)
 			continue;
 		if (ci->ci_data.cpu_idlelwp == NULL)
 			continue;
 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
 			continue;
 		cpu_init_idle_lwp(ci);
+	}
+}
 void
 cpu_start_secondary(struct cpu_info *ci)
+{
 @@ -777,27 +788,27 @@ cpu_debug_dump(void)
 	CPU_INFO_ITERATOR cii;
 	db_printf("addr		dev	id	flags	ipis	curlwp 		fpcurlwp\n");
 	for (CPU_INFO_FOREACH(cii, ci)) {
 		db_printf("%p	%s	%ld	%x	%x	%10p	%10p\n",
 		    ci,
 		    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
 		    (long)ci->ci_cpuid,
 		    ci->ci_flags, ci->ci_ipis,
 		    ci->ci_curlwp,
 		    ci->ci_fpcurlwp);
+	}
+}
-#endif
+#endif /* DDB */
 static void
 cpu_copy_trampoline(void)
+{
 	/*
 	 * Copy boot code.
 	 */
 	extern u_char cpu_spinup_trampoline[];
 	extern u_char cpu_spinup_trampoline_end[];
 	vaddr_t mp_trampoline_vaddr;
 	mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
 @@ -805,27 +816,27 @@ cpu_copy_trampoline(void)
 	pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr,
 		VM_PROT_READ | VM_PROT_WRITE, 0);
 	pmap_update(pmap_kernel());
 	memcpy((void *)mp_trampoline_vaddr,
 		cpu_spinup_trampoline,
 		cpu_spinup_trampoline_end - cpu_spinup_trampoline);
 	pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE);
 	pmap_update(pmap_kernel());
 	uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY);
+}
-#endif
+#endif /* MULTIPROCESSOR */
 #ifdef i386
 #if 0
 static void
 tss_init(struct i386tss *tss, void *stack, void *func)
+{
 	memset(tss, 0, sizeof *tss);
 	tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16);
 	tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
 	tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL);
 	tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL);
 	tss->tss_gs = tss->__tss_es = tss->__tss_ds =
 	    tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL);