 @@ -1,563 +1,563 @@
-/*	$NetBSD: arm32_machdep.c,v 1.83 2012/08/31 23:59:51 matt Exp $	*/
+/*	$NetBSD: arm32_machdep.c,v 1.84 2012/09/16 22:09:33 rmind Exp $	*/
 /*
  * Copyright (c) 1994-1998 Mark Brinicombe.
  * Copyright (c) 1994 Brini.
  * All rights reserved.
+ *
  * This code is derived from software written for Brini by Mark Brinicombe
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. All advertising materials mentioning features or use of this software
  *    must display the following acknowledgement:
  *	This product includes software developed by Mark Brinicombe
  *	for the NetBSD Project.
  * 4. The name of the company nor the name of the author may be used to
  *    endorse or promote products derived from this software without specific
  *    prior written permission.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  * Machine dependent functions for kernel setup
+ *
  * Created      : 17/09/94
  * Updated	: 18/04/01 updated for new wscons
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: arm32_machdep.c,v 1.83 2012/08/31 23:59:51 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: arm32_machdep.c,v 1.84 2012/09/16 22:09:33 rmind Exp $");
 #include "opt_modular.h"
 #include "opt_md.h"
 #include "opt_pmap_debug.h"
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/reboot.h>
 #include <sys/proc.h>
 #include <sys/kauth.h>
 #include <sys/kernel.h>
 #include <sys/mbuf.h>
 #include <sys/mount.h>
 #include <sys/buf.h>
 #include <sys/msgbuf.h>
 #include <sys/device.h>
 #include <sys/sysctl.h>
 #include <sys/cpu.h>
 #include <sys/intr.h>
 #include <sys/module.h>
 #include <sys/atomic.h>
 #include <sys/xcall.h>
 #include <uvm/uvm_extern.h>
 #include <dev/cons.h>
 #include <dev/mm.h>
 #include <arm/arm32/katelib.h>
 #include <arm/arm32/machdep.h>
 #include <machine/bootconfig.h>
 #include <machine/pcb.h>
 void (*cpu_reset_address)(void);	/* Used by locore */
 paddr_t cpu_reset_address_paddr;	/* Used by locore */
 struct vm_map *phys_map = NULL;
 #if defined(MEMORY_DISK_HOOKS) && !defined(MEMORY_DISK_ROOT_SIZE)
 extern size_t md_root_size;		/* Memory disc size */
 #endif	/* MEMORY_DISK_HOOKS && !MEMORY_DISK_ROOT_SIZE */
 pv_addr_t kernelstack;
 pv_addr_t abtstack;
 pv_addr_t fiqstack;
 pv_addr_t irqstack;
 pv_addr_t undstack;
 pv_addr_t idlestack;
 void *	msgbufaddr;
 extern paddr_t msgbufphys;
 int kernel_debug = 0;
 /* exported variable to be filled in by the bootloaders */
 char *booted_kernel;
 /* Prototypes */
 void data_abort_handler(trapframe_t *frame);
 void prefetch_abort_handler(trapframe_t *frame);
 extern void configure(void);
 /*
  * arm32_vector_init:
+ *
  *	Initialize the vector page, and select whether or not to
  *	relocate the vectors.
+ *
  *	NOTE: We expect the vector page to be mapped at its expected
  *	destination.
  */
 void
 arm32_vector_init(vaddr_t va, int which)
+{
 	if (CPU_IS_PRIMARY(curcpu())) {
 		extern unsigned int page0[], page0_data[];
 		unsigned int *vectors = (int *) va;
 		unsigned int *vectors_data = vectors + (page0_data - page0);
 		int vec;
 		/*
 		 * Loop through the vectors we're taking over, and copy the
 		 * vector's insn and data word.
 		 */
 		for (vec = 0; vec < ARM_NVEC; vec++) {
 			if ((which & (1 << vec)) == 0) {
 				/* Don't want to take over this vector. */
 				continue;
+			}
 			vectors[vec] = page0[vec];
 			vectors_data[vec] = page0_data[vec];
+		}
 		/* Now sync the vectors. */
 		cpu_icache_sync_range(va, (ARM_NVEC * 2) * sizeof(u_int));
 		vector_page = va;
+	}
 	if (va == ARM_VECTORS_HIGH) {
 		/*
 		 * Assume the MD caller knows what it's doing here, and
 		 * really does want the vector page relocated.
+		 *
 		 * Note: This has to be done here (and not just in
 		 * cpu_setup()) because the vector page needs to be
 		 * accessible *before* cpu_startup() is called.
 		 * Think ddb(9) ...
+		 *
 		 * NOTE: If the CPU control register is not readable,
 		 * this will totally fail!  We'll just assume that
 		 * any system that has high vector support has a
 		 * readable CPU control register, for now.  If we
 		 * ever encounter one that does not, we'll have to
 		 * rethink this.
 		 */
 		cpu_control(CPU_CONTROL_VECRELOC, CPU_CONTROL_VECRELOC);
+	}
+}
 /*
  * Debug function just to park the CPU
  */
 void
 halt(void)
+{
 	while (1)
 		cpu_sleep(0);
+}
 /* Sync the discs and unmount the filesystems */
 void
 bootsync(void)
+{
 	static bool bootsyncdone = false;
 	if (bootsyncdone) return;
 	bootsyncdone = true;
 	/* Make sure we can still manage to do things */
 	if (GetCPSR() & I32_bit) {
 		/*
 		 * If we get here then boot has been called without RB_NOSYNC
 		 * and interrupts were disabled. This means the boot() call
 		 * did not come from a user process e.g. shutdown, but must
 		 * have come from somewhere in the kernel.
 		 */
 		IRQenable;
 		printf("Warning IRQ's disabled during boot()\n");
+	}
 	vfs_shutdown();
+}
 /*
  * void cpu_startup(void)
+ *
  * Machine dependent startup code.
+ *
  */
 void
 cpu_startup(void)
+{
 	vaddr_t minaddr;
 	vaddr_t maxaddr;
 	u_int loop;
 	char pbuf[9];
 	/*
 	 * Until we better locking, we have to live under the kernel lock.
 	 */
 	//KERNEL_LOCK(1, NULL);
 	/* Set the CPU control register */
 	cpu_setup(boot_args);
 	/* Lock down zero page */
 	vector_page_setprot(VM_PROT_READ);
 	/*
 	 * Give pmap a chance to set up a few more things now the vm
 	 * is initialised
 	 */
 	pmap_postinit();
 	/*
 	 * Initialize error message buffer (at end of core).
 	 */
 	/* msgbufphys was setup during the secondary boot strap */
 	for (loop = 0; loop < btoc(MSGBUFSIZE); ++loop)
 		pmap_kenter_pa((vaddr_t)msgbufaddr + loop * PAGE_SIZE,
 		    msgbufphys + loop * PAGE_SIZE,
 		    VM_PROT_READ|VM_PROT_WRITE, 0);
 	pmap_update(pmap_kernel());
 	initmsgbuf(msgbufaddr, round_page(MSGBUFSIZE));
 	/*
 	 * Identify ourselves for the msgbuf (everything printed earlier will
 	 * not be buffered).
 	 */
 	printf("%s%s", copyright, version);
 	format_bytes(pbuf, sizeof(pbuf), arm_ptob(physmem));
 	printf("total memory = %s\n", pbuf);
 	minaddr = 0;
 	/*
 	 * Allocate a submap for physio
 	 */
 	phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
 				   VM_PHYS_SIZE, 0, false, NULL);
 	format_bytes(pbuf, sizeof(pbuf), ptoa(uvmexp.free));
 	printf("avail memory = %s\n", pbuf);
 	struct lwp * const l = &lwp0;
 	struct pcb * const pcb = lwp_getpcb(l);
 	pcb->pcb_sp = uvm_lwp_getuarea(l) + USPACE_SVC_STACK_TOP;
 	lwp_settrapframe(l, (struct trapframe *)pcb->pcb_sp - 1);
+}
 /*
  * machine dependent system variables.
  */
 static int
 sysctl_machdep_booted_device(SYSCTLFN_ARGS)
+{
 	struct sysctlnode node;
 	if (booted_device == NULL)
 		return (EOPNOTSUPP);
 	node = *rnode;
 	node.sysctl_data = booted_device->dv_xname;
 	node.sysctl_size = strlen(booted_device->dv_xname) + 1;
 	return (sysctl_lookup(SYSCTLFN_CALL(&node)));
+}
 static int
 sysctl_machdep_booted_kernel(SYSCTLFN_ARGS)
+{
 	struct sysctlnode node;
 	if (booted_kernel == NULL || booted_kernel[0] == '\0')
 		return (EOPNOTSUPP);
 	node = *rnode;
 	node.sysctl_data = booted_kernel;
 	node.sysctl_size = strlen(booted_kernel) + 1;
 	return (sysctl_lookup(SYSCTLFN_CALL(&node)));
+}
 static int
 sysctl_machdep_powersave(SYSCTLFN_ARGS)
+{
 	struct sysctlnode node = *rnode;
 	int error, newval;
 	newval = cpu_do_powersave;
 	node.sysctl_data = &newval;
 	if (cpufuncs.cf_sleep == (void *) cpufunc_nullop)
 		node.sysctl_flags &= ~CTLFLAG_READWRITE;
 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
 	if (error || newp == NULL || newval == cpu_do_powersave)
 		return (error);
 	if (newval < 0 || newval > 1)
 		return (EINVAL);
 	cpu_do_powersave = newval;
 	return (0);
+}
 SYSCTL_SETUP(sysctl_machdep_setup, "sysctl machdep subtree setup")
+{
 	sysctl_createv(clog, 0, NULL, NULL,
 		       CTLFLAG_PERMANENT,
 		       CTLTYPE_NODE, "machdep", NULL,
 		       NULL, 0, NULL, 0,
 		       CTL_MACHDEP, CTL_EOL);
 	sysctl_createv(clog, 0, NULL, NULL,
 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
 		       CTLTYPE_INT, "debug", NULL,
 		       NULL, 0, &kernel_debug, 0,
 		       CTL_MACHDEP, CPU_DEBUG, CTL_EOL);
 	sysctl_createv(clog, 0, NULL, NULL,
 		       CTLFLAG_PERMANENT,
 		       CTLTYPE_STRING, "booted_device", NULL,
 		       sysctl_machdep_booted_device, 0, NULL, 0,
 		       CTL_MACHDEP, CPU_BOOTED_DEVICE, CTL_EOL);
 	sysctl_createv(clog, 0, NULL, NULL,
 		       CTLFLAG_PERMANENT,
 		       CTLTYPE_STRING, "booted_kernel", NULL,
 		       sysctl_machdep_booted_kernel, 0, NULL, 0,
 		       CTL_MACHDEP, CPU_BOOTED_KERNEL, CTL_EOL);
 	sysctl_createv(clog, 0, NULL, NULL,
 		       CTLFLAG_PERMANENT,
 		       CTLTYPE_STRUCT, "console_device", NULL,
 		       sysctl_consdev, 0, NULL, sizeof(dev_t),
 		       CTL_MACHDEP, CPU_CONSDEV, CTL_EOL);
 	sysctl_createv(clog, 0, NULL, NULL,
 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
 		       CTLTYPE_INT, "powersave", NULL,
 		       sysctl_machdep_powersave, 0, &cpu_do_powersave, 0,
 		       CTL_MACHDEP, CPU_POWERSAVE, CTL_EOL);
+}
 void
 parse_mi_bootargs(char *args)
+{
 	int integer;
 	if (get_bootconf_option(args, "single", BOOTOPT_TYPE_BOOLEAN, &integer)
 	    || get_bootconf_option(args, "-s", BOOTOPT_TYPE_BOOLEAN, &integer))
 		if (integer)
 			boothowto |= RB_SINGLE;
 	if (get_bootconf_option(args, "kdb", BOOTOPT_TYPE_BOOLEAN, &integer)
 	    || get_bootconf_option(args, "-k", BOOTOPT_TYPE_BOOLEAN, &integer))
 		if (integer)
 			boothowto |= RB_KDB;
 	if (get_bootconf_option(args, "ask", BOOTOPT_TYPE_BOOLEAN, &integer)
 	    || get_bootconf_option(args, "-a", BOOTOPT_TYPE_BOOLEAN, &integer))
 		if (integer)
 			boothowto |= RB_ASKNAME;
 #ifdef PMAP_DEBUG
 	if (get_bootconf_option(args, "pmapdebug", BOOTOPT_TYPE_INT, &integer)) {
 		pmap_debug_level = integer;
 		pmap_debug(pmap_debug_level);
+	}
 #endif	/* PMAP_DEBUG */
 /*	if (get_bootconf_option(args, "nbuf", BOOTOPT_TYPE_INT, &integer))
 		bufpages = integer;*/
 #if defined(MEMORY_DISK_HOOKS) && !defined(MEMORY_DISK_ROOT_SIZE)
 	if (get_bootconf_option(args, "memorydisc", BOOTOPT_TYPE_INT, &integer)
 	    || get_bootconf_option(args, "memorydisk", BOOTOPT_TYPE_INT, &integer)) {
 		md_root_size = integer;
 		md_root_size *= 1024;
 		if (md_root_size < 32*1024)
 			md_root_size = 32*1024;
 		if (md_root_size > 2048*1024)
 			md_root_size = 2048*1024;
+	}
 #endif	/* MEMORY_DISK_HOOKS && !MEMORY_DISK_ROOT_SIZE */
 	if (get_bootconf_option(args, "quiet", BOOTOPT_TYPE_BOOLEAN, &integer)
 	    || get_bootconf_option(args, "-q", BOOTOPT_TYPE_BOOLEAN, &integer))
 		if (integer)
 			boothowto |= AB_QUIET;
 	if (get_bootconf_option(args, "verbose", BOOTOPT_TYPE_BOOLEAN, &integer)
 	    || get_bootconf_option(args, "-v", BOOTOPT_TYPE_BOOLEAN, &integer))
 		if (integer)
 			boothowto |= AB_VERBOSE;
+}
 #ifdef __HAVE_FAST_SOFTINTS
 #if IPL_SOFTSERIAL != IPL_SOFTNET + 1
 #error IPLs are screwed up
 #elif IPL_SOFTNET != IPL_SOFTBIO + 1
 #error IPLs are screwed up
 #elif IPL_SOFTBIO != IPL_SOFTCLOCK + 1
 #error IPLs are screwed up
 #elif !(IPL_SOFTCLOCK > IPL_NONE)
 #error IPLs are screwed up
 #elif (IPL_NONE != 0)
 #error IPLs are screwed up
 #endif
 #ifndef __HAVE_PIC_FAST_SOFTINTS
 #define	SOFTINT2IPLMAP \
 	(((IPL_SOFTSERIAL - IPL_SOFTCLOCK) << (SOFTINT_SERIAL * 4)) | \
 	 ((IPL_SOFTNET    - IPL_SOFTCLOCK) << (SOFTINT_NET    * 4)) | \
 	 ((IPL_SOFTBIO    - IPL_SOFTCLOCK) << (SOFTINT_BIO    * 4)) | \
 	 ((IPL_SOFTCLOCK  - IPL_SOFTCLOCK) << (SOFTINT_CLOCK  * 4)))
 #define	SOFTINT2IPL(l)	((SOFTINT2IPLMAP >> ((l) * 4)) & 0x0f)
 /*
  * This returns a mask of softint IPLs that be dispatch at <ipl>
  * SOFTIPLMASK(IPL_NONE)	= 0x0000000f
  * SOFTIPLMASK(IPL_SOFTCLOCK)	= 0x0000000e
  * SOFTIPLMASK(IPL_SOFTBIO)	= 0x0000000c
  * SOFTIPLMASK(IPL_SOFTNET)	= 0x00000008
  * SOFTIPLMASK(IPL_SOFTSERIAL)	= 0x00000000
  */
 #define	SOFTIPLMASK(ipl) ((0x0f << (ipl)) & 0x0f)
 void softint_switch(lwp_t *, int);
 void
 softint_trigger(uintptr_t mask)
+{
 	curcpu()->ci_softints |= mask;
+}
 void
 softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep)
+{
 	lwp_t ** lp = &l->l_cpu->ci_softlwps[level];
 	KASSERT(*lp == NULL || *lp == l);
 	*lp = l;
 	*machdep = 1 << SOFTINT2IPL(level);
 	KASSERT(level != SOFTINT_CLOCK || *machdep == (1 << (IPL_SOFTCLOCK - IPL_SOFTCLOCK)));
 	KASSERT(level != SOFTINT_BIO || *machdep == (1 << (IPL_SOFTBIO - IPL_SOFTCLOCK)));
 	KASSERT(level != SOFTINT_NET || *machdep == (1 << (IPL_SOFTNET - IPL_SOFTCLOCK)));
 	KASSERT(level != SOFTINT_SERIAL || *machdep == (1 << (IPL_SOFTSERIAL - IPL_SOFTCLOCK)));
+}
 void
 dosoftints(void)
+{
 	struct cpu_info * const ci = curcpu();
 	const int opl = ci->ci_cpl;
 	const uint32_t softiplmask = SOFTIPLMASK(opl);
 	splhigh();
 	for (;;) {
 		u_int softints = ci->ci_softints & softiplmask;
 		KASSERT((softints != 0) == ((ci->ci_softints >> opl) != 0));
 		KASSERT(opl == IPL_NONE || (softints & (1 << (opl - IPL_SOFTCLOCK))) == 0);
 		if (softints == 0) {
 			splx(opl);
 			return;
+		}
 #define	DOSOFTINT(n) \
 		if (ci->ci_softints & (1 << (IPL_SOFT ## n - IPL_SOFTCLOCK))) { \
 			ci->ci_softints &= \
 			    ~(1 << (IPL_SOFT ## n - IPL_SOFTCLOCK)); \
 			softint_switch(ci->ci_softlwps[SOFTINT_ ## n], \
 			    IPL_SOFT ## n); \
 			continue; \
+		}
 		DOSOFTINT(SERIAL);
 		DOSOFTINT(NET);
 		DOSOFTINT(BIO);
 		DOSOFTINT(CLOCK);
 		panic("dosoftints wtf (softints=%u?, ipl=%d)", softints, opl);
+	}
+}
 #endif /* !__HAVE_PIC_FAST_SOFTINTS */
 #endif /* __HAVE_FAST_SOFTINTS */
 #ifdef MODULAR
 /*
  * Push any modules loaded by the boot loader.
  */
 void
 module_init_md(void)
+{
+}
 #endif /* MODULAR */
 int
 mm_md_physacc(paddr_t pa, vm_prot_t prot)
+{
 	return (pa < ctob(physmem)) ? 0 : EFAULT;
+}
 #ifdef __HAVE_CPU_UAREA_ALLOC_IDLELWP
 vaddr_t
 cpu_uarea_alloc_idlelwp(struct cpu_info *ci)
+{
 	const vaddr_t va = idlestack.pv_va + ci->ci_cpuid * USPACE;
 	// printf("%s: %s: va=%lx\n", __func__, ci->ci_data.cpu_name, va);
 	return va;
+}
 #endif
 #ifdef MULTIPROCESSOR
 void
 cpu_boot_secondary_processors(void)
+{
 	uint32_t mbox;
-	kcpuset_copybits(kcpuset_attached, &mbox, sizeof(mbox));
+	kcpuset_export_u32(kcpuset_attached, &mbox, sizeof(mbox));
 	atomic_swap_32(&arm_cpu_mbox, mbox);
 	membar_producer();
 #ifdef _ARM_ARCH_7
 	__asm __volatile("sev; sev; sev");
 #endif
+}
 void
 xc_send_ipi(struct cpu_info *ci)
+{
 	KASSERT(kpreempt_disabled());
 	KASSERT(curcpu() != ci);
 	if (ci) {
 		/* Unicast, remote CPU */
 		printf("%s: -> %s", __func__, ci->ci_data.cpu_name);
 		intr_ipi_send(ci->ci_kcpuset, IPI_XCALL);
 	} else {
 		printf("%s: -> !%s", __func__, ci->ci_data.cpu_name);
 		/* Broadcast to all but ourselves */
 		kcpuset_t *kcp;
 		kcpuset_create(&kcp, (ci != NULL));
 		KASSERT(kcp != NULL);
 		kcpuset_copy(kcp, kcpuset_running);
 		kcpuset_clear(kcp, cpu_index(ci));
 		intr_ipi_send(kcp, IPI_XCALL);
 		kcpuset_destroy(kcp);
+	}
 	printf("\n");
+}
 #endif /* MULTIPROCESSOR */

 @@ -1,577 +1,577 @@
-/*	$NetBSD: gic.c,v 1.2 2012/09/14 03:52:50 matt Exp $	*/
+/*	$NetBSD: gic.c,v 1.3 2012/09/16 22:09:34 rmind Exp $	*/
 /*-
  * Copyright (c) 2012 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Matt Thomas of 3am Software Foundry.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #define _INTR_PRIVATE
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: gic.c,v 1.2 2012/09/14 03:52:50 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: gic.c,v 1.3 2012/09/16 22:09:34 rmind Exp $");
 #include <sys/param.h>
 #include <sys/bus.h>
 #include <sys/device.h>
 #include <sys/evcnt.h>
 #include <sys/intr.h>
 #include <sys/proc.h>
 #include <sys/xcall.h>		/* for xc_ipi_handler */
 #include <arm/armreg.h>
 #include <arm/cpufunc.h>
 #include <arm/atomic.h>
 #include <arm/cortex/gic_reg.h>
 #include <arm/cortex/mpcore_var.h>
 #define	ARMGIC_SGI_IPIBASE	(16 - NIPI)
 static int armgic_match(device_t, cfdata_t, void *);
 static void armgic_attach(device_t, device_t, void *);
 static void armgic_set_priority(struct pic_softc *, int);
 static void armgic_unblock_irqs(struct pic_softc *, size_t, uint32_t);
 static void armgic_block_irqs(struct pic_softc *, size_t, uint32_t);
 static void armgic_establish_irq(struct pic_softc *, struct intrsource *);
 #if 0
 static void armgic_source_name(struct pic_softc *, int, char *, size_t);
 #endif
 #ifdef MULTIPROCESSOR
 static void armgic_cpu_init(struct pic_softc *, struct cpu_info *);
 static void armgic_ipi_send(struct pic_softc *, const kcpuset_t *, u_long);
 #endif
 static const struct pic_ops armgic_picops = {
 	.pic_unblock_irqs = armgic_unblock_irqs,
 	.pic_block_irqs = armgic_block_irqs,
 	.pic_establish_irq = armgic_establish_irq,
 #if 0
 	.pic_source_name = armgic_source_name,
 #endif
 	.pic_set_priority = armgic_set_priority,
 #ifdef MULTIPROCESSOR
 	.pic_cpu_init = armgic_cpu_init,
 	.pic_ipi_send = armgic_ipi_send,
 #endif
 };
 #define	PICTOSOFTC(pic)		((struct armgic_softc *)(pic))
 static struct armgic_softc {
 	struct pic_softc sc_pic;
 	device_t sc_dev;
 	bus_space_tag_t sc_memt;
 	bus_space_handle_t sc_memh;
 	size_t sc_gic_lines;
 	uint32_t sc_gic_type;
 	uint32_t sc_gic_valid_lines[1024/32];
 	uint32_t sc_enabled_local;
 } armgic_softc = {
 	.sc_pic = {
 		.pic_ops = &armgic_picops,
 		.pic_name = "armgic",
 	},
 };
 static struct intrsource armgic_dummy_source;
 __CTASSERT(NIPL == 8);
 /*
  * GIC register are always in little-endian.
  */
 static inline uint32_t
 gicc_read(struct armgic_softc *sc, bus_size_t o)
+{
 	uint32_t v = bus_space_read_4(sc->sc_memt, sc->sc_memh, GICC_BASE + o);
 	return le32toh(v);
+}
 static inline void
 gicc_write(struct armgic_softc *sc, bus_size_t o, uint32_t v)
+{
 	v = htole32(v);
 	bus_space_write_4(sc->sc_memt, sc->sc_memh, GICC_BASE + o, v);
+}
 static inline uint32_t
 gicd_read(struct armgic_softc *sc, bus_size_t o)
+{
 	uint32_t v = bus_space_read_4(sc->sc_memt, sc->sc_memh, GICD_BASE + o);
 	return le32toh(v);
+}
 static inline void
 gicd_write(struct armgic_softc *sc, bus_size_t o, uint32_t v)
+{
 	v = htole32(v);
 	bus_space_write_4(sc->sc_memt, sc->sc_memh, GICD_BASE + o, v);
+}
 /*
  * In the GIC prioritization scheme, lower numbers have higher priority.
  */
 static inline uint32_t
 armgic_ipl_to_priority(int ipl)
+{
 	return (IPL_HIGH - ipl) * GICC_PMR_PRIORITIES / NIPL;
+}
 static inline int
 armgic_priority_to_ipl(uint32_t priority)
+{
 	return IPL_HIGH - priority * NIPL / GICC_PMR_PRIORITIES;
+}
 static void
 armgic_unblock_irqs(struct pic_softc *pic, size_t irq_base, uint32_t irq_mask)
+{
 	struct armgic_softc * const sc = PICTOSOFTC(pic);
 	const size_t group = irq_base / 32;
 	if (group == 0)
 		sc->sc_enabled_local |= irq_mask;
 	gicd_write(sc, GICD_ISENABLERn(group), irq_mask);
+}
 static void
 armgic_block_irqs(struct pic_softc *pic, size_t irq_base, uint32_t irq_mask)
+{
 	struct armgic_softc * const sc = PICTOSOFTC(pic);
 	const size_t group = irq_base / 32;
 	if (group == 0)
 		sc->sc_enabled_local &= ~irq_mask;
 	gicd_write(sc, GICD_ICENABLERn(group), irq_mask);
+}
 static uint32_t armgic_last_priority;
 static void
 armgic_set_priority(struct pic_softc *pic, int ipl)
+{
 	struct armgic_softc * const sc = PICTOSOFTC(pic);
 	const uint32_t priority = armgic_ipl_to_priority(ipl);
 	gicc_write(sc, GICC_PMR, priority);
 	armgic_last_priority = priority;
+}
 #ifdef __HAVE_PIC_FAST_SOFTINTS
 void
 softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep_p)
+{
 	lwp_t **lp = &l->l_cpu->ci_softlwps[level];
 	KASSERT(*lp == NULL || *lp == l);
 	*lp = l;
 	/*
 	 * Really easy.  Just tell it to trigger the local CPU.
 	 */
 	*machdep_p = GICD_SGIR_TargetListFilter_Me
 	    | __SHIFTIN(level, GICD_SGIR_SGIINTID);
+}
 void
 softint_trigger(uintptr_t machdep)
+{
 	gicd_write(&armgic_softc, GICD_SGIR, machdep);
+}
 #endif
 void
 armgic_irq_handler(void *tf)
+{
 	struct cpu_info * const ci = curcpu();
 	struct armgic_softc * const sc = &armgic_softc;
 	const int old_ipl = ci->ci_cpl;
 #ifdef DIAGNOSTIC
 	const int old_mtx_count = ci->ci_mtx_count;
 	const int old_l_biglocks = ci->ci_curlwp->l_biglocks;
 #endif
 #ifdef DEBUG
 	size_t n = 0;
 #endif
 	ci->ci_data.cpu_nintr++;
 	KASSERTMSG(old_ipl != IPL_HIGH, "old_ipl %d pmr %#x hppir %#x",
 	    old_ipl, gicc_read(sc, GICC_PMR), gicc_read(sc, GICC_HPPIR));
 #if 0
 	printf("%s(enter): %s: pmr=%u hppir=%u\n",
 	    __func__, ci->ci_data.cpu_name,
 	    gicc_read(sc, GICC_PMR),
 	    gicc_read(sc, GICC_HPPIR));
 #elif 0
 	printf("(%u:%d", ci->ci_index, old_ipl);
 #endif
 	for (;;) {
 		uint32_t iar = gicc_read(sc, GICC_IAR);
 		uint32_t irq = __SHIFTOUT(iar, GICC_IAR_IRQ);
 		//printf(".%u", irq);
 		if (irq == GICC_IAR_IRQ_SPURIOUS) {
 			iar = gicc_read(sc, GICC_IAR);
 			irq = __SHIFTOUT(iar, GICC_IAR_IRQ);
 			if (irq == GICC_IAR_IRQ_SPURIOUS)
 				break;
 			//printf(".%u", irq);
+		}
 		//const uint32_t cpuid = __SHIFTOUT(iar, GICC_IAR_CPUID_MASK);
 		struct intrsource * const is = sc->sc_pic.pic_sources[irq];
 		KASSERT(is != &armgic_dummy_source);
 		/*
 		 * GIC has asserted IPL for us so we can just update ci_cpl.
+		 *
 		 * But it's not that simple.  We may have already bumped ci_cpl
 		 * due to a high priority interrupt and now we are about to
 		 * dispatch one lower than the previous.  It's possible for
 		 * that previous interrupt to have deferred some interrupts
 		 * so we need deal with those when lowering to the current
 		 * interrupt's ipl.
+		 *
 		 * However, if are just raising ipl, we can just update ci_cpl.
 		 */
 #if 0
 		const int ipl = armgic_priority_to_ipl(gicc_read(sc, GICC_RPR));
 		KASSERTMSG(panicstr != NULL || ipl == is->is_ipl,
 		    "%s: irq %d: running ipl %d != source ipl %u",
 		    ci->ci_data.cpu_name, irq, ipl, is->is_ipl);
 #else
 		const int ipl = is->is_ipl;
 #endif
 		if (__predict_false(ipl < ci->ci_cpl)) {
 			//printf("<");
 			pic_do_pending_ints(I32_bit, ipl, tf);
 			KASSERT(ci->ci_cpl == ipl);
 		} else {
 			KASSERTMSG(ipl > ci->ci_cpl, "ipl %d cpl %d hw-ipl %#x",
 			    ipl, ci->ci_cpl,
 			    gicc_read(sc, GICC_PMR));
 			//printf(">");
 			gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(ipl));
 			ci->ci_cpl = ipl;
+		}
 		//printf("$");
 		cpsie(I32_bit);
 		pic_dispatch(is, tf);
 		cpsid(I32_bit);
 		gicc_write(sc, GICC_EOIR, iar);
 #ifdef DEBUG
 		n++;
 		KDASSERTMSG(n < 5, "%s: processed too many (%zu)",
 		    ci->ci_data.cpu_name, n);
 #endif
+	}
 	// printf("%s(%p): exit (%zu dispatched)\n", __func__, tf, n);
 	/*
 	 * Now handle any pending ints.
 	 */
 	//printf("!");
 	KASSERT(old_ipl != IPL_HIGH);
 	pic_do_pending_ints(I32_bit, old_ipl, tf);
 	KASSERTMSG(ci->ci_cpl == old_ipl, "ci_cpl %d old_ipl %d", ci->ci_cpl, old_ipl);
 	KASSERT(old_mtx_count == ci->ci_mtx_count);
 	KASSERT(old_l_biglocks == ci->ci_curlwp->l_biglocks);
 #if 0
 	printf("%s(exit): %s(%d): pmr=%u hppir=%u\n",
 	    __func__, ci->ci_data.cpu_name, ci->ci_cpl,
 	    gicc_read(sc, GICC_PMR),
 	    gicc_read(sc, GICC_HPPIR));
 #elif 0
 	printf("->%#x)", ((struct trapframe *)tf)->tf_pc);
 #endif
+}
 void
 armgic_establish_irq(struct pic_softc *pic, struct intrsource *is)
+{
 	struct armgic_softc * const sc = PICTOSOFTC(pic);
 	const size_t group = is->is_irq / 32;
 	const u_int irq = is->is_irq & 31;
 	const u_int byte_shift = 8 * (irq & 3);
 	const u_int twopair_shift = 2 * (irq & 15);
 	KASSERTMSG(sc->sc_gic_valid_lines[group] & __BIT(irq),
 	    "irq %u: not valid (group[%zu]=0x%08x [0x%08x])",
 	    is->is_irq, group, sc->sc_gic_valid_lines[group],
 	    (uint32_t)__BIT(irq));
 	KASSERTMSG(is->is_type == IST_LEVEL || is->is_type == IST_EDGE,
 	    "irq %u: type %u unsupported", is->is_irq, is->is_type);
 	const bus_size_t targets_reg = GICD_ITARGETSRn(is->is_irq / 4);
 	const bus_size_t cfg_reg = GICD_ICFGRn(is->is_irq / 16);
 	uint32_t targets = gicd_read(sc, targets_reg);
 	uint32_t cfg = gicd_read(sc, cfg_reg);
 	if (group > 0) {
 		/*
 		 * There are 4 irqs per TARGETS register.  For now bind
 		 * to the primary cpu.
 		 */
 		targets &= ~(0xff << byte_shift);
 		targets |= 1 << byte_shift;
 		gicd_write(sc, targets_reg, targets);
 		/*
 		 * There are 16 irqs per CFG register.  10=EDGE 00=LEVEL
 		 */
 		uint32_t new_cfg = cfg;
 		uint32_t old_cfg = (cfg >> twopair_shift) & 3;
 		if (is->is_type == IST_LEVEL && (old_cfg & 2) != 0) {
 			new_cfg &= ~(3 << twopair_shift);
 		} else if (is->is_type == IST_EDGE && (old_cfg & 2) == 0) {
 			new_cfg |= 2 << twopair_shift;
+		}
 		if (new_cfg != cfg) {
 			gicd_write(sc, cfg_reg, cfg);
 #if 0
 			printf("%s: irq %u: cfg changed from %#x to %#x\n",
 			    pic->pic_name, is->is_irq, cfg, new_cfg);
 #endif
+		}
+	}
 	/*
 	 * There are 4 irqs per PRIORITY register.  Map the IPL
 	 * to GIC priority.
 	 */
 	const bus_size_t priority_reg = GICD_IPRIORITYRn(is->is_irq / 4);
 	uint32_t priority = gicd_read(sc, priority_reg);
 	priority &= ~(0xff << byte_shift);
 	priority |= armgic_ipl_to_priority(is->is_ipl) << byte_shift;
 	gicd_write(sc, priority_reg, priority);
 #if 0
 	printf("%s: irq %u: target %#x cfg %u priority %#x (%u)\n",
 	    pic->pic_name, is->is_irq, (targets >> byte_shift) & 0xff,
 	    (cfg >> twopair_shift) & 3, (priority >> byte_shift) & 0xff,
 	    is->is_ipl);
 #endif
+}
 #ifdef MULTIPROCESSOR
 static void
 armgic_cpu_init_priorities(struct armgic_softc *sc)
+{
 	uint32_t enabled = sc->sc_enabled_local;
 	for (size_t i = 0; i < 32; i += 4, enabled >>= 4) {
 		/*
 		 * If there are no enabled interrupts for the priority register,
 		 * don't bother changing it.
 		 */
 		if ((enabled & 0x0f) == 0)
 			continue;
 		/*
 		 * Since priorities are in 3210 order, it'
 		 */
 		const bus_size_t priority_reg = GICD_IPRIORITYRn(i / 4);
 		uint32_t priority = gicd_read(sc, priority_reg);
 		uint32_t byte_mask = 0xff;
 		size_t byte_shift = 0;
 		for (size_t j = 0; j < 4; j++, byte_mask <<= 8, byte_shift += 8) {
 			struct intrsource * const is = sc->sc_pic.pic_sources[i+j];
 			if (is == NULL || is == &armgic_dummy_source)
 				continue;
 			priority &= ~byte_mask;
 			priority |= armgic_ipl_to_priority(is->is_ipl) << byte_shift;
+		}
 		gicd_write(sc, priority_reg, priority);
+	}
+}
 void
 armgic_cpu_init(struct pic_softc *pic, struct cpu_info *ci)
+{
 	struct armgic_softc * const sc = PICTOSOFTC(pic);
 	if (!CPU_IS_PRIMARY(ci) && sc->sc_enabled_local) {
 		armgic_cpu_init_priorities(sc);
+	}
 	KASSERTMSG(ci->ci_cpl == IPL_HIGH, "ipl %d not IPL_HIGH", ci->ci_cpl);
 	gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(ci->ci_cpl));	// set PMR
 	gicc_write(sc, GICC_CTRL, GICC_CTRL_V1_Enable);	// enable interrupt
 	if (!CPU_IS_PRIMARY(ci) && sc->sc_enabled_local)
 		gicd_write(sc, GICD_ISENABLERn(0), sc->sc_enabled_local);
 	cpsie(I32_bit);					// allow IRQ exceptions
+}
 void
 armgic_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi)
+{
 	struct armgic_softc * const sc = PICTOSOFTC(pic);
 	if (ipi == IPI_NOP) {
 		__asm __volatile("sev");
 		return;
+	}
 	uint32_t targets;
-	kcpuset_copybits(kcp, &targets, sizeof(targets));
+	kcpuset_export_u32(kcp, &targets, sizeof(targets));
 	uint32_t sgir = __SHIFTOUT(ARMGIC_SGI_IPIBASE + ipi, GICD_SGIR_SGIINTID);
 	sgir |= __SHIFTOUT(targets, GICD_SGIR_TargetList);
 	printf("%s: %s: %#x", __func__, curcpu()->ci_data.cpu_name, sgir);
 	gicd_write(sc, GICD_SGIR, sgir);
 	printf("\n");
+}
 #endif
 int
 armgic_match(device_t parent, cfdata_t cf, void *aux)
+{
 	struct mpcore_attach_args * const mpcaa = aux;
 	if (strcmp(cf->cf_name, mpcaa->mpcaa_name) != 0)
 		return 0;
 	if (!CPU_ID_CORTEX_P(cputype))
 		return 0;
 	if (CPU_ID_CORTEX_A8_P(cputype))
 		return 0;
 	return 1;
+}
 void
 armgic_attach(device_t parent, device_t self, void *aux)
+{
 	struct armgic_softc * const sc = &armgic_softc;
 	struct mpcore_attach_args * const mpcaa = aux;
 	sc->sc_dev = self;
 	self->dv_private = sc;
 	sc->sc_memt = mpcaa->mpcaa_memt;	/* provided for us */
 	sc->sc_memh = mpcaa->mpcaa_memh;	/* provided for us */
 	sc->sc_gic_type = gicd_read(sc, GICD_TYPER);
 	sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(sc->sc_gic_type);
 	gicc_write(sc, GICC_CTRL, 0);	/* disable all interrupts */
 	gicd_write(sc, GICD_CTRL, 0);	/* disable all interrupts */
 	gicc_write(sc, GICC_PMR, 0xff);
 	uint32_t pmr = gicc_read(sc, GICC_PMR);
 	u_int priorities = 1 << popcount32(pmr);
 	/*
 	 * Let's find out how many real sources we have.
 	 */
 	for (size_t i = 0, group = 0;
 	     i < sc->sc_pic.pic_maxsources;
 	     i += 32, group++) {
 		/*
 		 * To figure what sources are real, one enables all interrupts
 		 * and then reads back the enable mask so which ones really
 		 * got enabled.
 		 */
 		gicd_write(sc, GICD_ISENABLERn(group), 0xffffffff);
 		uint32_t valid = gicd_read(sc, GICD_ISENABLERn(group));
 		/*
 		 * Now disable (clear enable) them again.
 		 */
 		gicd_write(sc, GICD_ICENABLERn(group), valid);
 		/*
 		 * Count how many are valid.
 		 */
 		sc->sc_gic_lines += popcount32(valid);
 		sc->sc_gic_valid_lines[group] = valid;
+	}
 	pic_add(&sc->sc_pic, 0);
 	/*
 	 * Force the GICD to IPL_HIGH and then enable interrupts.
 	 */
 	struct cpu_info * const ci = curcpu();
 	KASSERTMSG(ci->ci_cpl == IPL_HIGH, "ipl %d not IPL_HIGH", ci->ci_cpl);
 	armgic_set_priority(&sc->sc_pic, ci->ci_cpl);	// set PMR
 	gicd_write(sc, GICD_CTRL, GICD_CTRL_Enable);	// enable Distributer
 	gicc_write(sc, GICC_CTRL, GICC_CTRL_V1_Enable);	// enable CPU interrupts
 	cpsie(I32_bit);					// allow interrupt exceptions
 	/*
 	 * For each line that isn't valid, we set the intrsource for it to
 	 * point at a dummy source so that pic_intr_establish will fail for it.
 	 */
 	for (size_t i = 0, group = 0;
 	     i < sc->sc_pic.pic_maxsources;
 	     i += 32, group++) {
 		uint32_t invalid = ~sc->sc_gic_valid_lines[group];
 		for (size_t j = 0; invalid && j < 32; j++, invalid >>= 1) {
 			if (invalid & 1) {
 				sc->sc_pic.pic_sources[i + j] =
 				     &armgic_dummy_source;
+			}
+		}
+	}
 #ifdef __HAVE_PIC_FAST_SOFTINTS
 	intr_establish(SOFTINT_BIO, IPL_SOFTBIO, IST_EDGE,
 	    pic_handle_softint, (void *)SOFTINT_BIO);
 	intr_establish(SOFTINT_CLOCK, IPL_SOFTCLOCK, IST_EDGE,
 	    pic_handle_softint, (void *)SOFTINT_CLOCK);
 	intr_establish(SOFTINT_NET, IPL_SOFTNET, IST_EDGE,
 	    pic_handle_softint, (void *)SOFTINT_NET);
 	intr_establish(SOFTINT_SERIAL, IPL_SOFTSERIAL, IST_EDGE,
 	    pic_handle_softint, (void *)SOFTINT_SERIAL);
 #endif
 #ifdef MULTIPROCESSOR
 	intr_establish(ARMGIC_SGI_IPIBASE + IPI_AST, IPL_VM, IST_EDGE,
 	    pic_ipi_nop, (void *)-1);
 	intr_establish(ARMGIC_SGI_IPIBASE + IPI_XCALL, IPL_VM, IST_EDGE,
 	    pic_ipi_xcall, (void *)-1);
 #if 0	/* Not needed */
 	intr_establish(ARMGIC_SGI_IPIBASE + IPI_NOP, IPL_VM, IST_EDGE,
 	    pic_ipi_nop, (void *)-1);
 #endif
 #ifdef __HAVE_PREEMPTION
 	intr_establish(ARMGIC_SGI_IPIBASE + IPI_KPREEMPT, IPL_VM, IST_EDGE,
 	    pic_ipi_nop, (void *)-1);
 #endif
 	armgic_cpu_init(&sc->sc_pic, curcpu());
 #endif
 	aprint_normal(": Generic Interrupt Controller, "
 	    "%zu sources (%zu valid)\n",
 	    sc->sc_pic.pic_maxsources, sc->sc_gic_lines);
 	const u_int ppis = popcount32(sc->sc_gic_valid_lines[0] >> 16);
 	const u_int sgis = popcount32(sc->sc_gic_valid_lines[0] & 0xffff);
 	aprint_normal_dev(sc->sc_dev, "%u Priorities, %zu SPIs, %u PPIs, %u SGIs\n",
 	    priorities, sc->sc_gic_lines - ppis - sgis, ppis, sgis);
+}
 CFATTACH_DECL_NEW(armgic, 0,
     armgic_match, armgic_attach, NULL, NULL);

 @@ -1,1114 +1,1114 @@
-/*	$NetBSD: x86_xpmap.c,v 1.48 2012/08/21 09:06:02 bouyer Exp $	*/
+/*	$NetBSD: x86_xpmap.c,v 1.49 2012/09/16 22:09:34 rmind Exp $	*/
 /*
  * Copyright (c) 2006 Mathieu Ropert <mro@adviseo.fr>
+ *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
  * copyright notice and this permission notice appear in all copies.
+ *
  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 /*
  * Copyright (c) 2006, 2007 Manuel Bouyer.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
  */
 /*
+ *
  * Copyright (c) 2004 Christian Limpach.
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: x86_xpmap.c,v 1.48 2012/08/21 09:06:02 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: x86_xpmap.c,v 1.49 2012/09/16 22:09:34 rmind Exp $");
 #include "opt_xen.h"
 #include "opt_ddb.h"
 #include "ksyms.h"
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/mutex.h>
 #include <sys/cpu.h>
 #include <uvm/uvm.h>
 #include <x86/pmap.h>
 #include <machine/gdt.h>
 #include <xen/xenfunc.h>
 #include <dev/isa/isareg.h>
 #include <machine/isa_machdep.h>
 #undef	XENDEBUG
 /* #define XENDEBUG_SYNC */
 /* #define	XENDEBUG_LOW */
 #ifdef XENDEBUG
 #define	XENPRINTF(x) printf x
 #define	XENPRINTK(x) printk x
 #define	XENPRINTK2(x) /* printk x */
 static char XBUF[256];
 #else
 #define	XENPRINTF(x)
 #define	XENPRINTK(x)
 #define	XENPRINTK2(x)
 #endif
 #define	PRINTF(x) printf x
 #define	PRINTK(x) printk x
 volatile shared_info_t *HYPERVISOR_shared_info;
 /* Xen requires the start_info struct to be page aligned */
 union start_info_union start_info_union __aligned(PAGE_SIZE);
 unsigned long *xpmap_phys_to_machine_mapping;
 kmutex_t pte_lock;
 void xen_failsafe_handler(void);
 #define HYPERVISOR_mmu_update_self(req, count, success_count) \
 	HYPERVISOR_mmu_update((req), (count), (success_count), DOMID_SELF)
 /*
  * kcpuset internally uses an array of uint32_t while xen uses an array of
  * u_long. As we're little-endian we can cast one to the other.
  */
 typedef union {
 #ifdef _LP64
 	uint32_t xcpum_km[2];
 #else
 	uint32_t xcpum_km[1];
 #endif
 	u_long   xcpum_xm;
 } xcpumask_t;
 void
 xen_failsafe_handler(void)
+{
 	panic("xen_failsafe_handler called!\n");
+}
 void
 xen_set_ldt(vaddr_t base, uint32_t entries)
+{
 	vaddr_t va;
 	vaddr_t end;
 	pt_entry_t *ptp;
 	int s;
 #ifdef __x86_64__
 	end = base + (entries << 3);
 #else
 	end = base + entries * sizeof(union descriptor);
 #endif
 	for (va = base; va < end; va += PAGE_SIZE) {
 		KASSERT(va >= VM_MIN_KERNEL_ADDRESS);
 		ptp = kvtopte(va);
 		XENPRINTF(("xen_set_ldt %#" PRIxVADDR " %d %p\n",
 		    base, entries, ptp));
 		pmap_pte_clearbits(ptp, PG_RW);
+	}
 	s = splvm();
 	xpq_queue_set_ldt(base, entries);
 	splx(s);
+}
 #ifdef XENDEBUG
 void xpq_debug_dump(void);
 #endif
 #define XPQUEUE_SIZE 2048
 static mmu_update_t xpq_queue_array[MAXCPUS][XPQUEUE_SIZE];
 static int xpq_idx_array[MAXCPUS];
 #ifdef i386
 extern union descriptor tmpgdt[];
 #endif /* i386 */
 void
 xpq_flush_queue(void)
+{
 	int i, ok = 0, ret;
 	mmu_update_t *xpq_queue = xpq_queue_array[curcpu()->ci_cpuid];
 	int xpq_idx = xpq_idx_array[curcpu()->ci_cpuid];
 	XENPRINTK2(("flush queue %p entries %d\n", xpq_queue, xpq_idx));
 	for (i = 0; i < xpq_idx; i++)
 		XENPRINTK2(("%d: 0x%08" PRIx64 " 0x%08" PRIx64 "\n", i,
 		    xpq_queue[i].ptr, xpq_queue[i].val));
 retry:
 	ret = HYPERVISOR_mmu_update_self(xpq_queue, xpq_idx, &ok);
 	if (xpq_idx != 0 && ret < 0) {
 		struct cpu_info *ci;
 		CPU_INFO_ITERATOR cii;
 		printf("xpq_flush_queue: %d entries (%d successful) on "
 		    "cpu%d (%ld)\n",
 		    xpq_idx, ok, curcpu()->ci_index, curcpu()->ci_cpuid);
 		if (ok != 0) {
 			xpq_queue += ok;
 			xpq_idx -= ok;
 			ok = 0;
 			goto retry;
+		}
 		for (CPU_INFO_FOREACH(cii, ci)) {
 			xpq_queue = xpq_queue_array[ci->ci_cpuid];
 			xpq_idx = xpq_idx_array[ci->ci_cpuid];
 			printf("cpu%d (%ld):\n", ci->ci_index, ci->ci_cpuid);
 			for (i = 0; i < xpq_idx; i++) {
 				printf("  0x%016" PRIx64 ": 0x%016" PRIx64 "\n",
 				   xpq_queue[i].ptr, xpq_queue[i].val);
+			}
 #ifdef __x86_64__
 			for (i = 0; i < PDIR_SLOT_PTE; i++) {
 				if (ci->ci_kpm_pdir[i] == 0)
 					continue;
 				printf(" kpm_pdir[%d]: 0x%" PRIx64 "\n",
 				    i, ci->ci_kpm_pdir[i]);
+			}
 #endif
+		}
 		panic("HYPERVISOR_mmu_update failed, ret: %d\n", ret);
+	}
 	xpq_idx_array[curcpu()->ci_cpuid] = 0;
+}
 static inline void
 xpq_increment_idx(void)
+{
 	if (__predict_false(++xpq_idx_array[curcpu()->ci_cpuid] == XPQUEUE_SIZE))
 		xpq_flush_queue();
+}
 void
 xpq_queue_machphys_update(paddr_t ma, paddr_t pa)
+{
 	mmu_update_t *xpq_queue = xpq_queue_array[curcpu()->ci_cpuid];
 	int xpq_idx = xpq_idx_array[curcpu()->ci_cpuid];
 	XENPRINTK2(("xpq_queue_machphys_update ma=0x%" PRIx64 " pa=0x%" PRIx64
 	    "\n", (int64_t)ma, (int64_t)pa));
 	xpq_queue[xpq_idx].ptr = ma | MMU_MACHPHYS_UPDATE;
 	xpq_queue[xpq_idx].val = pa >> PAGE_SHIFT;
 	xpq_increment_idx();
 #ifdef XENDEBUG_SYNC
 	xpq_flush_queue();
 #endif
+}
 void
 xpq_queue_pte_update(paddr_t ptr, pt_entry_t val)
+{
 	mmu_update_t *xpq_queue = xpq_queue_array[curcpu()->ci_cpuid];
 	int xpq_idx = xpq_idx_array[curcpu()->ci_cpuid];
 	KASSERT((ptr & 3) == 0);
 	xpq_queue[xpq_idx].ptr = (paddr_t)ptr | MMU_NORMAL_PT_UPDATE;
 	xpq_queue[xpq_idx].val = val;
 	xpq_increment_idx();
 #ifdef XENDEBUG_SYNC
 	xpq_flush_queue();
 #endif
+}
 void
 xpq_queue_pt_switch(paddr_t pa)
+{
 	struct mmuext_op op;
 	xpq_flush_queue();
 	XENPRINTK2(("xpq_queue_pt_switch: 0x%" PRIx64 " 0x%" PRIx64 "\n",
 	    (int64_t)pa, (int64_t)pa));
 	op.cmd = MMUEXT_NEW_BASEPTR;
 	op.arg1.mfn = pa >> PAGE_SHIFT;
 	if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0)
 		panic("xpq_queue_pt_switch");
+}
 void
 xpq_queue_pin_table(paddr_t pa, int lvl)
+{
 	struct mmuext_op op;
 	xpq_flush_queue();
 	XENPRINTK2(("xpq_queue_pin_l%d_table: %#" PRIxPADDR "\n",
 	    lvl + 1, pa));
 	op.arg1.mfn = pa >> PAGE_SHIFT;
 	op.cmd = lvl;
 	if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0)
 		panic("xpq_queue_pin_table");
+}
 void
 xpq_queue_unpin_table(paddr_t pa)
+{
 	struct mmuext_op op;
 	xpq_flush_queue();
 	XENPRINTK2(("xpq_queue_unpin_table: %#" PRIxPADDR "\n", pa));
 	op.arg1.mfn = pa >> PAGE_SHIFT;
 	op.cmd = MMUEXT_UNPIN_TABLE;
 	if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0)
 		panic("xpq_queue_unpin_table");
+}
 void
 xpq_queue_set_ldt(vaddr_t va, uint32_t entries)
+{
 	struct mmuext_op op;
 	xpq_flush_queue();
 	XENPRINTK2(("xpq_queue_set_ldt\n"));
 	KASSERT(va == (va & ~PAGE_MASK));
 	op.cmd = MMUEXT_SET_LDT;
 	op.arg1.linear_addr = va;
 	op.arg2.nr_ents = entries;
 	if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0)
 		panic("xpq_queue_set_ldt");
+}
 void
 xpq_queue_tlb_flush(void)
+{
 	struct mmuext_op op;
 	xpq_flush_queue();
 	XENPRINTK2(("xpq_queue_tlb_flush\n"));
 	op.cmd = MMUEXT_TLB_FLUSH_LOCAL;
 	if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0)
 		panic("xpq_queue_tlb_flush");
+}
 void
 xpq_flush_cache(void)
+{
 	struct mmuext_op op;
 	int s = splvm(), err;
 	xpq_flush_queue();
 	XENPRINTK2(("xpq_queue_flush_cache\n"));
 	op.cmd = MMUEXT_FLUSH_CACHE;
 	if ((err = HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF)) < 0) {
 		panic("xpq_flush_cache, err %d", err);
+	}
 	splx(s); /* XXX: removeme */
+}
 void
 xpq_queue_invlpg(vaddr_t va)
+{
 	struct mmuext_op op;
 	xpq_flush_queue();
 	XENPRINTK2(("xpq_queue_invlpg %#" PRIxVADDR "\n", va));
 	op.cmd = MMUEXT_INVLPG_LOCAL;
 	op.arg1.linear_addr = (va & ~PAGE_MASK);
 	if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0)
 		panic("xpq_queue_invlpg");
+}
 void
 xen_mcast_invlpg(vaddr_t va, kcpuset_t *kc)
+{
 	xcpumask_t xcpumask;
 	mmuext_op_t op;
-	kcpuset_copybits(kc, &xcpumask.xcpum_km[0], sizeof(xcpumask));
+	kcpuset_export_u32(kc, &xcpumask.xcpum_km[0], sizeof(xcpumask));
 	/* Flush pending page updates */
 	xpq_flush_queue();
 	op.cmd = MMUEXT_INVLPG_MULTI;
 	op.arg1.linear_addr = va;
 	op.arg2.vcpumask = &xcpumask.xcpum_xm;
 	if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0) {
 		panic("xpq_queue_invlpg_all");
+	}
 	return;
+}
 void
 xen_bcast_invlpg(vaddr_t va)
+{
 	mmuext_op_t op;
 	/* Flush pending page updates */
 	xpq_flush_queue();
 	op.cmd = MMUEXT_INVLPG_ALL;
 	op.arg1.linear_addr = va;
 	if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0) {
 		panic("xpq_queue_invlpg_all");
+	}
 	return;
+}
 /* This is a synchronous call. */
 void
 xen_mcast_tlbflush(kcpuset_t *kc)
+{
 	xcpumask_t xcpumask;
 	mmuext_op_t op;
-	kcpuset_copybits(kc, &xcpumask.xcpum_km[0], sizeof(xcpumask));
+	kcpuset_export_u32(kc, &xcpumask.xcpum_km[0], sizeof(xcpumask));
 	/* Flush pending page updates */
 	xpq_flush_queue();
 	op.cmd = MMUEXT_TLB_FLUSH_MULTI;
 	op.arg2.vcpumask = &xcpumask.xcpum_xm;
 	if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0) {
 		panic("xpq_queue_invlpg_all");
+	}
 	return;
+}
 /* This is a synchronous call. */
 void
 xen_bcast_tlbflush(void)
+{
 	mmuext_op_t op;
 	/* Flush pending page updates */
 	xpq_flush_queue();
 	op.cmd = MMUEXT_TLB_FLUSH_ALL;
 	if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0) {
 		panic("xpq_queue_invlpg_all");
+	}
 	return;
+}
 /* This is a synchronous call. */
 void
 xen_vcpu_mcast_invlpg(vaddr_t sva, vaddr_t eva, kcpuset_t *kc)
+{
 	KASSERT(eva > sva);
 	/* Flush pending page updates */
 	xpq_flush_queue();
 	/* Align to nearest page boundary */
 	sva &= ~PAGE_MASK;
 	eva &= ~PAGE_MASK;
 	for ( ; sva <= eva; sva += PAGE_SIZE) {
 		xen_mcast_invlpg(sva, kc);
+	}
 	return;
+}
 /* This is a synchronous call. */
 void
 xen_vcpu_bcast_invlpg(vaddr_t sva, vaddr_t eva)
+{
 	KASSERT(eva > sva);
 	/* Flush pending page updates */
 	xpq_flush_queue();
 	/* Align to nearest page boundary */
 	sva &= ~PAGE_MASK;
 	eva &= ~PAGE_MASK;
 	for ( ; sva <= eva; sva += PAGE_SIZE) {
 		xen_bcast_invlpg(sva);
+	}
 	return;
+}
 int
 xpq_update_foreign(paddr_t ptr, pt_entry_t val, int dom)
+{
 	mmu_update_t op;
 	int ok;
 	xpq_flush_queue();
 	op.ptr = ptr;
 	op.val = val;
 	if (HYPERVISOR_mmu_update(&op, 1, &ok, dom) < 0)
 		return EFAULT;
 	return (0);
+}
 #ifdef XENDEBUG
 void
 xpq_debug_dump(void)
+{
 	int i;
 	mmu_update_t *xpq_queue = xpq_queue_array[curcpu()->ci_cpuid];
 	int xpq_idx = xpq_idx_array[curcpu()->ci_cpuid];
 	XENPRINTK2(("idx: %d\n", xpq_idx));
 	for (i = 0; i < xpq_idx; i++) {
 		snprintf(XBUF, sizeof(XBUF), "%" PRIx64 " %08" PRIx64,
 		    xpq_queue[i].ptr, xpq_queue[i].val);
 		if (++i < xpq_idx)
 			snprintf(XBUF + strlen(XBUF),
 			    sizeof(XBUF) - strlen(XBUF),
 			    "%" PRIx64 " %08" PRIx64,
 			    xpq_queue[i].ptr, xpq_queue[i].val);
 		if (++i < xpq_idx)
 			snprintf(XBUF + strlen(XBUF),
 			    sizeof(XBUF) - strlen(XBUF),
 			    "%" PRIx64 " %08" PRIx64,
 			    xpq_queue[i].ptr, xpq_queue[i].val);
 		if (++i < xpq_idx)
 			snprintf(XBUF + strlen(XBUF),
 			    sizeof(XBUF) - strlen(XBUF),
 			    "%" PRIx64 " %08" PRIx64,
 			    xpq_queue[i].ptr, xpq_queue[i].val);
 		XENPRINTK2(("%d: %s\n", xpq_idx, XBUF));
+	}
+}
 #endif
 extern volatile struct xencons_interface *xencons_interface; /* XXX */
 extern struct xenstore_domain_interface *xenstore_interface; /* XXX */
 static void xen_bt_set_readonly (vaddr_t);
 static void xen_bootstrap_tables (vaddr_t, vaddr_t, int, int, int);
 /* How many PDEs ? */
 #if L2_SLOT_KERNBASE > 0
 #define TABLE_L2_ENTRIES (2 * (NKL2_KIMG_ENTRIES + 1))
 #else
 #define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1)
 #endif
 /*
  * Construct and switch to new pagetables
  * first_avail is the first vaddr we can use after
  * we get rid of Xen pagetables
  */
 vaddr_t xen_pmap_bootstrap (void);
 /*
  * Function to get rid of Xen bootstrap tables
  */
 /* How many PDP do we need: */
 #ifdef PAE
 /*
  * For PAE, we consider a single contigous L2 "superpage" of 4 pages,
  * all of them mapped by the L3 page. We also need a shadow page
  * for L3[3].
  */
 static const int l2_4_count = 6;
 #elif defined(__x86_64__)
 static const int l2_4_count = PTP_LEVELS;
 #else
 static const int l2_4_count = PTP_LEVELS - 1;
 #endif
 vaddr_t
 xen_pmap_bootstrap(void)
+{
 	int count, oldcount;
 	long mapsize;
 	vaddr_t bootstrap_tables, init_tables;
 	memset(xpq_idx_array, 0, sizeof xpq_idx_array);
 	xpmap_phys_to_machine_mapping =
 	    (unsigned long *)xen_start_info.mfn_list;
 	init_tables = xen_start_info.pt_base;
 	__PRINTK(("xen_arch_pmap_bootstrap init_tables=0x%lx\n", init_tables));
 	/* Space after Xen boostrap tables should be free */
 	bootstrap_tables = xen_start_info.pt_base +
 		(xen_start_info.nr_pt_frames * PAGE_SIZE);
 	/*
 	 * Calculate how many space we need
 	 * first everything mapped before the Xen bootstrap tables
 	 */
 	mapsize = init_tables - KERNTEXTOFF;
 	/* after the tables we'll have:
 	 *  - UAREA
 	 *  - dummy user PGD (x86_64)
 	 *  - HYPERVISOR_shared_info
 	 *  - early_zerop
 	 *  - ISA I/O mem (if needed)
 	 */
 	mapsize += UPAGES * NBPG;
 #ifdef __x86_64__
 	mapsize += NBPG;
 #endif
 	mapsize += NBPG;
 	mapsize += NBPG;
 #ifdef DOM0OPS
 	if (xendomain_is_dom0()) {
 		/* space for ISA I/O mem */
 		mapsize += IOM_SIZE;
+	}
 #endif
 	/* at this point mapsize doens't include the table size */
 #ifdef __x86_64__
 	count = TABLE_L2_ENTRIES;
 #else
 	count = (mapsize + (NBPD_L2 -1)) >> L2_SHIFT;
 #endif /* __x86_64__ */
 	/* now compute how many L2 pages we need exactly */
 	XENPRINTK(("bootstrap_final mapsize 0x%lx count %d\n", mapsize, count));
 	while (mapsize + (count + l2_4_count) * PAGE_SIZE + KERNTEXTOFF >
 	    ((long)count << L2_SHIFT) + KERNBASE) {
 		count++;
+	}
 #ifndef __x86_64__
 	/*
 	 * one more L2 page: we'll alocate several pages after kva_start
 	 * in pmap_bootstrap() before pmap_growkernel(), which have not been
 	 * counted here. It's not a big issue to allocate one more L2 as
 	 * pmap_growkernel() will be called anyway.
 	 */
 	count++;
 	nkptp[1] = count;
 #endif
 	/*
 	 * install bootstrap pages. We may need more L2 pages than will
 	 * have the final table here, as it's installed after the final table
 	 */
 	oldcount = count;
 bootstrap_again:
 	XENPRINTK(("bootstrap_again oldcount %d\n", oldcount));
 	/*
 	 * Xen space we'll reclaim may not be enough for our new page tables,
 	 * move bootstrap tables if necessary
 	 */
 	if (bootstrap_tables < init_tables + ((count + l2_4_count) * PAGE_SIZE))
 		bootstrap_tables = init_tables +
 					((count + l2_4_count) * PAGE_SIZE);
 	/* make sure we have enough to map the bootstrap_tables */
 	if (bootstrap_tables + ((oldcount + l2_4_count) * PAGE_SIZE) >
 	    ((long)oldcount << L2_SHIFT) + KERNBASE) {
 		oldcount++;
 		goto bootstrap_again;
+	}
 	/* Create temporary tables */
 	xen_bootstrap_tables(xen_start_info.pt_base, bootstrap_tables,
 		xen_start_info.nr_pt_frames, oldcount, 0);
 	/* Create final tables */
 	xen_bootstrap_tables(bootstrap_tables, init_tables,
 	    oldcount + l2_4_count, count, 1);
 	/* zero out free space after tables */
 	memset((void *)(init_tables + ((count + l2_4_count) * PAGE_SIZE)), 0,
 	    (UPAGES + 1) * NBPG);
 	/* Finally, flush TLB. */
 	xpq_queue_tlb_flush();
 	return (init_tables + ((count + l2_4_count) * PAGE_SIZE));
+}
 /*
  * Build a new table and switch to it
  * old_count is # of old tables (including PGD, PDTPE and PDE)
  * new_count is # of new tables (PTE only)
  * we assume areas don't overlap
  */
 static void
 xen_bootstrap_tables (vaddr_t old_pgd, vaddr_t new_pgd,
 	int old_count, int new_count, int final)
+{
 	pd_entry_t *pdtpe, *pde, *pte;
 	pd_entry_t *cur_pgd, *bt_pgd;
 	paddr_t addr;
 	vaddr_t page, avail, text_end, map_end;
 	int i;
 	extern char __data_start;
 	extern char *early_zerop; /* from pmap.c */
 	__PRINTK(("xen_bootstrap_tables(%#" PRIxVADDR ", %#" PRIxVADDR ","
 	    " %d, %d)\n",
 	    old_pgd, new_pgd, old_count, new_count));
 	text_end = ((vaddr_t)&__data_start) & ~PAGE_MASK;
 	/*
 	 * size of R/W area after kernel text:
 	 *  xencons_interface (if present)
 	 *  xenstore_interface (if present)
 	 *  table pages (new_count + l2_4_count entries)
 	 * extra mappings (only when final is true):
 	 *  UAREA
 	 *  dummy user PGD (x86_64 only)/gdt page (i386 only)
 	 *  HYPERVISOR_shared_info
 	 *  early_zerop
 	 *  ISA I/O mem (if needed)
 	 */
 	map_end = new_pgd + ((new_count + l2_4_count) * NBPG);
 	if (final) {
 		map_end += (UPAGES + 1) * NBPG;
 		HYPERVISOR_shared_info = (shared_info_t *)map_end;
 		map_end += NBPG;
 		early_zerop = (char *)map_end;
 		map_end += NBPG;
+	}
 	/*
 	 * we always set atdevbase, as it's used by init386 to find the first
 	 * available VA. map_end is updated only if we are dom0, so
 	 * atdevbase -> atdevbase + IOM_SIZE will be mapped only in
 	 * this case.
 	 */
 	if (final)
 		atdevbase = map_end;
 #ifdef DOM0OPS
 	if (final && xendomain_is_dom0()) {
 		/* ISA I/O mem */
 		map_end += IOM_SIZE;
+	}
 #endif /* DOM0OPS */
 	__PRINTK(("xen_bootstrap_tables text_end 0x%lx map_end 0x%lx\n",
 	    text_end, map_end));
 	__PRINTK(("console %#lx ", xen_start_info.console_mfn));
 	__PRINTK(("xenstore %#" PRIx32 "\n", xen_start_info.store_mfn));
 	/*
 	 * Create bootstrap page tables
 	 * What we need:
 	 * - a PGD (level 4)
 	 * - a PDTPE (level 3)
 	 * - a PDE (level2)
 	 * - some PTEs (level 1)
 	 */
 	cur_pgd = (pd_entry_t *) old_pgd;
 	bt_pgd = (pd_entry_t *) new_pgd;
 	memset (bt_pgd, 0, PAGE_SIZE);
 	avail = new_pgd + PAGE_SIZE;
 #if PTP_LEVELS > 3
 	/* per-cpu L4 PD */
 	pd_entry_t *bt_cpu_pgd = bt_pgd;
 	/* pmap_kernel() "shadow" L4 PD */
 	bt_pgd = (pd_entry_t *) avail;
 	memset(bt_pgd, 0, PAGE_SIZE);
 	avail += PAGE_SIZE;
 	/* Install level 3 */
 	pdtpe = (pd_entry_t *) avail;
 	memset (pdtpe, 0, PAGE_SIZE);
 	avail += PAGE_SIZE;
 	addr = ((u_long) pdtpe) - KERNBASE;
 	bt_pgd[pl4_pi(KERNTEXTOFF)] = bt_cpu_pgd[pl4_pi(KERNTEXTOFF)] =
 	    xpmap_ptom_masked(addr) | PG_k | PG_RW | PG_V;
 	__PRINTK(("L3 va %#lx pa %#" PRIxPADDR " entry %#" PRIxPADDR
 	    " -> L4[%#x]\n",
 	    pdtpe, addr, bt_pgd[pl4_pi(KERNTEXTOFF)], pl4_pi(KERNTEXTOFF)));
 #else
 	pdtpe = bt_pgd;
 #endif /* PTP_LEVELS > 3 */
 #if PTP_LEVELS > 2
 	/* Level 2 */
 	pde = (pd_entry_t *) avail;
 	memset(pde, 0, PAGE_SIZE);
 	avail += PAGE_SIZE;
 	addr = ((u_long) pde) - KERNBASE;
 	pdtpe[pl3_pi(KERNTEXTOFF)] =
 	    xpmap_ptom_masked(addr) | PG_k | PG_V | PG_RW;
 	__PRINTK(("L2 va %#lx pa %#" PRIxPADDR " entry %#" PRIxPADDR
 	    " -> L3[%#x]\n",
 	    pde, addr, pdtpe[pl3_pi(KERNTEXTOFF)], pl3_pi(KERNTEXTOFF)));
 #elif defined(PAE)
 	/* our PAE-style level 2: 5 contigous pages (4 L2 + 1 shadow) */
 	pde = (pd_entry_t *) avail;
 	memset(pde, 0, PAGE_SIZE * 5);
 	avail += PAGE_SIZE * 5;
 	addr = ((u_long) pde) - KERNBASE;
 	/*
 	 * enter L2 pages in the L3.
 	 * The real L2 kernel PD will be the last one (so that
 	 * pde[L2_SLOT_KERN] always point to the shadow).
 	 */
 	for (i = 0; i < 3; i++, addr += PAGE_SIZE) {
 		/*
 		 * Xen doesn't want R/W mappings in L3 entries, it'll add it
 		 * itself.
 		 */
 		pdtpe[i] = xpmap_ptom_masked(addr) | PG_k | PG_V;
 		__PRINTK(("L2 va %#lx pa %#" PRIxPADDR " entry %#" PRIxPADDR
 		    " -> L3[%#x]\n",
 		    (vaddr_t)pde + PAGE_SIZE * i, addr, pdtpe[i], i));
+	}
 	addr += PAGE_SIZE;
 	pdtpe[3] = xpmap_ptom_masked(addr) | PG_k | PG_V;
 	__PRINTK(("L2 va %#lx pa %#" PRIxPADDR " entry %#" PRIxPADDR
 	    " -> L3[%#x]\n",
 	    (vaddr_t)pde + PAGE_SIZE * 4, addr, pdtpe[3], 3));
 #else /* PAE */
 	pde = bt_pgd;
 #endif /* PTP_LEVELS > 2 */
 	/* Level 1 */
 	page = KERNTEXTOFF;
 	for (i = 0; i < new_count; i ++) {
 		vaddr_t cur_page = page;
 		pte = (pd_entry_t *) avail;
 		avail += PAGE_SIZE;
 		memset(pte, 0, PAGE_SIZE);
 		while (pl2_pi(page) == pl2_pi (cur_page)) {
 			if (page >= map_end) {
 				/* not mapped at all */
 				pte[pl1_pi(page)] = 0;
 				page += PAGE_SIZE;
 				continue;
+			}
 			pte[pl1_pi(page)] = xpmap_ptom_masked(page - KERNBASE);
 			if (page == (vaddr_t)HYPERVISOR_shared_info) {
 				pte[pl1_pi(page)] = xen_start_info.shared_info;
 				__PRINTK(("HYPERVISOR_shared_info "
 				    "va %#lx pte %#" PRIxPADDR "\n",
 				    HYPERVISOR_shared_info, pte[pl1_pi(page)]));
+			}
 			if ((xpmap_ptom_masked(page - KERNBASE) >> PAGE_SHIFT)
 			    == xen_start_info.console.domU.mfn) {
 				xencons_interface = (void *)page;
 				pte[pl1_pi(page)] = xen_start_info.console_mfn;
 				pte[pl1_pi(page)] <<= PAGE_SHIFT;
 				__PRINTK(("xencons_interface "
 				    "va %#lx pte %#" PRIxPADDR "\n",
 				    xencons_interface, pte[pl1_pi(page)]));
+			}
 			if ((xpmap_ptom_masked(page - KERNBASE) >> PAGE_SHIFT)
 			    == xen_start_info.store_mfn) {
 				xenstore_interface = (void *)page;
 				pte[pl1_pi(page)] = xen_start_info.store_mfn;
 				pte[pl1_pi(page)] <<= PAGE_SHIFT;
 				__PRINTK(("xenstore_interface "
 				    "va %#lx pte %#" PRIxPADDR "\n",
 				    xenstore_interface, pte[pl1_pi(page)]));
+			}
 #ifdef DOM0OPS
 			if (page >= (vaddr_t)atdevbase &&
 			    page < (vaddr_t)atdevbase + IOM_SIZE) {
 				pte[pl1_pi(page)] =
 				    IOM_BEGIN + (page - (vaddr_t)atdevbase);
+			}
 #endif
 			pte[pl1_pi(page)] |= PG_k | PG_V;
 			if (page < text_end) {
 				/* map kernel text RO */
 				pte[pl1_pi(page)] |= 0;
 			} else if (page >= old_pgd
 			    && page < old_pgd + (old_count * PAGE_SIZE)) {
 				/* map old page tables RO */
 				pte[pl1_pi(page)] |= 0;
 			} else if (page >= new_pgd &&
 			    page < new_pgd + ((new_count + l2_4_count) * PAGE_SIZE)) {
 				/* map new page tables RO */
 				pte[pl1_pi(page)] |= 0;
 #ifdef i386
 			} else if (page == (vaddr_t)tmpgdt) {
 				/*
 				 * Map bootstrap gdt R/O. Later, we
 				 * will re-add this to page to uvm
 				 * after making it writable.
 				 */
 				pte[pl1_pi(page)] = 0;
 				page += PAGE_SIZE;
 				continue;
 #endif /* i386 */
 			} else {
 				/* map page RW */
 				pte[pl1_pi(page)] |= PG_RW;
+			}
 			if ((page  >= old_pgd && page < old_pgd + (old_count * PAGE_SIZE))
 			    || page >= new_pgd) {
 				__PRINTK(("va %#lx pa %#lx "
 				    "entry 0x%" PRIxPADDR " -> L1[%#x]\n",
 				    page, page - KERNBASE,
 				    pte[pl1_pi(page)], pl1_pi(page)));
+			}
 			page += PAGE_SIZE;
+		}
 		addr = ((u_long) pte) - KERNBASE;
 		pde[pl2_pi(cur_page)] =
 		    xpmap_ptom_masked(addr) | PG_k | PG_RW | PG_V;
 		__PRINTK(("L1 va %#lx pa %#" PRIxPADDR " entry %#" PRIxPADDR
 		    " -> L2[%#x]\n",
 		    pte, addr, pde[pl2_pi(cur_page)], pl2_pi(cur_page)));
 		/* Mark readonly */
 		xen_bt_set_readonly((vaddr_t) pte);
+	}
 	/* Install recursive page tables mapping */
 #ifdef PAE
 	/*
 	 * we need a shadow page for the kernel's L2 page
 	 * The real L2 kernel PD will be the last one (so that
 	 * pde[L2_SLOT_KERN] always point to the shadow.
 	 */
 	memcpy(&pde[L2_SLOT_KERN + NPDPG], &pde[L2_SLOT_KERN], PAGE_SIZE);
 	cpu_info_primary.ci_kpm_pdir = &pde[L2_SLOT_KERN + NPDPG];
 	cpu_info_primary.ci_kpm_pdirpa =
 	    (vaddr_t) cpu_info_primary.ci_kpm_pdir - KERNBASE;
 	/*
 	 * We don't enter a recursive entry from the L3 PD. Instead,
 	 * we enter the first 4 L2 pages, which includes the kernel's L2
 	 * shadow. But we have to entrer the shadow after switching
 	 * %cr3, or Xen will refcount some PTE with the wrong type.
 	 */
 	addr = (u_long)pde - KERNBASE;
 	for (i = 0; i < 3; i++, addr += PAGE_SIZE) {
 		pde[PDIR_SLOT_PTE + i] = xpmap_ptom_masked(addr) | PG_k | PG_V;
 		__PRINTK(("pde[%d] va %#" PRIxVADDR " pa %#" PRIxPADDR
 		    " entry %#" PRIxPADDR "\n",
 		    (int)(PDIR_SLOT_PTE + i), pde + PAGE_SIZE * i,
 		    addr, pde[PDIR_SLOT_PTE + i]));
+	}
 #if 0
 	addr += PAGE_SIZE; /* point to shadow L2 */
 	pde[PDIR_SLOT_PTE + 3] = xpmap_ptom_masked(addr) | PG_k | PG_V;
 	__PRINTK(("pde[%d] va 0x%lx pa 0x%lx entry 0x%" PRIx64 "\n",
 	    (int)(PDIR_SLOT_PTE + 3), pde + PAGE_SIZE * 4, (long)addr,
 	    (int64_t)pde[PDIR_SLOT_PTE + 3]));
 #endif
 	/* Mark tables RO, and pin the kernel's shadow as L2 */
 	addr = (u_long)pde - KERNBASE;
 	for (i = 0; i < 5; i++, addr += PAGE_SIZE) {
 		xen_bt_set_readonly(((vaddr_t)pde) + PAGE_SIZE * i);
 		if (i == 2 || i == 3)
 			continue;
 #if 0
 		__PRINTK(("pin L2 %d addr 0x%" PRIx64 "\n", i, (int64_t)addr));
 		xpq_queue_pin_l2_table(xpmap_ptom_masked(addr));
 #endif
+	}
 	if (final) {
 		addr = (u_long)pde - KERNBASE + 3 * PAGE_SIZE;
 		__PRINTK(("pin L2 %d addr %#" PRIxPADDR "\n", 2, addr));
 		xpq_queue_pin_l2_table(xpmap_ptom_masked(addr));
+	}
 #if 0
 	addr = (u_long)pde - KERNBASE + 2 * PAGE_SIZE;
 	__PRINTK(("pin L2 %d addr 0x%" PRIx64 "\n", 2, (int64_t)addr));
 	xpq_queue_pin_l2_table(xpmap_ptom_masked(addr));
 #endif
 #else /* PAE */
 	/* recursive entry in higher-level per-cpu PD and pmap_kernel() */
 	bt_pgd[PDIR_SLOT_PTE] = xpmap_ptom_masked((paddr_t)bt_pgd - KERNBASE) | PG_k | PG_V;
 #ifdef __x86_64__
 	   bt_cpu_pgd[PDIR_SLOT_PTE] =
 		   xpmap_ptom_masked((paddr_t)bt_cpu_pgd - KERNBASE) | PG_k | PG_V;
 #endif /* __x86_64__ */
 	__PRINTK(("bt_pgd[PDIR_SLOT_PTE] va %#" PRIxVADDR " pa %#" PRIxPADDR
 	    " entry %#" PRIxPADDR "\n", new_pgd, (paddr_t)new_pgd - KERNBASE,
 	    bt_pgd[PDIR_SLOT_PTE]));
 	/* Mark tables RO */
 	xen_bt_set_readonly((vaddr_t) pde);
 #endif
 #if PTP_LEVELS > 2 || defined(PAE)
 	xen_bt_set_readonly((vaddr_t) pdtpe);
 #endif
 #if PTP_LEVELS > 3
 	xen_bt_set_readonly(new_pgd);
 #endif
 	/* Pin the PGD */
 	__PRINTK(("pin PGD: %"PRIxVADDR"\n", new_pgd - KERNBASE));
 #ifdef __x86_64__
 	xpq_queue_pin_l4_table(xpmap_ptom_masked(new_pgd - KERNBASE));
 #elif PAE
 	xpq_queue_pin_l3_table(xpmap_ptom_masked(new_pgd - KERNBASE));
 #else
 	xpq_queue_pin_l2_table(xpmap_ptom_masked(new_pgd - KERNBASE));
 #endif
 	/* Save phys. addr of PDP, for libkvm. */
 #ifdef PAE
 	PDPpaddr = (u_long)pde - KERNBASE; /* PDP is the L2 with PAE */
 #else
 	PDPpaddr = (u_long)bt_pgd - KERNBASE;
 #endif
 	/* Switch to new tables */
 	__PRINTK(("switch to PGD\n"));
 	xpq_queue_pt_switch(xpmap_ptom_masked(new_pgd - KERNBASE));
 	__PRINTK(("bt_pgd[PDIR_SLOT_PTE] now entry %#" PRIxPADDR "\n",
 	    bt_pgd[PDIR_SLOT_PTE]));
 #ifdef PAE
 	if (final) {
 		/* save the address of the L3 page */
 		cpu_info_primary.ci_pae_l3_pdir = pdtpe;
 		cpu_info_primary.ci_pae_l3_pdirpa = (new_pgd - KERNBASE);
 		/* now enter kernel's PTE mappings */
 		addr =  (u_long)pde - KERNBASE + PAGE_SIZE * 3;
 		xpq_queue_pte_update(
 		    xpmap_ptom(((vaddr_t)&pde[PDIR_SLOT_PTE + 3]) - KERNBASE),
 		    xpmap_ptom_masked(addr) | PG_k | PG_V);
 		xpq_flush_queue();
+	}
 #elif defined(__x86_64__)
 	if (final) {
 		/* save the address of the real per-cpu L4 pgd page */
 		cpu_info_primary.ci_kpm_pdir = bt_cpu_pgd;
 		cpu_info_primary.ci_kpm_pdirpa = ((paddr_t) bt_cpu_pgd - KERNBASE);
+	}
 #endif
 	/* Now we can safely reclaim space taken by old tables */
 	__PRINTK(("unpin old PGD\n"));
 	/* Unpin old PGD */
 	xpq_queue_unpin_table(xpmap_ptom_masked(old_pgd - KERNBASE));
 	/* Mark old tables RW */
 	page = old_pgd;
 	addr = (paddr_t) pde[pl2_pi(page)] & PG_FRAME;
 	addr = xpmap_mtop(addr);
 	pte = (pd_entry_t *) ((u_long)addr + KERNBASE);
 	pte += pl1_pi(page);
 	__PRINTK(("*pde %#" PRIxPADDR " addr %#" PRIxPADDR " pte %#lx\n",
 	    pde[pl2_pi(page)], addr, (long)pte));
 	while (page < old_pgd + (old_count * PAGE_SIZE) && page < map_end) {
 		addr = xpmap_ptom(((u_long) pte) - KERNBASE);
 		XENPRINTK(("addr %#" PRIxPADDR " pte %#lx "
 		   "*pte %#" PRIxPADDR "\n",
 		   addr, (long)pte, *pte));
 		xpq_queue_pte_update(addr, *pte | PG_RW);
 		page += PAGE_SIZE;
 		/*
 		 * Our ptes are contiguous
 		 * so it's safe to just "++" here
 		 */
 		pte++;
+	}
 	xpq_flush_queue();
+}
 /*
  * Bootstrap helper functions
  */
 /*
  * Mark a page readonly
  * XXX: assuming vaddr = paddr + KERNBASE
  */
 static void
 xen_bt_set_readonly (vaddr_t page)
+{
 	pt_entry_t entry;
 	entry = xpmap_ptom_masked(page - KERNBASE);
 	entry |= PG_k | PG_V;
 	HYPERVISOR_update_va_mapping (page, entry, UVMF_INVLPG);
+}
 #ifdef __x86_64__
 void
 xen_set_user_pgd(paddr_t page)
+{
 	struct mmuext_op op;
 	int s = splvm();
 	xpq_flush_queue();
 	op.cmd = MMUEXT_NEW_USER_BASEPTR;
 	op.arg1.mfn = xpmap_ptom_masked(page) >> PAGE_SHIFT;
         if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0)
 		panic("xen_set_user_pgd: failed to install new user page"
 			" directory %#" PRIxPADDR, page);
 	splx(s);
+}
 #endif /* __x86_64__ */

 @@ -1,462 +1,462 @@
-/*	$NetBSD: subr_kcpuset.c,v 1.7 2012/08/20 22:01:29 rmind Exp $	*/
+/*	$NetBSD: subr_kcpuset.c,v 1.8 2012/09/16 22:09:33 rmind Exp $	*/
 /*-
  * Copyright (c) 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Mindaugas Rasiukevicius.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 /*
  * Kernel CPU set implementation.
+ *
  * Interface can be used by kernel subsystems as a unified dynamic CPU
  * bitset implementation handling many CPUs.  Facility also supports early
  * use by MD code on boot, as it fixups bitsets on further boot.
+ *
  * TODO:
  * - Handle "reverse" bitset on fixup/grow.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_kcpuset.c,v 1.7 2012/08/20 22:01:29 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_kcpuset.c,v 1.8 2012/09/16 22:09:33 rmind Exp $");
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/atomic.h>
 #include <sys/sched.h>
 #include <sys/kcpuset.h>
 #include <sys/pool.h>
 /* Number of CPUs to support. */
 #define	KC_MAXCPUS		roundup2(MAXCPUS, 32)
 /*
  * Structure of dynamic CPU set in the kernel.
  */
 struct kcpuset {
 	uint32_t		bits[0];
 };
 typedef struct kcpuset_impl {
 	/* Reference count. */
 	u_int			kc_refcnt;
 	/* Next to free, if non-NULL (used when multiple references). */
 	struct kcpuset *	kc_next;
 	/* Actual variable-sized field of bits. */
 	struct kcpuset		kc_field;
 } kcpuset_impl_t;
 #define	KC_BITS_OFF		(offsetof(struct kcpuset_impl, kc_field))
 #define	KC_GETSTRUCT(b)		((kcpuset_impl_t *)((char *)(b) - KC_BITS_OFF))
 /* Sizes of a single bitset. */
 #define	KC_SHIFT		5
 #define	KC_MASK			31
 /* An array of noted early kcpuset creations and data. */
 #define	KC_SAVE_NITEMS		8
 /* Structures for early boot mechanism (must be statically initialised). */
 static kcpuset_t **		kc_noted_early[KC_SAVE_NITEMS];
 static uint32_t			kc_bits_early[KC_SAVE_NITEMS];
 static int			kc_last_idx = 0;
 static bool			kc_initialised = false;
 #define	KC_BITSIZE_EARLY	sizeof(kc_bits_early[0])
 #define	KC_NFIELDS_EARLY	1
 /*
  * The size of whole bitset fields and amount of fields.
  * The whole size must statically initialise for early case.
  */
 static size_t			kc_bitsize __read_mostly = KC_BITSIZE_EARLY;
 static size_t			kc_nfields __read_mostly = KC_NFIELDS_EARLY;
 static pool_cache_t		kc_cache __read_mostly;
 static kcpuset_t *		kcpuset_create_raw(bool);
 /*
  * kcpuset_sysinit: initialize the subsystem, transfer early boot cases
  * to dynamically allocated sets.
  */
 void
 kcpuset_sysinit(void)
+{
 	kcpuset_t *kc_dynamic[KC_SAVE_NITEMS], *kcp;
 	int i, s;
 	/* Set a kcpuset_t sizes. */
 	kc_nfields = (KC_MAXCPUS >> KC_SHIFT);
 	kc_bitsize = sizeof(uint32_t) * kc_nfields;
 	KASSERT(kc_nfields != 0 && kc_bitsize != 0);
 	kc_cache = pool_cache_init(sizeof(kcpuset_impl_t) + kc_bitsize,
 	    coherency_unit, 0, 0, "kcpuset", NULL, IPL_NONE, NULL, NULL, NULL);
 	/* First, pre-allocate kcpuset entries. */
 	for (i = 0; i < kc_last_idx; i++) {
 		kcp = kcpuset_create_raw(true);
 		kc_dynamic[i] = kcp;
+	}
 	/*
 	 * Prepare to convert all early noted kcpuset uses to dynamic sets.
 	 * All processors, except the one we are currently running (primary),
 	 * must not be spinned yet.  Since MD facilities can use kcpuset,
 	 * raise the IPL to high.
 	 */
 	KASSERT(mp_online == false);
 	s = splhigh();
 	for (i = 0; i < kc_last_idx; i++) {
 		/*
 		 * Transfer the bits from early static storage to the kcpuset.
 		 */
 		KASSERT(kc_bitsize >= KC_BITSIZE_EARLY);
 		memcpy(kc_dynamic[i], &kc_bits_early[i], KC_BITSIZE_EARLY);
 		/*
 		 * Store the new pointer, pointing to the allocated kcpuset.
 		 * Note: we are not in an interrupt context and it is the only
 		 * CPU running - thus store is safe (e.g. no need for pointer
 		 * variable to be volatile).
 		 */
 		*kc_noted_early[i] = kc_dynamic[i];
+	}
 	kc_initialised = true;
 	kc_last_idx = 0;
 	splx(s);
+}
 /*
  * kcpuset_early_ptr: note an early boot use by saving the pointer and
  * returning a pointer to a static, temporary bit field.
  */
 static kcpuset_t *
 kcpuset_early_ptr(kcpuset_t **kcptr)
+{
 	kcpuset_t *kcp;
 	int s;
 	s = splhigh();
 	if (kc_last_idx < KC_SAVE_NITEMS) {
 		/*
 		 * Save the pointer, return pointer to static early field.
 		 * Need to zero it out.
 		 */
 		kc_noted_early[kc_last_idx] = kcptr;
 		kcp = (kcpuset_t *)&kc_bits_early[kc_last_idx];
 		kc_last_idx++;
 		memset(kcp, 0, KC_BITSIZE_EARLY);
 		KASSERT(kc_bitsize == KC_BITSIZE_EARLY);
 	} else {
 		panic("kcpuset(9): all early-use entries exhausted; "
 		    "increase KC_SAVE_NITEMS\n");
+	}
 	splx(s);
 	return kcp;
+}
 /*
  * Routines to create or destroy the CPU set.
  * Early boot case is handled.
  */
 static kcpuset_t *
 kcpuset_create_raw(bool zero)
+{
 	kcpuset_impl_t *kc;
 	kc = pool_cache_get(kc_cache, PR_WAITOK);
 	kc->kc_refcnt = 1;
 	kc->kc_next = NULL;
 	if (zero) {
 		memset(&kc->kc_field, 0, kc_bitsize);
+	}
 	/* Note: return pointer to the actual field of bits. */
 	KASSERT((uint8_t *)kc + KC_BITS_OFF == (uint8_t *)&kc->kc_field);
 	return &kc->kc_field;
+}
 void
 kcpuset_create(kcpuset_t **retkcp, bool zero)
+{
 	if (__predict_false(!kc_initialised)) {
 		/* Early boot use - special case. */
 		*retkcp = kcpuset_early_ptr(retkcp);
 		return;
+	}
 	*retkcp = kcpuset_create_raw(zero);
+}
 void
 kcpuset_destroy(kcpuset_t *kcp)
+{
 	kcpuset_impl_t *kc;
 	KASSERT(kc_initialised);
 	KASSERT(kcp != NULL);
 	do {
 		kc = KC_GETSTRUCT(kcp);
 		kcp = kc->kc_next;
 		pool_cache_put(kc_cache, kc);
 	} while (kcp);
+}
 /*
  * Routines to reference/unreference the CPU set.
  * Note: early boot case is not supported by these routines.
  */
 void
 kcpuset_use(kcpuset_t *kcp)
+{
 	kcpuset_impl_t *kc = KC_GETSTRUCT(kcp);
 	KASSERT(kc_initialised);
 	atomic_inc_uint(&kc->kc_refcnt);
+}
 void
 kcpuset_unuse(kcpuset_t *kcp, kcpuset_t **lst)
+{
 	kcpuset_impl_t *kc = KC_GETSTRUCT(kcp);
 	KASSERT(kc_initialised);
 	KASSERT(kc->kc_refcnt > 0);
 	if (atomic_dec_uint_nv(&kc->kc_refcnt) != 0) {
 		return;
+	}
 	KASSERT(kc->kc_next == NULL);
 	if (lst == NULL) {
 		kcpuset_destroy(kcp);
 		return;
+	}
 	kc->kc_next = *lst;
 	*lst = kcp;
+}
 /*
  * Routines to transfer the CPU set from / to userspace.
  * Note: early boot case is not supported by these routines.
  */
 int
 kcpuset_copyin(const cpuset_t *ucp, kcpuset_t *kcp, size_t len)
+{
 	kcpuset_impl_t *kc __unused = KC_GETSTRUCT(kcp);
 	KASSERT(kc_initialised);
 	KASSERT(kc->kc_refcnt > 0);
 	KASSERT(kc->kc_next == NULL);
 	if (len > kc_bitsize) { /* XXX */
 		return EINVAL;
+	}
 	return copyin(ucp, kcp, len);
+}
 int
 kcpuset_copyout(kcpuset_t *kcp, cpuset_t *ucp, size_t len)
+{
 	kcpuset_impl_t *kc __unused = KC_GETSTRUCT(kcp);
 	KASSERT(kc_initialised);
 	KASSERT(kc->kc_refcnt > 0);
 	KASSERT(kc->kc_next == NULL);
 	if (len > kc_bitsize) { /* XXX */
 		return EINVAL;
+	}
 	return copyout(kcp, ucp, len);
+}
 void
-kcpuset_copybits(const kcpuset_t *kcp, uint32_t *bitfield, size_t len)
+kcpuset_export_u32(const kcpuset_t *kcp, uint32_t *bitfield, size_t len)
+{
 	size_t rlen = MIN(kc_bitsize, len);
 	KASSERT(kcp != NULL);
 	memcpy(bitfield, kcp->bits, rlen);
+}
 /*
  * Routines to change bit field - zero, fill, copy, set, unset, etc.
  */
 void
 kcpuset_zero(kcpuset_t *kcp)
+{
 	KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_refcnt > 0);
 	KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL);
 	memset(kcp, 0, kc_bitsize);
+}
 void
 kcpuset_fill(kcpuset_t *kcp)
+{
 	KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_refcnt > 0);
 	KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL);
 	memset(kcp, ~0, kc_bitsize);
+}
 void
 kcpuset_copy(kcpuset_t *dkcp, kcpuset_t *skcp)
+{
 	KASSERT(!kc_initialised || KC_GETSTRUCT(dkcp)->kc_refcnt > 0);
 	KASSERT(!kc_initialised || KC_GETSTRUCT(dkcp)->kc_next == NULL);
 	memcpy(dkcp, skcp, kc_bitsize);
+}
 void
 kcpuset_set(kcpuset_t *kcp, cpuid_t i)
+{
 	const size_t j = i >> KC_SHIFT;
 	KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL);
 	KASSERT(j < kc_nfields);
 	kcp->bits[j] |= 1 << (i & KC_MASK);
+}
 void
 kcpuset_clear(kcpuset_t *kcp, cpuid_t i)
+{
 	const size_t j = i >> KC_SHIFT;
 	KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL);
 	KASSERT(j < kc_nfields);
 	kcp->bits[j] &= ~(1 << (i & KC_MASK));
+}
 bool
 kcpuset_isset(kcpuset_t *kcp, cpuid_t i)
+{
 	const size_t j = i >> KC_SHIFT;
 	KASSERT(kcp != NULL);
 	KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_refcnt > 0);
 	KASSERT(!kc_initialised || KC_GETSTRUCT(kcp)->kc_next == NULL);
 	KASSERT(j < kc_nfields);
 	return ((1 << (i & KC_MASK)) & kcp->bits[j]) != 0;
+}
 bool
 kcpuset_isotherset(kcpuset_t *kcp, cpuid_t i)
+{
 	const size_t j2 = i >> KC_SHIFT;
 	const uint32_t mask = ~(1 << (i & KC_MASK));
 	for (size_t j = 0; j < kc_nfields; j++) {
 		const uint32_t bits = kcp->bits[j];
 		if (bits && (j != j2 || (bits & mask) != 0)) {
 			return true;
+		}
+	}
 	return false;
+}
 bool
 kcpuset_iszero(kcpuset_t *kcp)
+{
 	for (size_t j = 0; j < kc_nfields; j++) {
 		if (kcp->bits[j] != 0) {
 			return false;
+		}
+	}
 	return true;
+}
 bool
 kcpuset_match(const kcpuset_t *kcp1, const kcpuset_t *kcp2)
+{
 	return memcmp(kcp1, kcp2, kc_bitsize) == 0;
+}
 void
 kcpuset_merge(kcpuset_t *kcp1, kcpuset_t *kcp2)
+{
 	for (size_t j = 0; j < kc_nfields; j++) {
 		kcp1->bits[j] |= kcp2->bits[j];
+	}
+}
 void
 kcpuset_intersect(kcpuset_t *kcp1, kcpuset_t *kcp2)
+{
 	for (size_t j = 0; j < kc_nfields; j++) {
 		kcp1->bits[j] &= kcp2->bits[j];
+	}
+}
 int
 kcpuset_countset(kcpuset_t *kcp)
+{
 	int count = 0;
 	for (size_t j = 0; j < kc_nfields; j++) {
 		count += popcount32(kcp->bits[j]);
+	}
 	return count;
+}
 /*
  * Routines to set/clear the flags atomically.
  */
 void
 kcpuset_atomic_set(kcpuset_t *kcp, cpuid_t i)
+{
 	const size_t j = i >> KC_SHIFT;
 	KASSERT(j < kc_nfields);
 	atomic_or_32(&kcp->bits[j], 1 << (i & KC_MASK));
+}
 void
 kcpuset_atomic_clear(kcpuset_t *kcp, cpuid_t i)
+{
 	const size_t j = i >> KC_SHIFT;
 	KASSERT(j < kc_nfields);
 	atomic_and_32(&kcp->bits[j], ~(1 << (i & KC_MASK)));
+}