 @@ -1,14 +1,14 @@
-/*	$NetBSD: pmap.c,v 1.8 2011/06/23 20:46:15 matt Exp $	*/
+/*	$NetBSD: pmap.c,v 1.9 2011/06/29 05:53:44 matt Exp $	*/
 /*-
  * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
  * NASA Ames Research Center and by Chris G. Demetriou.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
 @@ -57,27 +57,27 @@
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  *	@(#)pmap.c	8.4 (Berkeley) 1/26/94
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.8 2011/06/23 20:46:15 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.9 2011/06/29 05:53:44 matt Exp $");
 /*
  *	Manages physical address maps.
+ *
  *	In addition to hardware address maps, this
  *	module is called upon to provide software-use-only
  *	maps which may or may not be stored in the same
  *	form as hardware maps.  These pseudo-maps are
  *	used to store intermediate results from copy
  *	operations to and from address spaces.
+ *
  *	Since the information managed by this module is
  *	also stored by the logical address mapping module,
 @@ -314,27 +314,27 @@ static void
 pmap_page_syncicache(struct vm_page *pg)
+{
 #ifndef MULTIPROCESSOR
 	struct pmap * const curpmap = curcpu()->ci_curpm;
 #endif
 	struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
 	pv_entry_t pv = &mdpg->mdpg_first;
 	__cpuset_t onproc = CPUSET_NULLSET;
 	(void)VM_PAGEMD_PVLIST_LOCK(mdpg, false);
 	if (pv->pv_pmap != NULL) {
 		for (; pv != NULL; pv = pv->pv_next) {
 #ifdef MULTIPROCESSOR
 			CPUSET_MERGE(onproc, pv->pv_pmap->pm_onproc);
-			if (CPUSET_EQUAL_P(onproc, cpus_running)) {
+			if (CPUSET_EQUAL_P(onproc, cpuset_info.cpus_running)) {
 				break;
+			}
 #else
 			if (pv->pv_pmap == curpmap) {
 				onproc = CPUSET_SINGLE(0);
 				break;
+			}
 #endif
+		}
+	}
 	VM_PAGEMD_PVLIST_UNLOCK(mdpg);
 	kpreempt_disable();
 	pmap_md_page_syncicache(pg, onproc);
 @@ -575,27 +575,27 @@ pmap_deactivate(struct lwp *l)
 	UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0);
+}
 void
 pmap_update(struct pmap *pmap)
+{
 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist);
 	UVMHIST_LOG(pmaphist, "(pmap=%p)", pmap, 0,0,0);
 	PMAP_COUNT(update);
 	kpreempt_disable();
-#ifdef MULTIPROCESSOR
+#if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN)
 	u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0);
 	if (pending && pmap_tlb_shootdown_bystanders(pmap))
 		PMAP_COUNT(shootdown_ipis);
 #endif
 #ifdef DEBUG
 	pmap_tlb_check(pmap, pmap_md_tlb_check_entry);
 #endif /* DEBUG */
 	/*
 	 * If pmap_remove_all was called, we deactivated ourselves and nuked
 	 * our ASID.  Now we have to reactivate ourselves.
 	 */
 	if (__predict_false(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE)) {

 @@ -1,14 +1,14 @@
-/*	$NetBSD: pmap_tlb.c,v 1.9 2011/06/23 08:11:56 matt Exp $	*/
+/*	$NetBSD: pmap_tlb.c,v 1.10 2011/06/29 05:53:44 matt Exp $	*/
 /*-
  * Copyright (c) 2010 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Matt Thomas at 3am Software Foundry.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -21,27 +21,27 @@
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.9 2011/06/23 08:11:56 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.10 2011/06/29 05:53:44 matt Exp $");
 /*
  * Manages address spaces in a TLB.
+ *
  * Normally there is a 1:1 mapping between a TLB and a CPU.  However, some
  * implementations may share a TLB between multiple CPUs (really CPU thread
  * contexts).  This requires the TLB abstraction to be separated from the
  * CPU abstraction.  It also requires that the TLB be locked while doing
  * TLB activities.
+ *
  * For each TLB, we track the ASIDs in use in a bitmap and a list of pmaps
  * that have a valid ASID.
+ *
 @@ -152,30 +152,30 @@ struct pmap_tlb_info pmap_tlb0_info = {
 	.ti_wired = PMAP_TLB_WIRED_UPAGES,
 #endif
 	.ti_lock = &pmap_tlb0_mutex,
 	.ti_pais = LIST_HEAD_INITIALIZER(pmap_tlb0_info.ti_pais),
 #if defined(MULTIPROCESSOR)
 	.ti_cpu_mask = 1,
 	.ti_tlbinvop = TLBINV_NOBODY,
 #endif
 };
 #undef IFCONSTANT
 #if defined(MULTIPROCESSOR)
-static struct pmap_tlb_info *pmap_tlbs[MAXCPUS] = {
+struct pmap_tlb_info *pmap_tlbs[MAXCPUS] = {
 	[0] = &pmap_tlb0_info,
 };
-static u_int pmap_ntlbs = 1;
+u_int pmap_ntlbs = 1;
 #endif
 #define	__BITMAP_SET(bm, n) \
 	((bm)[(n) / (8*sizeof(bm[0]))] |= 1LU << ((n) % (8*sizeof(bm[0]))))
 #define	__BITMAP_CLR(bm, n) \
 	((bm)[(n) / (8*sizeof(bm[0]))] &= ~(1LU << ((n) % (8*sizeof(bm[0])))))
 #define	__BITMAP_ISSET_P(bm, n) \
 	(((bm)[(n) / (8*sizeof(bm[0]))] & (1LU << ((n) % (8*sizeof(bm[0]))))) != 0)
 #define	TLBINFO_ASID_MARK_USED(ti, asid) \
 	__BITMAP_SET((ti)->ti_asid_bitmap, (asid))
 #define	TLBINFO_ASID_INUSE_P(ti, asid) \
 	__BITMAP_ISSET_P((ti)->ti_asid_bitmap, (asid))
 @@ -453,27 +453,27 @@ pmap_tlb_shootdown_process(void)
 	KASSERTMSG(ci->ci_cpl >= IPL_SCHED,
 	    ("%s: cpl (%d) < IPL_SCHED (%d)",
 	    __func__, ci->ci_cpl, IPL_SCHED));
 	TLBINFO_LOCK(ti);
 	switch (ti->ti_tlbinvop) {
 	case TLBINV_ONE: {
 		/*
 		 * We only need to invalidate one user ASID.
 		 */
 		struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti);
 		KASSERT(ti->ti_victim != pmap_kernel());
-		if (!CPU_EMPTY_P(CPUSET_SUBSET(ti->ti_victim->pm_onproc, ti->ti_cpu_mask))) {
+		if (!CPUSET_EMPTY_P(CPUSET_SUBSET(ti->ti_victim->pm_onproc, ti->ti_cpu_mask))) {
 			/*
 			 * The victim is an active pmap so we will just
 			 * invalidate its TLB entries.
 			 */
 			KASSERT(pai->pai_asid > KERNEL_PID);
 			pmap_tlb_asid_check();
 			tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
 			pmap_tlb_asid_check();
 		} else if (pai->pai_asid) {
 			/*
 			 * The victim is no longer an active pmap for this TLB.
 			 * So simply clear its ASID and when pmap_activate is
 			 * next called for this pmap, it will allocate a new
 @@ -592,27 +592,27 @@ pmap_tlb_shootdown_bystanders(pmap_t pm)
 						ti->ti_victim = NULL;
+				}
+			}
 			TLBINFO_UNLOCK(ti);
 			/*
 			 * Now we can send out the shootdown IPIs to a CPU
 			 * that shares this TLB and is currently using this
 			 * pmap.  That CPU will process the IPI and do the
 			 * all the work.  Any other CPUs sharing that TLB
 			 * will take advantage of that work.  pm_onproc might
 			 * change now that we have released the lock but we
 			 * can tolerate spurious shootdowns.
 			 */
-			KASSERT(!CPU_EMPTY_P(onproc));
+			KASSERT(!CPUSET_EMPTY_P(onproc));
 			u_int j = CPUSET_NEXT(onproc);
 			cpu_send_ipi(cpu_lookup(j), IPI_SHOOTDOWN);
 			ipi_sent = true;
 			continue;
+		}
 		if (!CPUSET_EMPTY_P(CPUSET_SUBSET(pm->pm_active, ti->ti_cpu_mask) {
 			/*
 			 * If this pmap has an ASID assigned but it's not
 			 * currently running, nuke its ASID.  Next time the
 			 * pmap is activated, it will allocate a new ASID.
 			 * And best of all, we avoid an IPI.
 			 */
 			KASSERT(!kernel_p);
 @@ -672,28 +672,28 @@ pmap_tlb_invalidate_addr(pmap_t pm, vadd
 static inline void
 pmap_tlb_asid_alloc(struct pmap_tlb_info *ti, pmap_t pm,
 	struct pmap_asid_info *pai)
+{
 	/*
 	 * We shouldn't have an ASID assigned, and thusly must not be onproc
 	 * nor active.
 	 */
 	KASSERT(pm != pmap_kernel());
 	KASSERT(pai->pai_asid == 0);
 	KASSERT(pai->pai_link.le_prev == NULL);
 #if defined(MULTIPROCESSOR)
-	KASSERT(CPU_EMPTY_P(CPUSET_SUBSET(pm->pm_onproc, ti->ti_cpu_mask)));
+	KASSERT(CPUSET_EMPTY_P(CPUSET_SUBSET(pm->pm_onproc, ti->ti_cpu_mask)));
-	KASSERT(CPU_EMPTY_P(CPUSET_SUBSET(pm->pm_active, ti->ti_cpu_mask)));
+	KASSERT(CPUSET_EMPTY_P(CPUSET_SUBSET(pm->pm_active, ti->ti_cpu_mask)));
 #endif
 	KASSERT(ti->ti_asids_free > 0);
 	KASSERT(ti->ti_asid_hint <= ti->ti_asid_max);
 	/*
 	 * Let's see if the hinted ASID is free.  If not search for
 	 * a new one.
 	 */
 	if (__predict_false(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) {
 #ifdef DIAGNOSTIC
 		const size_t words = __arraycount(ti->ti_asid_bitmap);
 #endif
 		const size_t nbpw = 8 * sizeof(ti->ti_asid_bitmap[0]);
 @@ -780,27 +780,27 @@ pmap_tlb_asid_acquire(pmap_t pm, struct
 		/*
 		 * Get an ASID.
 		 */
 		pmap_tlb_asid_alloc(ti, pm, pai);
+	}
 	if (l == curlwp) {
 #if defined(MULTIPROCESSOR)
 		/*
 		 * The bits in pm_onproc belonging to this TLB can only
 		 * be changed while this TLBs lock is held unless atomic
 		 * operations are used.
 		 */
-		CPUSET_ADD(&pm->pm_onproc, cpu_index(ci));
+		CPUSET_ADD(pm->pm_onproc, cpu_index(ci));
 #endif
 		ci->ci_pmap_asid_cur = pai->pai_asid;
 		tlb_set_asid(pai->pai_asid);
 		pmap_tlb_asid_check();
 	} else {
 		printf("%s: l (%p) != curlwp %p\n", __func__, l, curlwp);
+	}
 	TLBINFO_UNLOCK(ti);
+}
 void
 pmap_tlb_asid_deactivate(pmap_t pm)
+{
 @@ -827,31 +827,31 @@ pmap_tlb_asid_deactivate(pmap_t pm)
+	}
 #elif defined(DEBUG)
 	curcpu()->ci_pmap_asid_cur = 0;
 	tlb_set_asid(0);
 	pmap_tlb_asid_check();
 #endif
+}
 void
 pmap_tlb_asid_release_all(struct pmap *pm)
+{
 	KASSERT(pm != pmap_kernel());
 #if defined(MULTIPROCESSOR)
-	KASSERT(CPUSET_EMPTY(pm->pm_onproc));
+	KASSERT(CPUSET_EMPTY_P(pm->pm_onproc));
-	for (u_int i = 0; !CPUSET_EMPTY(pm->pm_active); i++) {
+	for (u_int i = 0; !CPUSET_EMPTY_P(pm->pm_active); i++) {
 		KASSERT(i < pmap_ntlbs);
 		struct pmap_tlb_info * const ti = pmap_tlbs[i];
-		if (!CPU_EMPTY_P(CPUSET_SUBSET(pm->pm_active, ti->ti_cpu_mask))) {
+		if (!CPUSET_EMPTY_P(CPUSET_SUBSET(pm->pm_active, ti->ti_cpu_mask))) {
 			struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
 			TLBINFO_LOCK(ti);
 			KASSERT(ti->ti_victim != pm);
 			pmap_pai_reset(ti, pai, pm);
 			TLBINFO_UNLOCK(ti);
+		}
+	}
 #else
 	/*
 	 * Handle the case of an UP kernel which only has, at most, one ASID.
 	 * If the pmap has an ASID allocated, free it.
 	 */
 	struct pmap_tlb_info * const ti = &pmap_tlb0_info;