 @@ -1,14 +1,14 @@
-/*	$NetBSD: pmap.c,v 1.181 2008/07/09 23:22:15 scw Exp $	*/
+/*	$NetBSD: pmap.c,v 1.182 2008/07/16 00:19:57 matt Exp $	*/
 /*
  * Copyright 2003 Wasabi Systems, Inc.
  * All rights reserved.
+ *
  * Written by Steve C. Woodford for Wasabi Systems, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
 @@ -201,27 +201,27 @@
 #include <sys/user.h>
 #include <sys/pool.h>
 #include <sys/cdefs.h>
 #include <sys/cpu.h>
 #include <uvm/uvm.h>
 #include <machine/bus.h>
 #include <machine/pmap.h>
 #include <machine/pcb.h>
 #include <machine/param.h>
 #include <arm/arm32/katelib.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.181 2008/07/09 23:22:15 scw Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.182 2008/07/16 00:19:57 matt Exp $");
 #ifdef PMAP_DEBUG
 /* XXX need to get rid of all refs to this */
 int pmap_debug_level = 0;
 /*
  * for switching to potentially finer grained debugging
  */
 #define	PDB_FOLLOW	0x0001
 #define	PDB_INIT	0x0002
 #define	PDB_ENTER	0x0004
 #define	PDB_REMOVE	0x0008
 @@ -304,38 +304,41 @@ static vaddr_t pmap_kernel_l2ptp_kva;
 static paddr_t pmap_kernel_l2ptp_phys;
 #ifdef PMAPCOUNT
 #define	PMAP_EVCNT_INITIALIZER(name) \
 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", name)
 #ifdef PMAP_CACHE_VIPT
 static struct evcnt pmap_ev_vac_color_new =
    PMAP_EVCNT_INITIALIZER("new page color");
 static struct evcnt pmap_ev_vac_color_reuse =
    PMAP_EVCNT_INITIALIZER("ok first page color");
 static struct evcnt pmap_ev_vac_color_ok =
    PMAP_EVCNT_INITIALIZER("ok page color");
 static struct evcnt pmap_ev_vac_color_blind =
    PMAP_EVCNT_INITIALIZER("blind page color");
 static struct evcnt pmap_ev_vac_color_change =
    PMAP_EVCNT_INITIALIZER("change page color");
 static struct evcnt pmap_ev_vac_color_erase =
    PMAP_EVCNT_INITIALIZER("erase page color");
 static struct evcnt pmap_ev_vac_color_none =
    PMAP_EVCNT_INITIALIZER("no page color");
 static struct evcnt pmap_ev_vac_color_restore =
    PMAP_EVCNT_INITIALIZER("restore page color");
 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_new);
 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_reuse);
 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_ok);
 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_blind);
 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_change);
 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_erase);
 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_none);
 EVCNT_ATTACH_STATIC(pmap_ev_vac_color_restore);
 #endif
 static struct evcnt pmap_ev_mappings =
    PMAP_EVCNT_INITIALIZER("pages mapped");
 static struct evcnt pmap_ev_unmappings =
    PMAP_EVCNT_INITIALIZER("pages unmapped");
 static struct evcnt pmap_ev_remappings =
    PMAP_EVCNT_INITIALIZER("pages remapped");
 @@ -769,28 +772,27 @@ static inline void
 pmap_dcache_wbinv_all(pmap_t pm)
+{
 	if (pm->pm_cstate.cs_cache_d) {
 		cpu_dcache_wbinv_all();
 		pm->pm_cstate.cs_cache_d = 0;
+	}
+}
 #endif /* PMAP_CACHE_VIVT */
 static inline bool
 pmap_is_current(pmap_t pm)
+{
-	if (pm == pmap_kernel() ||
+	if (pm == pmap_kernel() || curproc->p_vmspace->vm_map.pmap == pm)
 	    (curproc && curproc->p_vmspace->vm_map.pmap == pm))
 		return true;
 	return false;
+}
 static inline bool
 pmap_is_cached(pmap_t pm)
+{
 	if (pm == pmap_kernel() || pmap_recent_user == NULL ||
 	    pmap_recent_user == pm)
 		return (true);
 @@ -829,38 +831,49 @@ do {					\
 /*
  * pmap_enter_pv: enter a mapping onto a vm_page lst
+ *
  * => caller should hold the proper lock on pmap_main_lock
  * => caller should have pmap locked
  * => we will gain the lock on the vm_page and allocate the new pv_entry
  * => caller should adjust ptp's wire_count before calling
  * => caller should not adjust pmap's wire_count
  */
 static void
 pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, pmap_t pm,
     vaddr_t va, u_int flags)
+{
 	struct pv_entry **pvp;
 	NPDEBUG(PDB_PVDUMP,
 	    printf("pmap_enter_pv: pm %p, pg %p, flags 0x%x\n", pm, pg, flags));
 	pve->pv_pmap = pm;
 	pve->pv_va = va;
 	pve->pv_flags = flags;
 	simple_lock(&pg->mdpage.pvh_slock);	/* lock vm_page */
-	pve->pv_next = pg->mdpage.pvh_list;	/* add to ... */
+	pvp = &pg->mdpage.pvh_list;
-	pg->mdpage.pvh_list = pve;		/* ... locked list */
+#ifdef PMAP_CACHE_VIPT
-	pg->mdpage.pvh_attrs |= flags & (PVF_REF | PVF_MOD);
+	/*
 	 * Insert unmapped entries at the head of the pv list.
 	 */
 	if (__predict_true((flags & PVF_KENTRY) == 0)) {
 		while (*pvp != NULL && (*pvp)->pv_flags & PVF_KENTRY)
 			pvp = &(*pvp)->pv_next;
+	}
 #endif
 	pve->pv_next = *pvp;			/* add to ... */
 	*pvp = pve;				/* ... locked list */
 	pg->mdpage.pvh_attrs |= flags & (PVF_REF | PVF_MOD | PVF_KENTRY);
 	if (pm == pmap_kernel()) {
 		PMAPCOUNT(kernel_mappings);
 		if (flags & PVF_WRITE)
 			pg->mdpage.krw_mappings++;
 		else
 			pg->mdpage.kro_mappings++;
 	} else
 	if (flags & PVF_WRITE)
 		pg->mdpage.urw_mappings++;
 	else
 		pg->mdpage.uro_mappings++;
 #ifdef PMAP_CACHE_VIPT
 @@ -923,26 +936,38 @@ pmap_remove_pv(struct vm_page *pg, pmap_
 	prevptr = &pg->mdpage.pvh_list;		/* previous pv_entry pointer */
 	pve = *prevptr;
 	while (pve) {
 		if (pve->pv_pmap == pm && pve->pv_va == va) {	/* match? */
 			NPDEBUG(PDB_PVDUMP, printf("pmap_remove_pv: pm %p, pg "
 			    "%p, flags 0x%x\n", pm, pg, pve->pv_flags));
 			if (pve->pv_flags & PVF_WIRED) {
 				if (skip_wired)
 					return (NULL);
 				--pm->pm_stats.wired_count;
+			}
 #ifdef PMAP_CACHE_VIPT
 			/*
 			 * If we are removing the first pv entry and its
 			 * a KENTRY, if the next one isn't also a KENTER,
 			 * clear KENTRY from the page attributes.
 			 */
 			if (pg->mdpage.pvh_list == pve
 			    && (pve->pv_flags & PVF_KENTRY)
 			    && (pve->pv_next == NULL
 				|| (pve->pv_next->pv_flags & PVF_KENTRY) == 0))
 				pg->mdpage.pvh_attrs &= ~PVF_KENTRY;
 #endif
 			*prevptr = pve->pv_next;		/* remove it! */
 			if (pm == pmap_kernel()) {
 				PMAPCOUNT(kernel_unmappings);
 				if (pve->pv_flags & PVF_WRITE)
 					pg->mdpage.krw_mappings--;
 				else
 					pg->mdpage.kro_mappings--;
 			} else
 			if (pve->pv_flags & PVF_WRITE)
 				pg->mdpage.urw_mappings--;
 			else
 				pg->mdpage.uro_mappings--;
 @@ -961,26 +986,36 @@ pmap_remove_pv(struct vm_page *pg, pmap_
 					PMAPCOUNT(exec_discarded_unmap);
 				} else {
 					pmap_syncicache_page(pg);
 					PMAPCOUNT(exec_synced_unmap);
+				}
+			}
 #endif /* PMAP_CACHE_VIPT */
 			break;
+		}
 		prevptr = &pve->pv_next;		/* previous pointer */
 		pve = pve->pv_next;			/* advance */
+	}
 #ifdef PMAP_CACHE_VIPT
 	/*
 	 * If this was a writeable page and there are no more writeable
 	 * mappings (ignoring KMPAGE), clear the WRITE flag.
 	 */
 	if ((pg->mdpage.pvh_attrs & PVF_WRITE)
 	    && pg->mdpage.krw_mappings + pg->mdpage.urw_mappings == 0)
 		pg->mdpage.pvh_attrs &= ~PVF_WRITE;
 #endif /* PMAP_CACHE_VIPT */
 	return(pve);				/* return removed pve */
+}
 /*
+ *
  * pmap_modify_pv: Update pv flags
+ *
  * => caller should hold lock on vm_page [so that attrs can be adjusted]
  * => caller should NOT adjust pmap's wire_count
  * => caller must call pmap_vac_me_harder() if writable status of a page
  *    may have changed.
  * => we return the old flags
+ *
 @@ -1025,26 +1060,28 @@ pmap_modify_pv(struct vm_page *pg, pmap_
 				pg->mdpage.kro_mappings++;
 				pg->mdpage.krw_mappings--;
+			}
 		} else
 		if (flags & PVF_WRITE) {
 			pg->mdpage.urw_mappings++;
 			pg->mdpage.uro_mappings--;
 		} else {
 			pg->mdpage.uro_mappings++;
 			pg->mdpage.urw_mappings--;
+		}
+	}
 #ifdef PMAP_CACHE_VIPT
 	if (pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0)
 		pg->mdpage.pvh_attrs &= ~PVF_WRITE;
 	/*
 	 * We have two cases here: the first is from enter_pv (new exec
 	 * page), the second is a combined pmap_remove_pv/pmap_enter_pv.
 	 * Since in latter, pmap_enter_pv won't do anything, we just have
 	 * to do what pmap_remove_pv would do.
 	 */
 	if ((PV_IS_EXEC_P(flags) && !PV_IS_EXEC_P(pg->mdpage.pvh_attrs))
 	    || (PV_IS_EXEC_P(pg->mdpage.pvh_attrs)
 		|| (!(flags & PVF_WRITE) && (oflags & PVF_WRITE)))) {
 		pmap_syncicache_page(pg);
 		PMAPCOUNT(exec_synced_remap);
+	}
 #endif
 @@ -1761,188 +1798,189 @@ pmap_check_sets(paddr_t pa)
 			asm("mrc	p15, 3, %0, c15, c0, 0" : "=r"(v));
 			if ((v & (1 | ~(PAGE_SIZE-1))) == pa) {
 				mask |= 1 << (set >> 7);
+			}
+		}
+	}
 	return mask;
+}
 #endif
 static void
 pmap_vac_me_harder(struct vm_page *pg, pmap_t pm, vaddr_t va)
+{
-	struct pv_entry *pv, pv0;
+	struct pv_entry *pv;
 	vaddr_t tst_mask;
 	bool bad_alias;
 	struct l2_bucket *l2b;
 	pt_entry_t *ptep, pte, opte;
 	/* do we need to do anything? */
 	if (arm_cache_prefer_mask == 0)
 		return;
 	NPDEBUG(PDB_VAC, printf("pmap_vac_me_harder: pg=%p, pmap=%p va=%08lx\n",
 	    pg, pm, va));
 #define popc4(x) \
 	(((0x94 >> ((x & 3) << 1)) & 3) + ((0x94 >> ((x & 12) >> 1)) & 3))
 #if 0
 	tst_mask = pmap_check_sets(pg->phys_addr);
 	KASSERT(popc4(tst_mask) < 2);
 #endif
-	KASSERT(!va || pm || (pg->mdpage.pvh_attrs & PVF_KENTRY));
+	KASSERT(!va || pm);
 	/* Already a conflict? */
 	if (__predict_false(pg->mdpage.pvh_attrs & PVF_NC)) {
 		/* just an add, things are already non-cached */
 		bad_alias = false;
 		if (va) {
 			PMAPCOUNT(vac_color_none);
 			bad_alias = true;
 			KASSERT((pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE));
 			goto fixup;
+		}
 		pv = pg->mdpage.pvh_list;
 		/* the list can't be empty because it would be cachable */
-		if (pg->mdpage.pvh_attrs & PVF_KENTRY) {
+		if (pg->mdpage.pvh_attrs & PVF_KMPAGE) {
 			tst_mask = pg->mdpage.pvh_attrs;
 		} else {
 			KASSERT(pv);
 			tst_mask = pv->pv_va;
 			pv = pv->pv_next;
+		}
 		/*
 		 * Only check for a bad alias if we have writable mappings.
 		 */
-		if (pg->mdpage.urw_mappings + pg->mdpage.krw_mappings > 0
+		if (pg->mdpage.urw_mappings + pg->mdpage.krw_mappings > 0) {
 		    || (pg->mdpage.pvh_attrs & PVF_KENTRY)) {
 			tst_mask &= arm_cache_prefer_mask;
 			for (; pv && !bad_alias; pv = pv->pv_next) {
 				/* if there's a bad alias, stop checking. */
 				if (tst_mask != (pv->pv_va & arm_cache_prefer_mask))
 					bad_alias = true;
+			}
 			pg->mdpage.pvh_attrs |= PVF_WRITE;
+		}
 		/* If no conflicting colors, set everything back to cached */
 		if (!bad_alias) {
 			PMAPCOUNT(vac_color_restore);
-			pg->mdpage.pvh_attrs |= PVF_COLORED;
+			pg->mdpage.pvh_attrs &= (PAGE_SIZE - 1) & ~PVF_NC;
-			if (!(pg->mdpage.pvh_attrs & PVF_KENTRY)) {
+			pg->mdpage.pvh_attrs |= tst_mask | PVF_COLORED;
 				pg->mdpage.pvh_attrs &= PAGE_SIZE - 1;
 				pg->mdpage.pvh_attrs |= tst_mask;
 			pg->mdpage.pvh_attrs &= ~PVF_NC;
 		} else {
 			KASSERT(pg->mdpage.pvh_list != NULL);
-			KASSERT((pg->mdpage.pvh_attrs & PVF_KENTRY)
+			KASSERT(pg->mdpage.pvh_list->pv_next != NULL);
 			     || pg->mdpage.pvh_list->pv_next != NULL);
+		}
 		KASSERT((pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE));
 	} else if (!va) {
 		KASSERT(pmap_is_page_colored_p(pg));
-		if (pm == NULL)
+		pg->mdpage.pvh_attrs &= (PAGE_SIZE - 1) | arm_cache_prefer_mask;
-			pg->mdpage.pvh_attrs &=
+		if (pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0)
-			    (PAGE_SIZE - 1) | arm_cache_prefer_mask;
+			pg->mdpage.pvh_attrs &= ~PVF_WRITE;
 		KASSERT((pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE));
 		return;
 	} else if (!pmap_is_page_colored_p(pg)) {
 		/* not colored so we just use its color */
 		PMAPCOUNT(vac_color_new);
 		pg->mdpage.pvh_attrs &= PAGE_SIZE - 1;
-		if (pm == NULL)
+		pg->mdpage.pvh_attrs |= PVF_COLORED
-			pg->mdpage.pvh_attrs |= PVF_COLORED | va;
+		    | (va & arm_cache_prefer_mask);
-		else
+		if (pg->mdpage.urw_mappings + pg->mdpage.krw_mappings > 0)
-			pg->mdpage.pvh_attrs |= PVF_COLORED
+			pg->mdpage.pvh_attrs |= PVF_WRITE;
-			    | (va & arm_cache_prefer_mask);
+		KASSERT((pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE));
 		return;
-	} else if (!((pg->mdpage.pvh_attrs ^ va) & arm_cache_prefer_mask)
+	} else if (((pg->mdpage.pvh_attrs ^ va) & arm_cache_prefer_mask) == 0) {
 		bad_alias = false;
 		if (pg->mdpage.urw_mappings + pg->mdpage.krw_mappings > 0) {
 			/*
 			 * We now have writeable mappings and more than one
 			 * readonly mapping, verify the colors don't clash
 			 * and mark the page as writeable.
 			 */
 			if (pg->mdpage.uro_mappings + pg->mdpage.kro_mappings > 1
 			    && (pg->mdpage.pvh_attrs & PVF_WRITE) == 0) {
 				tst_mask = pg->mdpage.pvh_attrs & arm_cache_prefer_mask;
 				for (pv = pg->mdpage.pvh_list;
 				     pv && !bad_alias;
 				     pv = pv->pv_next) {
 					/* if there's a bad alias, stop checking. */
 					if (tst_mask != (pv->pv_va & arm_cache_prefer_mask))
 						bad_alias = true;
+				}
+			}
 			pg->mdpage.pvh_attrs |= PVF_WRITE;
+		}
 		/* If no conflicting colors, set everything back to cached */
 		if (!bad_alias) {
 			if (pg->mdpage.pvh_list)
 				PMAPCOUNT(vac_color_reuse);
 			else
 				PMAPCOUNT(vac_color_ok);
 			/* matching color, just return */
 			KASSERT((pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE));
 			return;
+		}
 		KASSERT(pg->mdpage.pvh_list != NULL);
 		KASSERT(pg->mdpage.pvh_list->pv_next != NULL);
 		/* color conflict.  evict from cache. */
 		pmap_flush_page(pg);
 		pg->mdpage.pvh_attrs &= ~PVF_COLORED;
 		pg->mdpage.pvh_attrs |= PVF_NC;
 	} else if (pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0
 		   && (pg->mdpage.pvh_attrs & PVF_KMPAGE) == 0) {
 		KASSERT((pg->mdpage.pvh_attrs & PVF_WRITE) == 0);
 		/*
-		 * If the VA matches the existing color or if all the mappings
+		 * If all the mappings are read-only, don't do anything.
 		 * are read-only, don't do anything.
 		 */
-		|| (pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0
+		PMAPCOUNT(vac_color_blind);
-		    && (pg->mdpage.pvh_attrs & PVF_KENTRY) == 0)) {
+		KASSERT((pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE));
 		if (pm == NULL) {
 			pg->mdpage.pvh_attrs &= PAGE_SIZE - 1;
 			pg->mdpage.pvh_attrs |= va;
 		if (pg->mdpage.pvh_list)
 			PMAPCOUNT(vac_color_reuse);
 		else
 			PMAPCOUNT(vac_color_ok);
 		/* matching color, just return */
 		return;
 	} else {
 		if (pg->mdpage.urw_mappings + pg->mdpage.krw_mappings > 0)
 			pg->mdpage.pvh_attrs |= PVF_WRITE;
 		/* color conflict.  evict from cache. */
 		pmap_flush_page(pg);
 		/* the list can't be empty because this was a enter/modify */
 		pv = pg->mdpage.pvh_list;
-		KASSERT((pg->mdpage.pvh_attrs & PVF_KENTRY) || pv);
+		KASSERT(pv);
 		/*
 		 * If there's only one mapped page, change color to the
 		 * page's new color and return.
 		 */
-		if (((pg->mdpage.pvh_attrs & PVF_KENTRY)
+		if (pv->pv_next == NULL) {
 		    ? pv : pv->pv_next) == NULL) {
 			PMAPCOUNT(vac_color_change);
 			pg->mdpage.pvh_attrs &= PAGE_SIZE - 1;
-			if (pm == NULL)
+			pg->mdpage.pvh_attrs |= (va & arm_cache_prefer_mask);
-				pg->mdpage.pvh_attrs |= va;
+			KASSERT((pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE));
 			else
 				pg->mdpage.pvh_attrs |=
 				    (va & arm_cache_prefer_mask);
 			return;
+		}
 		bad_alias = true;
 		pg->mdpage.pvh_attrs &= ~PVF_COLORED;
 		pg->mdpage.pvh_attrs |= PVF_NC;
 		PMAPCOUNT(vac_color_erase);
+	}
   fixup:
-	/*
+	KASSERT((pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE));
 	 * If the pmap is NULL, then we got called from pmap_kenter_pa
 	 * and we must save the kenter'ed va.  And this changes the
 	 * color to match the kenter'ed page.  if this is a remove clear
 	 * saved va bits which retaining the color bits.
 	 */
 	if (pm == NULL) {
 		if (va) {
 			pg->mdpage.pvh_attrs &= (PAGE_SIZE - 1);
 			pg->mdpage.pvh_attrs |= va;
 		} else {
 			pg->mdpage.pvh_attrs &=
 			    ((PAGE_SIZE - 1) | arm_cache_prefer_mask);
 	pv = pg->mdpage.pvh_list;
 	/*
 	 * If this page has an kenter'ed mapping, fake up a pv entry.
 	 */
 	if (__predict_false(pg->mdpage.pvh_attrs & PVF_KENTRY)) {
 		pv0.pv_pmap = pmap_kernel();
 		pv0.pv_va = pg->mdpage.pvh_attrs & ~(PAGE_SIZE - 1);
 		pv0.pv_next = pv;
 		pv0.pv_flags = PVF_REF;
 		pv = &pv0;
 	/*
 	 * Turn cacheing on/off for all pages.
 	 */
-	for (; pv; pv = pv->pv_next) {
+	for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) {
 		l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
 		ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
 		opte = *ptep;
 		pte = opte & ~L2_S_CACHE_MASK;
 		if (bad_alias) {
 			pv->pv_flags |= PVF_NC;
 		} else {
 			pv->pv_flags &= ~PVF_NC;
 			pte |= pte_l2_s_cache_mode;
+		}
 		if (opte == pte)	/* only update is there's a change */
 			continue;
 @@ -2082,26 +2120,28 @@ pmap_clearbit(struct vm_page *pg, u_int
 			if (maskbits & oflags & PVF_WRITE) {
 				/*
 				 * Keep alias accounting up to date
 				 */
 				if (pv->pv_pmap == pmap_kernel()) {
 					pg->mdpage.krw_mappings--;
 					pg->mdpage.kro_mappings++;
 				} else {
 					pg->mdpage.urw_mappings--;
 					pg->mdpage.uro_mappings++;
+				}
 #ifdef PMAP_CACHE_VIPT
 				if (pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0)
 					pg->mdpage.pvh_attrs &= ~PVF_WRITE;
 				if (want_syncicache)
 					need_syncicache = true;
 #endif
+			}
+		}
 		if (maskbits & PVF_REF) {
 #ifdef PMAP_CACHE_VIVT
 			if ((pv->pv_flags & PVF_NC) == 0 &&
 			    (maskbits & (PVF_WRITE|PVF_MOD)) == 0 &&
 			    l2pte_valid(npte)) {
 				/*
 				 * Check npte here; we may have already
 @@ -2192,30 +2232,27 @@ pmap_clean_page(struct pv_entry *pv, boo
 	u_int flags = 0;
 	vaddr_t page_to_clean = 0;
 	if (pv == NULL) {
 		/* nothing mapped in so nothing to flush */
 		return (0);
+	}
 	/*
 	 * Since we flush the cache each time we change to a different
 	 * user vmspace, we only need to flush the page if it is in the
 	 * current pmap.
 	 */
-	if (curproc)
+	pm = curproc->p_vmspace->vm_map.pmap;
 		pm = curproc->p_vmspace->vm_map.pmap;
 	else
 		pm = pmap_kernel();
 	for (npv = pv; npv; npv = npv->pv_next) {
 		if (npv->pv_pmap == pmap_kernel() || npv->pv_pmap == pm) {
 			flags |= npv->pv_flags;
 			/*
 			 * The page is mapped non-cacheable in
 			 * this map.  No need to flush the cache.
 			 */
 			if (npv->pv_flags & PVF_NC) {
 #ifdef DIAGNOSTIC
 				if (cache_needs_cleaning)
 					panic("pmap_clean_page: "
 					    "cache inconsistency");
 @@ -2346,82 +2383,102 @@ pmap_flush_page(struct vm_page *pg)
 #endif /* PMAP_CACHE_VIPT */
 /*
  * Routine:	pmap_page_remove
  * Function:
  *		Removes this physical page from
  *		all physical maps in which it resides.
  *		Reflects back modify bits to the pager.
  */
 static void
 pmap_page_remove(struct vm_page *pg)
+{
 	struct l2_bucket *l2b;
-	struct pv_entry *pv, *npv;
+	struct pv_entry *pv, *npv, **pvp;
 	pmap_t pm, curpm;
 	pt_entry_t *ptep, pte;
 	bool flush;
 	u_int flags;
 	NPDEBUG(PDB_FOLLOW,
 	    printf("pmap_page_remove: pg %p (0x%08lx)\n", pg,
 	    VM_PAGE_TO_PHYS(pg)));
 	PMAP_HEAD_TO_MAP_LOCK();
 	simple_lock(&pg->mdpage.pvh_slock);
 	pv = pg->mdpage.pvh_list;
 	if (pv == NULL) {
 #ifdef PMAP_CACHE_VIPT
 		/*
 		 * We *know* the page contents are about to be replaced.
 		 * Discard the exec contents
 		 */
 		if (PV_IS_EXEC_P(pg->mdpage.pvh_attrs))
 			PMAPCOUNT(exec_discarded_page_protect);
 		pg->mdpage.pvh_attrs &= ~PVF_EXEC;
 		KASSERT((pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE));
 #endif
 		simple_unlock(&pg->mdpage.pvh_slock);
 		PMAP_HEAD_TO_MAP_UNLOCK();
 		return;
+	}
 #ifdef PMAP_CACHE_VIPT
 	KASSERT(pmap_is_page_colored_p(pg));
 #endif
 	/*
 	 * Clear alias counts
 	 */
 #ifdef PMAP_CACHE_VIVT
 	pg->mdpage.k_mappings = 0;
 #endif
 	pg->mdpage.urw_mappings = pg->mdpage.uro_mappings = 0;
 	flush = false;
 	flags = 0;
-	if (curproc)
+	curpm = curproc->p_vmspace->vm_map.pmap;
 		curpm = curproc->p_vmspace->vm_map.pmap;
 	else
 		curpm = pmap_kernel();
 #ifdef PMAP_CACHE_VIVT
 	pmap_clean_page(pv, false);
 #endif
 	pvp = &pg->mdpage.pvh_list;
 	while (pv) {
 		pm = pv->pv_pmap;
 		npv = pv->pv_next;
 		if (flush == false && (pm == curpm || pm == pmap_kernel()))
 			flush = true;
-		if (pm == pmap_kernel())
+		if (pm == pmap_kernel()) {
 #ifdef PMAP_CACHE_VIPT
 			/*
 			 * If this was unmanaged mapping, it must be preserved.
 			 * Move it back on the list and advance the end-of-list
 			 * pointer.
 			 */
 			if (pv->pv_flags & PVF_KENTRY) {
 				*pvp = pv;
 				pvp = &pv->pv_next;
 				pv = npv;
 				continue;
+			}
 			if (pv->pv_flags & PVF_WRITE)
 				pg->mdpage.krw_mappings--;
 			else
 				pg->mdpage.kro_mappings--;
 #endif
 			PMAPCOUNT(kernel_unmappings);
+		}
 		PMAPCOUNT(unmappings);
 		pmap_acquire_pmap_lock(pm);
 		l2b = pmap_get_l2_bucket(pm, pv->pv_va);
 		KDASSERT(l2b != NULL);
 		ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
 		pte = *ptep;
 		/*
 		 * Update statistics
 		 */
 @@ -2430,47 +2487,49 @@ pmap_page_remove(struct vm_page *pg)
 		/* Wired bit */
 		if (pv->pv_flags & PVF_WIRED)
 			--pm->pm_stats.wired_count;
 		flags |= pv->pv_flags;
 		/*
 		 * Invalidate the PTEs.
 		 */
 		*ptep = 0;
 		PTE_SYNC_CURRENT(pm, ptep);
 		pmap_free_l2_bucket(pm, l2b, 1);
 		npv = pv->pv_next;
 		pool_put(&pmap_pv_pool, pv);
 		pv = npv;
 		/*
 		 * if we reach the end of the list and there are still
 		 * mappings, they might be able to be cached now.
 		 */
 		if (pv == NULL) {
-			pg->mdpage.pvh_list = NULL;
+			*pvp = NULL;
-			if (pg->mdpage.pvh_attrs & PVF_KENTRY)
+			if (pg->mdpage.pvh_list != NULL)
 				pmap_vac_me_harder(pg, pm, 0);
+		}
 		pmap_release_pmap_lock(pm);
+	}
 #ifdef PMAP_CACHE_VIPT
 	/*
-	 * Since there are now no mappings, there isn't reason to mark it
+	 * Its EXEC cache is now gone.
 	 * as uncached.  Its EXEC cache is also gone.
 	 */
 	if (PV_IS_EXEC_P(pg->mdpage.pvh_attrs))
 		PMAPCOUNT(exec_discarded_page_protect);
-	pg->mdpage.pvh_attrs &= ~(PVF_NC|PVF_EXEC);
+	pg->mdpage.pvh_attrs &= ~PVF_EXEC;
-#endif
+	if (pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0)
-#ifdef PMAP_CACHE_VIVT
+		pg->mdpage.pvh_attrs &= ~PVF_WRITE;
-	pg->mdpage.pvh_list = NULL;
+	KASSERT((pg->mdpage.urw_mappings + pg->mdpage.krw_mappings == 0) == !(pg->mdpage.pvh_attrs & PVF_WRITE));
 #endif
 	simple_unlock(&pg->mdpage.pvh_slock);
 	PMAP_HEAD_TO_MAP_UNLOCK();
 	if (flush) {
 		/*
 		 * Note: We can't use pmap_tlb_flush{I,}D() here since that
 		 * would need a subsequent call to pmap_update() to ensure
 		 * curpm->pm_cstate.cs_all is reset. Our callers are not
 		 * required to do that (see pmap(9)), so we can't modify
 		 * the current pmap's state.
 		 */
 		if (PV_BEEN_EXECD(flags))
 @@ -3026,119 +3085,154 @@ pmap_do_remove(pmap_t pm, vaddr_t sva, v
 #endif
 				pm->pm_remove_all = true;
+			}
+		}
 		pmap_free_l2_bucket(pm, l2b, mappings);
 		pm->pm_stats.resident_count -= mappings;
+	}
 	pmap_release_pmap_lock(pm);
 	PMAP_MAP_TO_HEAD_UNLOCK();
+}
 #ifdef PMAP_CACHE_VIPT
 static struct pv_entry *
 pmap_kremove_pg(struct vm_page *pg, vaddr_t va)
+{
 	struct pv_entry *pv;
 	simple_lock(&pg->mdpage.pvh_slock);
 	KASSERT(pg->mdpage.pvh_attrs & (PVF_COLORED|PVF_NC));
 	KASSERT((pg->mdpage.pvh_attrs & PVF_KMPAGE) == 0);
 	pv = pmap_remove_pv(pg, pmap_kernel(), va, false);
 	KASSERT(pv);
 	KASSERT(pv->pv_flags & PVF_KENTRY);
 	/*
 	 * If we are removing a writeable mapping to a cached exec page,
 	 * if it's the last mapping then clear it execness other sync
 	 * the page to the icache.
 	 */
 	if ((pg->mdpage.pvh_attrs & (PVF_NC|PVF_EXEC)) == PVF_EXEC
 	    && (pv->pv_flags & PVF_WRITE) != 0) {
 		if (pg->mdpage.pvh_list == NULL) {
 			pg->mdpage.pvh_attrs &= ~PVF_EXEC;
 			PMAPCOUNT(exec_discarded_kremove);
 		} else {
 			pmap_syncicache_page(pg);
 			PMAPCOUNT(exec_synced_kremove);
+		}
+	}
 	pmap_vac_me_harder(pg, pmap_kernel(), 0);
 	simple_unlock(&pg->mdpage.pvh_slock);
 	return pv;
+}
 #endif /* PMAP_CACHE_VIPT */
 /*
  * pmap_kenter_pa: enter an unmanaged, wired kernel mapping
+ *
  * We assume there is already sufficient KVM space available
  * to do this, as we can't allocate L2 descriptor tables/metadata
  * from here.
  */
 void
 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot)
+{
 	struct l2_bucket *l2b;
 	pt_entry_t *ptep, opte;
 #ifdef PMAP_CACHE_VIPT
 	struct vm_page *pg = PHYS_TO_VM_PAGE(pa);
 	struct vm_page *opg;
 	struct pv_entry *pv = NULL;
 #endif
 	NPDEBUG(PDB_KENTER,
 	    printf("pmap_kenter_pa: va 0x%08lx, pa 0x%08lx, prot 0x%x\n",
 	    va, pa, prot));
 	l2b = pmap_get_l2_bucket(pmap_kernel(), va);
 	KDASSERT(l2b != NULL);
 	ptep = &l2b->l2b_kva[l2pte_index(va)];
 	opte = *ptep;
 	if (opte == 0) {
 		PMAPCOUNT(kenter_mappings);
 		l2b->l2b_occupancy++;
 	} else {
 		PMAPCOUNT(kenter_remappings);
 #ifdef PMAP_CACHE_VIPT
 		opg = PHYS_TO_VM_PAGE(l2pte_pa(opte));
 		if (opg) {
 			KASSERT(opg != pg);
 			KASSERT((opg->mdpage.pvh_attrs & PVF_KMPAGE) == 0);
 			KASSERT((prot & PMAP_KMPAGE) == 0);
 			simple_lock(&opg->mdpage.pvh_slock);
-			KASSERT(opg->mdpage.pvh_attrs & PVF_KENTRY);
+			pv = pmap_kremove_pg(opg, va);
 			if (PV_IS_EXEC_P(opg->mdpage.pvh_attrs)
 			    && !(opg->mdpage.pvh_attrs & PVF_NC)) {
 				if (opg->mdpage.pvh_list == NULL) {
 					opg->mdpage.pvh_attrs &= ~PVF_EXEC;
 					PMAPCOUNT(exec_discarded_kremove);
 				} else {
 					pmap_syncicache_page(opg);
 					PMAPCOUNT(exec_synced_kremove);
 			KASSERT(opg->mdpage.pvh_attrs & (PVF_COLORED|PVF_NC));
 			if (L2_AP(AP_W) & opte) {
 				KASSERT(opg->mdpage.pvh_attrs & PVF_KENTRY);
 				opg->mdpage.pvh_attrs &= ~PVF_KENTRY;
 				pmap_vac_me_harder(opg, NULL, 0);
 			} else {
 				KASSERT(opg->mdpage.kro_mappings > 0);
 				opg->mdpage.kro_mappings--;
 			simple_unlock(&opg->mdpage.pvh_slock);
+		}
 #endif
 		if (l2pte_valid(opte)) {
 #ifdef PMAP_CACHE_VIVT
 			cpu_dcache_wbinv_range(va, PAGE_SIZE);
 #endif
 			cpu_tlb_flushD_SE(va);
 			cpu_cpwait();
+		}
+	}
 	*ptep = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) |
 	    pte_l2_s_cache_mode;
 	PTE_SYNC(ptep);
 #ifdef PMAP_CACHE_VIPT
 	if (pg) {
-		simple_lock(&pg->mdpage.pvh_slock);
+		if (prot & PMAP_KMPAGE) {
-		if (prot & VM_PROT_WRITE) {
+			KASSERT(pv == NULL);
-			/*
+			KASSERT((va & PVF_COLORED) == 0);
-			 * If they want a writeable page, make sure it
+			simple_lock(&pg->mdpage.pvh_slock);
 			 * isn't already mapped in the kernel.
 			 */
 			KASSERT((pg->mdpage.pvh_attrs & PVF_KENTRY) == 0);
 			KASSERT(pg->mdpage.kro_mappings == 0);
 			pg->mdpage.pvh_attrs |= PVF_KENTRY;
 			pmap_vac_me_harder(pg, NULL, va);
 		} else {
 			KASSERT(pg->mdpage.krw_mappings == 0);
 			KASSERT(pg->mdpage.urw_mappings == 0);
 			KASSERT(pg->mdpage.uro_mappings == 0);
 			KASSERT(pg->mdpage.krw_mappings == 0);
 			KASSERT(pg->mdpage.kro_mappings == 0);
 			KASSERT((pg->mdpage.pvh_attrs & PVF_NC) == 0);
-			pg->mdpage.kro_mappings++;
+			/* if there is a color conflict, evict from cache. */
 			if (pmap_is_page_colored_p(pg)
 			    && ((va ^ pg->mdpage.pvh_attrs) & arm_cache_prefer_mask))
 				pmap_flush_page(pg);
 			pg->mdpage.pvh_attrs &= PAGE_SIZE - 1;
 			pg->mdpage.pvh_attrs |= PVF_KMPAGE
 				| PVF_COLORED
 				| (va & arm_cache_prefer_mask);
 			simple_unlock(&pg->mdpage.pvh_slock);
 		} else {
 			if (pv == NULL) {
 				pv = pool_get(&pmap_pv_pool, PR_NOWAIT);
 				KASSERT(pv != NULL);
+			}
 			pmap_enter_pv(pg, pv, pmap_kernel(), va,
 			    PVF_WIRED | PVF_KENTRY
 				| (prot & VM_PROT_WRITE ? PVF_WRITE : 0));
 			simple_lock(&pg->mdpage.pvh_slock);
 			pmap_vac_me_harder(pg, pmap_kernel(), va);
 			simple_unlock(&pg->mdpage.pvh_slock);
+		}
-		simple_unlock(&pg->mdpage.pvh_slock);
+	} else {
 		if (pv != NULL)
 			pool_put(&pmap_pv_pool, pv);
+	}
 #endif
+}
 void
 pmap_kremove(vaddr_t va, vsize_t len)
+{
 	struct l2_bucket *l2b;
 	pt_entry_t *ptep, *sptep, opte;
 	vaddr_t next_bucket, eva;
 	u_int mappings;
 #ifdef PMAP_CACHE_VIPT
 	struct vm_page *opg;
 @@ -3157,48 +3251,39 @@ pmap_kremove(vaddr_t va, vsize_t len)
 			next_bucket = eva;
 		l2b = pmap_get_l2_bucket(pmap_kernel(), va);
 		KDASSERT(l2b != NULL);
 		sptep = ptep = &l2b->l2b_kva[l2pte_index(va)];
 		mappings = 0;
 		while (va < next_bucket) {
 			opte = *ptep;
 #ifdef PMAP_CACHE_VIPT
 			opg = PHYS_TO_VM_PAGE(l2pte_pa(opte));
 			if (opg) {
-				simple_lock(&opg->mdpage.pvh_slock);
+				if (opg->mdpage.pvh_attrs & PVF_KMPAGE) {
-				if (PV_IS_EXEC_P(opg->mdpage.pvh_attrs)
+					simple_lock(&opg->mdpage.pvh_slock);
-				    && !(opg->mdpage.pvh_attrs & PVF_NC)) {
+					KASSERT(opg->mdpage.urw_mappings == 0);
-					if (opg->mdpage.pvh_list == NULL) {
+					KASSERT(opg->mdpage.uro_mappings == 0);
-						opg->mdpage.pvh_attrs &=
+					KASSERT(opg->mdpage.krw_mappings == 0);
-						    ~PVF_EXEC;
+					KASSERT(opg->mdpage.kro_mappings == 0);
-						PMAPCOUNT(exec_discarded_kremove);
+					opg->mdpage.pvh_attrs &=
-					} else {
+					    ~(PVF_KMPAGE|PVF_WRITE);
-						pmap_syncicache_page(opg);
+					simple_unlock(&opg->mdpage.pvh_slock);
 						PMAPCOUNT(exec_synced_kremove);
 				KASSERT(opg->mdpage.pvh_attrs & (PVF_COLORED|PVF_NC));
 				if (L2_AP(AP_W) & opte) {
 					KASSERT(opg->mdpage.pvh_attrs & PVF_KENTRY);
 					opg->mdpage.pvh_attrs &= ~PVF_KENTRY;
 					pmap_vac_me_harder(opg, NULL, 0);
 				} else {
-					KASSERT(opg->mdpage.kro_mappings > 0);
+					pool_put(&pmap_pv_pool,
-					opg->mdpage.kro_mappings--;
+					    pmap_kremove_pg(opg, va));
+				}
 				simple_unlock(&opg->mdpage.pvh_slock);
+			}
 #endif
 			if (l2pte_valid(opte)) {
 #ifdef PMAP_CACHE_VIVT
 				cpu_dcache_wbinv_range(va, PAGE_SIZE);
 #endif
 				cpu_tlb_flushD_SE(va);
+			}
 			if (opte) {
 				*ptep = 0;
 				mappings++;
+			}
 			va += PAGE_SIZE;
 @@ -4453,51 +4538,31 @@ pmap_grow_map(vaddr_t va, pt_entry_t cac
 		pa = pv.pv_pa;
 #else
 		if (uvm_page_physget(&pa) == false)
 			return (1);
 #endif	/* PMAP_STEAL_MEMORY */
 	} else {
 		struct vm_page *pg;
 		pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
 		if (pg == NULL)
 			return (1);
 		pa = VM_PAGE_TO_PHYS(pg);
 #ifdef PMAP_CACHE_VIPT
 		/*
-		 * This new page must not have any mappings.  However, it might
+		 * This new page must not have any mappings.  Enter it via
-		 * have previously used and therefore present in the cache.  If
+		 * pmap_kenter_pa and let that routine do the hard work.
 		 * it doesn't have the desired color, we have to flush it from
 		 * the cache.  And while we are at it, make sure to clear its
 		 * EXEC status.
 		 */
 		KASSERT(!(pg->mdpage.pvh_attrs & PVF_KENTRY));
 		KASSERT(pg->mdpage.pvh_list == NULL);
-		if (pmap_is_page_colored_p(pg)) {
+		pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE|PMAP_KMPAGE);
 			if ((va ^ pg->mdpage.pvh_attrs) & arm_cache_prefer_mask) {
 				pmap_flush_page(pg);
 				PMAPCOUNT(vac_color_change);
 			} else {
 				PMAPCOUNT(vac_color_reuse);
 		} else {
 			PMAPCOUNT(vac_color_new);
 		if (PV_IS_EXEC_P(pg->mdpage.pvh_attrs))
 			PMAPCOUNT(exec_discarded_kremove);
 		/*
 		 * We'll pretend this page was entered by pmap_kenter_pa
 		 */
 		pg->mdpage.pvh_attrs &= (PAGE_SIZE - 1) & ~PVF_EXEC;
 		pg->mdpage.pvh_attrs |= va | PVF_KENTRY | PVF_COLORED | PVF_REF | PVF_MOD;
 #endif
+	}
 	if (pap)
 		*pap = pa;
 	PMAPCOUNT(pt_mappings);
 	l2b = pmap_get_l2_bucket(pmap_kernel(), va);
 	KDASSERT(l2b != NULL);
 	ptep = &l2b->l2b_kva[l2pte_index(va)];
 	*ptep = L2_S_PROTO | pa | cache_mode |
 	    L2_S_PROT(PTE_KERNEL, VM_PROT_READ | VM_PROT_WRITE);
 @@ -5160,27 +5225,28 @@ pmap_postinit(void)
 		error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start,
 		    physical_end, L1_TABLE_SIZE, 0, &plist, 1, M_WAITOK);
 		if (error)
 			panic("Cannot allocate L1 physical pages");
 		m = TAILQ_FIRST(&plist);
 		eva = va + L1_TABLE_SIZE;
 		pl1pt = (pd_entry_t *)va;
 		while (m && va < eva) {
 			paddr_t pa = VM_PAGE_TO_PHYS(m);
-			pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE);
+			pmap_kenter_pa(va, pa,
 			    VM_PROT_READ|VM_PROT_WRITE|PMAP_KMPAGE);
 			/*
 			 * Make sure the L1 descriptor table is mapped
 			 * with the cache-mode set to write-through.
 			 */
 			l2b = pmap_get_l2_bucket(pmap_kernel(), va);
 			ptep = &l2b->l2b_kva[l2pte_index(va)];
 			pte = *ptep;
 			pte = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt;
 			*ptep = pte;
 			PTE_SYNC(ptep);
 			cpu_tlb_flushD_SE(va);

 @@ -1,14 +1,14 @@
-/*	$NetBSD: pmap.h,v 1.85 2008/04/27 18:58:44 matt Exp $	*/
+/*	$NetBSD: pmap.h,v 1.86 2008/07/16 00:19:57 matt Exp $	*/
 /*
  * Copyright (c) 2002, 2003 Wasabi Systems, Inc.
  * All rights reserved.
+ *
  * Written by Jason R. Thorpe & Steve C. Woodford for Wasabi Systems, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
 @@ -226,26 +226,27 @@ extern pv_addr_t kernel_l1pt;
+ *
  * Note the "non-cacheable" flag generally means the page has
  * multiple mappings in a given address space.
  */
 #define	PVF_MOD		0x01		/* page is modified */
 #define	PVF_REF		0x02		/* page is referenced */
 #define	PVF_WIRED	0x04		/* mapping is wired */
 #define	PVF_WRITE	0x08		/* mapping is writable */
 #define	PVF_EXEC	0x10		/* mapping is executable */
 #define	PVF_UNC		0x20		/* mapping is 'user' non-cacheable */
 #define	PVF_KNC		0x40		/* mapping is 'kernel' non-cacheable */
 #define	PVF_COLORED	0x80		/* page has or had a color */
 #define	PVF_KENTRY	0x0100		/* page entered via pmap_kenter_pa */
 #define	PVF_KMPAGE	0x0200		/* page is used for kmem */
 #define	PVF_NC		(PVF_UNC|PVF_KNC)
 /*
  * Commonly referenced structures
  */
 extern struct pmap	kernel_pmap_store;
 extern int		pmap_debug_level; /* Only exists if PMAP_DEBUG */
 /*
  * Macros that we need to export
  */
 #define pmap_kernel()			(&kernel_pmap_store)
 #define	pmap_resident_count(pmap)	((pmap)->pm_stats.resident_count)