 @@ -1,14 +1,14 @@
-/*	$NetBSD: pmap.c,v 1.179.16.12 2010/01/22 07:41:10 matt Exp $	*/
+/*	$NetBSD: pmap.c,v 1.179.16.13 2010/01/26 21:19:25 matt Exp $	*/
 /*-
  * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
  * NASA Ames Research Center and by Chris G. Demetriou.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
 @@ -57,27 +57,27 @@
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  *	@(#)pmap.c	8.4 (Berkeley) 1/26/94
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.179.16.12 2010/01/22 07:41:10 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.179.16.13 2010/01/26 21:19:25 matt Exp $");
 /*
  *	Manages physical address maps.
+ *
  *	In addition to hardware address maps, this
  *	module is called upon to provide software-use-only
  *	maps which may or may not be stored in the same
  *	form as hardware maps.  These pseudo-maps are
  *	used to store intermediate results from copy
  *	operations to and from address spaces.
+ *
  *	Since the information managed by this module is
  *	also stored by the logical address mapping module,
 @@ -141,212 +141,303 @@ CTASSERT((intptr_t)MIPS_PHYS_TO_KSEG0(0x
 CTASSERT(MIPS_KSEG1_START < 0);
 CTASSERT((intptr_t)MIPS_PHYS_TO_KSEG1(0x1000) < 0);
 CTASSERT(MIPS_KSEG2_START < 0);
 CTASSERT(MIPS_MAX_MEM_ADDR < 0);
 CTASSERT(MIPS_RESERVED_ADDR < 0);
 CTASSERT((uint32_t)MIPS_KSEG0_START == 0x80000000);
 CTASSERT((uint32_t)MIPS_KSEG1_START == 0xa0000000);
 CTASSERT((uint32_t)MIPS_KSEG2_START == 0xc0000000);
 CTASSERT((uint32_t)MIPS_MAX_MEM_ADDR == 0xbe000000);
 CTASSERT((uint32_t)MIPS_RESERVED_ADDR == 0xbfc80000);
 CTASSERT(MIPS_KSEG0_P(MIPS_PHYS_TO_KSEG0(0)));
 CTASSERT(MIPS_KSEG1_P(MIPS_PHYS_TO_KSEG1(0)));
-#ifdef DEBUG
+#define	PMAP_COUNT(name)	(pmap_evcnt_##name.ev_count++ + 0)
-struct {
+#define PMAP_COUNTER(name, desc) \
-	int kernel;	/* entering kernel mapping */
+static struct evcnt pmap_evcnt_##name = \
-	int user;	/* entering user mapping */
+	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", desc); \
-	int ptpneeded;	/* needed to allocate a PT page */
+EVCNT_ATTACH_STATIC(pmap_evcnt_##name)
 	int pwchange;	/* no mapping change, just wiring or protection */
-	int wchange;	/* no mapping change, just wiring */
+PMAP_COUNTER(remove_kernel_calls, "remove kernel calls");
-	int mchange;	/* was mapped but mapping to different page */
+PMAP_COUNTER(remove_kernel_pages, "kernel pages unmapped");
-	int managed;	/* a managed page */
+PMAP_COUNTER(remove_user_calls, "remove user calls");
-	int firstpv;	/* first mapping for this PA */
+PMAP_COUNTER(remove_user_pages, "user pages unmapped");
-	int secondpv;	/* second mapping for this PA */
+PMAP_COUNTER(remove_flushes, "remove cache flushes");
-	int ci;		/* cache inhibited */
+PMAP_COUNTER(remove_tlb_ops, "remove tlb ops");
-	int unmanaged;	/* not a managed page */
+PMAP_COUNTER(remove_pvfirst, "remove pv first");
-	int flushes;	/* cache flushes */
+PMAP_COUNTER(remove_pvsearch, "remove pv search");
 	int cachehit;	/* new entry forced valid entry out */
-} enter_stats;
+PMAP_COUNTER(prefer_requests, "prefer requests");
-struct {
+PMAP_COUNTER(prefer_adjustments, "prefer adjustments");
 	int calls;
-	int removes;
+PMAP_COUNTER(idlezeroed_pages, "pages idle zeroed");
-	int flushes;
+PMAP_COUNTER(zeroed_pages, "pages zeroed");
-	int pidflushes;	/* HW pid stolen */
+PMAP_COUNTER(copied_pages, "pages copied");
 	int pvfirst;
-	int pvsearch;
+PMAP_COUNTER(kenter_pa, "kernel fast mapped pages");
-} remove_stats;
+PMAP_COUNTER(kenter_pa_bad, "kernel fast mapped pages (bad color)");
 PMAP_COUNTER(kenter_pa_unmanaged, "kernel fast mapped unmanaged pages");
 PMAP_COUNTER(kremove_pages, "kernel fast unmapped pages");
 PMAP_COUNTER(page_cache_evictions, "pages changed to uncacheable");
 PMAP_COUNTER(page_cache_restorations, "pages changed to cacheable");
 PMAP_COUNTER(kernel_mappings_bad, "kernel pages mapped (bad color)");
 PMAP_COUNTER(user_mappings_bad, "user pages mapped (bad color)");
 PMAP_COUNTER(kernel_mappings, "kernel pages mapped");
 PMAP_COUNTER(user_mappings, "user pages mapped");
 PMAP_COUNTER(user_mappings_changed, "user mapping changed");
 PMAP_COUNTER(kernel_mappings_changed, "kernel mapping changed");
 PMAP_COUNTER(uncached_mappings, "uncached pages mapped");
 PMAP_COUNTER(unmanaged_mappings, "unmanaged pages mapped");
 PMAP_COUNTER(managed_mappings, "managed pages mapped");
 PMAP_COUNTER(mappings, "pages mapped");
 PMAP_COUNTER(remappings, "pages remapped");
 PMAP_COUNTER(unmappings, "pages unmapped");
 PMAP_COUNTER(primary_mappings, "page initial mappings");
 PMAP_COUNTER(primary_unmappings, "page final unmappings");
 PMAP_COUNTER(tlb_hit, "page mapping");
 PMAP_COUNTER(exec_mappings, "exec pages mapped");
 PMAP_COUNTER(exec_synced_mappings, "exec pages synced");
 PMAP_COUNTER(exec_synced_remove, "exec pages synced (PR)");
 PMAP_COUNTER(exec_synced_clear_modify, "exec pages synced (CM)");
 PMAP_COUNTER(exec_synced_page_protect, "exec pages synced (PP)");
 PMAP_COUNTER(exec_synced_protect, "exec pages synced (P)");
 PMAP_COUNTER(exec_uncached_page_protect, "exec pages uncached (PP)");
 PMAP_COUNTER(exec_uncached_clear_modify, "exec pages uncached (CM)");
 PMAP_COUNTER(exec_uncached_zero_page, "exec pages uncached (ZP)");
 PMAP_COUNTER(exec_uncached_copy_page, "exec pages uncached (CP)");
 PMAP_COUNTER(exec_uncached_remove, "exec pages uncached (PR)");
 PMAP_COUNTER(create, "creates");
 PMAP_COUNTER(reference, "references");
 PMAP_COUNTER(dereference, "dereferences");
 PMAP_COUNTER(destroy, "destroyed");
 PMAP_COUNTER(activate, "activations");
 PMAP_COUNTER(deactivate, "deactivations");
 PMAP_COUNTER(update, "updates");
 PMAP_COUNTER(unwire, "unwires");
 PMAP_COUNTER(copy, "copies");
 PMAP_COUNTER(collect, "collects");
 PMAP_COUNTER(clear_modify, "clear_modifies");
 PMAP_COUNTER(protect, "protects");
 PMAP_COUNTER(page_protect, "page_protects");
 #define PDB_FOLLOW	0x0001
 #define PDB_INIT	0x0002
 #define PDB_ENTER	0x0004
 #define PDB_REMOVE	0x0008
 #define PDB_CREATE	0x0010
 #define PDB_PTPAGE	0x0020
 #define PDB_PVENTRY	0x0040
 #define PDB_BITS	0x0080
 #define PDB_COLLECT	0x0100
 #define PDB_PROTECT	0x0200
 #define PDB_TLBPID	0x0400
 #define PDB_PARANOIA	0x2000
 #define PDB_WIRING	0x4000
 #define PDB_PVDUMP	0x8000
 int pmapdebug = 0;
 #endif
 #define PMAP_ASID_RESERVED 0
 CTASSERT(PMAP_ASID_RESERVED == 0);
 /*
  * Initialize the kernel pmap.
  */
 #ifdef MULTIPROCESSOR
 #define	PMAP_SIZE	offsetof(struct pmap, pm_pai[MAXCPUS])
 #else
 #define	PMAP_SIZE	sizeof(struct pmap)
 #endif
 struct pmap_kernel kernel_pmap_store = {
 	.kernel_pmap = {
 		.pm_count = 1,
 		.pm_segtab = (void *)(MIPS_KSEG2_START + 0x1eadbeef),
 	},
 };
 paddr_t mips_avail_start;	/* PA of first available physical page */
 paddr_t mips_avail_end;		/* PA of last available physical page */
 vaddr_t mips_virtual_end;	/* VA of last avail page (end of kernel AS) */
 struct pv_entry	*pv_table;
 int		 pv_table_npages;
 pt_entry_t	*Sysmap;		/* kernel pte table */
 unsigned int	Sysmapsize;		/* number of pte's in Sysmap */
 /*
  * The pools from which pmap structures and sub-structures are allocated.
  */
 struct pool pmap_pmap_pool;
 struct pool pmap_pv_pool;
 #ifndef PMAP_PV_LOWAT
 #define	PMAP_PV_LOWAT	16
 #endif
 int		pmap_pv_lowat = PMAP_PV_LOWAT;
 bool		pmap_initialized = false;
 #define	PMAP_PAGE_COLOROK_P(a, b) \
 		((((int)(a) ^ (int)(b)) & pmap_page_colormask) == 0)
 u_int		pmap_page_colormask;
 #define PAGE_IS_MANAGED(pa)	\
 	(pmap_initialized == true && vm_physseg_find(atop(pa), NULL) != -1)
 #define PMAP_IS_ACTIVE(pm)						\
 	((pm) == pmap_kernel() || 					\
 	 (pm) == curlwp->l_proc->p_vmspace->vm_map.pmap)
 /* Forward function declarations */
-void pmap_remove_pv(pmap_t, vaddr_t, struct vm_page *);
+void pmap_remove_pv(pmap_t, vaddr_t, struct vm_page *, bool);
 void pmap_enter_pv(pmap_t, vaddr_t, struct vm_page *, u_int *);
 pt_entry_t *pmap_pte(pmap_t, vaddr_t);
 /*
  * PV table management functions.
  */
 void	*pmap_pv_page_alloc(struct pool *, int);
 void	pmap_pv_page_free(struct pool *, void *);
 struct pool_allocator pmap_pv_page_allocator = {
 	pmap_pv_page_alloc, pmap_pv_page_free, 0,
 };
 #define	pmap_pv_alloc()		pool_get(&pmap_pv_pool, PR_NOWAIT)
 #define	pmap_pv_free(pv)	pool_put(&pmap_pv_pool, (pv))
 /*
  * Misc. functions.
  */
 static inline bool
-pmap_clear_page_attributes(struct vm_page *pg, u_int attributes)
+pmap_clear_page_attributes(struct vm_page *pg, u_int clear_attributes)
+{
 	volatile u_int * const attrp = &pg->mdpage.pvh_attrs;
 #ifdef MULTIPROCESSOR
 	for (;;) {
 		u_int old_attr = *attrp;
-		if ((old_attr & attributes) == 0)
+		if ((old_attr & clear_attributes) == 0)
 			return false;
-		u_int new_attr = old_attr & ~attributes;
+		u_int new_attr = old_attr & ~clear_attributes;
 		if (old_attr == atomic_cas_uint(attrp, old_attr, new_attr))
 			return true;
+	}
 #else
 	u_int old_attr = *attrp;
-	if ((old_attr & attributes) == 0)
+	if ((old_attr & clear_attributes) == 0)
 		return false;
-	*attrp &= ~attributes;
+	*attrp &= ~clear_attributes;
 	return true;
 #endif
+}
 static inline void
-pmap_set_page_attributes(struct vm_page *pg, u_int attributes)
+pmap_set_page_attributes(struct vm_page *pg, u_int set_attributes)
+{
 #ifdef MULTIPROCESSOR
-	atomic_or_uint(&pg->mdpage.pvh_attrs, attributes);
+	atomic_or_uint(&pg->mdpage.pvh_attrs, set_attributes);
 #else
-	pg->mdpage.pvh_attrs |= attributes;
+	pg->mdpage.pvh_attrs |= set_attributes;
 #endif
+}
-#if defined(MIPS3_PLUS)	/* XXX mmu XXX */
+static inline void
-void mips_dump_segtab(struct proc *);
+pmap_page_syncicache(struct vm_page *pg)
 static void mips_flushcache_allpvh(paddr_t);
 	if (MIPS_HAS_R4K_MMU) {
 		if (PG_MD_CACHED_P(pg)) {
 			mips_icache_sync_range_index(
 			    pg->mdpage.pvh_first.pv_va, PAGE_SIZE);
+		}
 	} else {
 		mips_icache_sync_range(MIPS_PHYS_TO_KSEG0(VM_PAGE_TO_PHYS(pg)),
 		    PAGE_SIZE);
+	}
+}
-/*
+static vaddr_t
- * Flush virtual addresses associated with a given physical address
+pmap_map_ephemeral_page(struct vm_page *pg, int prot, pt_entry_t *old_pt_entry_p)
  */
 static void
 mips_flushcache_allpvh(paddr_t pa)
+{
-	struct vm_page *pg;
+	const paddr_t pa = VM_PAGE_TO_PHYS(pg);
-	struct pv_entry *pv;
+	pv_entry_t pv = &pg->mdpage.pvh_first;
-	pg = PHYS_TO_VM_PAGE(pa);
+#ifdef _LP64
-	if (pg == NULL) {
+	vaddr_t va = MIPS_PHYS_TO_XKPHYS_CACHED(pa);
-		/* page is unmanaged */
+#else
-#ifdef DIAGNOSTIC
+	vaddr_t va;
-		printf("mips_flushcache_allpvh(): unmanaged pa = %#"PRIxPADDR"\n",
+	if (pa <= MIPS_PHYS_MASK) {
-		    pa);
+		va = MIPS_PHYS_TO_KSEG0(pa);
-#endif
+	} else {
-		return;
+		KASSERT(pmap_initialized);
 		/*
 		 * Make sure to use a congruent mapping to the last mapped
 		 * address so we don't have to worry about virtual aliases.
 		 */
 		kpreempt_disable();
 		struct cpu_info * const ci = curcpu();
-	pv = pg->mdpage.pvh_list;
+		va = (prot & VM_PROT_WRITE ? ci->ci_pmap_dstbase : ci->ci_pmap_srcbase)
 		    + mips_cache_indexof(MIPS_CACHE_VIRTUAL_ALIAS ? pv->pv_va : pa);
 		*old_pt_entry_p = *kvtopte(va);
 		pmap_kenter_pa(va, pa, prot);
+	}
 #endif /* _LP64 */
 	if (MIPS_CACHE_VIRTUAL_ALIAS) {
 		/*
 		 * If we are forced to use an incompatible alias, flush the
 		 * page from the cache so we will copy the correct contents.
 		 */
 		if (PG_MD_CACHED_P(pg)
 		    && mips_cache_badalias(pv->pv_va, va))
 			mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE);
 		if (pv->pv_pmap == NULL);
 			pv->pv_va = va;
+	}
-#if defined(MIPS3_NO_PV_UNCACHED)
+	return va;
 	/* No current mapping.  Cache was flushed by pmap_remove_pv() */
 	if (pv->pv_pmap == NULL)
 		return;
-	/* Only one index is allowed at a time */
+static void
-	if (mips_cache_indexof(pa) != mips_cache_indexof(pv->pv_va))
+pmap_unmap_ephemeral_page(struct vm_page *pg, vaddr_t va,
-		mips_dcache_wbinv_range_index(pv->pv_va, NBPG);
+	pt_entry_t old_pt_entry)
 #else
-	while (pv) {
+	pv_entry_t pv = &pg->mdpage.pvh_first;
 		mips_dcache_wbinv_range_index(pv->pv_va, NBPG);
-		pv = pv->pv_next;
+	if (MIPS_CACHE_VIRTUAL_ALIAS
 	    && (PG_MD_UNCACHED_P(pg)
 		|| (pv->pv_pmap != NULL
 		    && mips_cache_badalias(pv->pv_va, va)))) {
 		/*
 		 * If this page was previously uncached or we had to use an
 		 * incompatible alias and it has a valid mapping, flush it
 		 * from the cache.
 		 */
 		mips_dcache_wbinv_range(va, PAGE_SIZE);
+	}
 #ifndef _LP64
 	/*
 	 * If we had to map using a page table entry, unmap it now.
 	 */
 	if (va >= VM_MIN_KERNEL_ADDRESS) {
 		pmap_kremove(va, PAGE_SIZE);
 		if (mips_pg_v(old_pt_entry.pt_entry)) {
 			*kvtopte(va) = old_pt_entry;
 			pmap_tlb_update(pmap_kernel(), va, old_pt_entry.pt_entry);
+		}
 		kpreempt_enable();
+	}
 #endif
+}
 #endif /* MIPS3_PLUS */
 /*
  *	Bootstrap the system enough to run with virtual memory.
  *	firstaddr is the first unused kseg0 address (not page aligned).
  */
 void
 pmap_bootstrap(void)
+{
 	vsize_t bufsz;
 	/*
 	 * Compute the number of pages kmem_map will have.
 	 */
 @@ -390,39 +481,26 @@ pmap_bootstrap(void)
 		Sysmapsize =
 		    (VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / NBPG;
+	}
 #endif
 	/*
 	 * Now actually allocate the kernel PTE array (must be done
 	 * after virtual_end is initialized).
 	 */
 	Sysmap = (pt_entry_t *)
 	    uvm_pageboot_alloc(sizeof(pt_entry_t) * Sysmapsize);
 	/*
 	 * Allocate memory for the pv_heads.  (A few more of the latter
 	 * are allocated than are needed.)
 	 * We could do this in pmap_init when we know the actual
 	 * managed page pool size, but its better to use kseg0
 	 * addresses rather than kernel virtual addresses mapped
 	 * through the TLB.
 	 */
 	pv_table_npages = physmem;
 	pv_table = (struct pv_entry *)
 	    uvm_pageboot_alloc(sizeof(struct pv_entry) * pv_table_npages);
 	/*
 	 * Initialize the pools.
 	 */
 	pool_init(&pmap_pmap_pool, PMAP_SIZE, 0, 0, 0, "pmappl",
 	    &pool_allocator_nointr, IPL_NONE);
 	pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvpl",
 	    &pmap_pv_page_allocator, IPL_NONE);
 	tlb_set_asid(0);
 #ifdef MIPS3_PLUS	/* XXX mmu XXX */
 	/*
 	 * The R4?00 stores only one copy of the Global bit in the
 	 * translation lookaside buffer for each 2 page entry.
 @@ -540,59 +618,58 @@ pmap_steal_memory(vsize_t size, vaddr_t
 	 * If we got here, there was no memory left.
 	 */
 	panic("pmap_steal_memory: no memory to steal");
+}
 /*
  *	Initialize the pmap module.
  *	Called by vm_init, to initialize any structures that the pmap
  *	system needs to map virtual memory.
  */
 void
 pmap_init(void)
+{
 	vsize_t		s;
 	int		bank, i;
 	pv_entry_t	pv;
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_INIT))
 		printf("pmap_init()\n");
 #endif
 	/*
 	 * Memory for the pv entry heads has
 	 * already been allocated.  Initialize the physical memory
 	 * segments.
 	 */
 	pv = pv_table;
 	for (bank = 0; bank < vm_nphysseg; bank++) {
 		s = vm_physmem[bank].end - vm_physmem[bank].start;
 		for (i = 0; i < s; i++)
 			vm_physmem[bank].pgs[i].mdpage.pvh_list = pv++;
 	/*
 	 * Set a low water mark on the pv_entry pool, so that we are
 	 * more likely to have these around even in extreme memory
 	 * starvation.
 	 */
 	pool_setlowat(&pmap_pv_pool, pmap_pv_lowat);
 	/*
 	 * Now it is safe to enable pv entry recording.
 	 */
 	pmap_initialized = true;
 #ifndef _LP64
 	/*
 	 * If we have more memory than can be mapped by KSEG0, we need allocate
 	 * enough VA so we can map pages with the right color (to avoid cache
 	 * alias problems).
 	 */
 	if (mips_avail_end > MIPS_KSEG1_START - MIPS_KSEG0_START) {
 		curcpu()->ci_pmap_dstbase = uvm_km_alloc(kernel_map,
 		    uvmexp.ncolors * PAGE_SIZE, 0, UVM_KMF_VAONLY);
 		curcpu()->ci_pmap_srcbase = uvm_km_alloc(kernel_map,
 		    uvmexp.ncolors * PAGE_SIZE, 0, UVM_KMF_VAONLY);
+	}
 #endif
 #ifdef MIPS3
 	if (MIPS_HAS_R4K_MMU) {
 		/*
 		 * XXX
 		 * Disable sosend_loan() in src/sys/kern/uipc_socket.c
 		 * on MIPS3 CPUs to avoid possible virtual cache aliases
 		 * and uncached mappings in pmap_enter_pv().
+		 *
 		 * Ideally, read only shared mapping won't cause aliases
 		 * so pmap_enter_pv() should handle any shared read only
 		 * mappings without uncached ops like ARM pmap.
+		 *
 		 * On the other hand, R4000 and R4400 have the virtual
 @@ -612,109 +689,115 @@ pmap_init(void)
  *	is zero, the map is an actual physical
  *	map, and may be referenced by the
  *	hardware.
+ *
  *	If the size specified is non-zero,
  *	the map will be used in software only, and
  *	is bounded by that size.
  */
 pmap_t
 pmap_create(void)
+{
 	pmap_t pmap;
 	PMAP_COUNT(create);
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_CREATE))
 		printf("pmap_create()\n");
 #endif
 	pmap = pool_get(&pmap_pmap_pool, PR_WAITOK);
 	memset(pmap, 0, PMAP_SIZE);
 	pmap->pm_count = 1;
 	pmap_segtab_alloc(pmap);
 	return pmap;
+}
 /*
  *	Retire the given physical map from service.
  *	Should only be called if the map contains
  *	no valid mappings.
  */
 void
 pmap_destroy(pmap_t pmap)
+{
 	int count;
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_CREATE))
 		printf("pmap_destroy(%p)\n", pmap);
 #endif
-	count = --pmap->pm_count;
+	if (--pmap->pm_count) {
-	if (count > 0)
+		PMAP_COUNT(dereference);
 		return;
+	}
 	PMAP_COUNT(destroy);
 	pmap_segtab_free(pmap);
 	pool_put(&pmap_pmap_pool, pmap);
+}
 /*
  *	Add a reference to the specified pmap.
  */
 void
 pmap_reference(pmap_t pmap)
+{
 #ifdef DEBUG
 	if (pmapdebug & PDB_FOLLOW)
 		printf("pmap_reference(%p)\n", pmap);
 #endif
 	if (pmap != NULL) {
 		pmap->pm_count++;
+	}
 	PMAP_COUNT(reference);
+}
 /*
  *	Make a new pmap (vmspace) active for the given process.
  */
 void
 pmap_activate(struct lwp *l)
+{
 	pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap;
-	unsigned int asid;
+	uint32_t asid;
 	PMAP_COUNT(activate);
 	asid = pmap_tlb_asid_alloc(pmap, l->l_cpu);
 	if (l == curlwp) {
 		pmap_segtab_activate(l);
 		tlb_set_asid(asid);
+	}
+}
 /*
  *	Make a previously active pmap (vmspace) inactive.
  */
 void
 pmap_deactivate(struct lwp *l)
+{
 	PMAP_COUNT(deactivate);
 	/* Nothing to do. */
+}
 void
 pmap_update(struct pmap *pmap)
+{
 	PMAP_COUNT(update);
 #if 0
 	__asm __volatile(
 		"mtc0\t$ra,$%0; nop; eret"
+	    :
 	    : "n"(MIPS_COP_0_ERROR_PC));
 #endif
+}
 /*
  *	Remove the given range of addresses from the specified map.
+ *
  *	It is assumed that the start and end are properly
  *	rounded to the page size.
 @@ -726,85 +809,88 @@ pmap_pte_remove(pmap_t pmap, vaddr_t sva
+{
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_REMOVE|PDB_PROTECT)) {
 		printf("%s: %p, %"PRIxVADDR", %"PRIxVADDR", %p, %"PRIxPTR"\n",
 		   __func__, pmap, sva, eva, pte, flags);
+	}
 #endif
 	for (; sva < eva; sva += NBPG, pte++) {
 		struct vm_page *pg;
 		uint32_t pt_entry = pte->pt_entry;
 		if (!mips_pg_v(pt_entry))
 			continue;
 		PMAP_COUNT(remove_user_pages);
 		if (mips_pg_wired(pt_entry))
 			pmap->pm_stats.wired_count--;
 		pmap->pm_stats.resident_count--;
 		pg = PHYS_TO_VM_PAGE(mips_tlbpfn_to_paddr(pt_entry));
-		if (pg)
+		if (pg) {
-			pmap_remove_pv(pmap, sva, pg);
+			pmap_remove_pv(pmap, sva, pg,
 			   pt_entry & mips_pg_m_bit());
+		}
 		pte->pt_entry = mips_pg_nv_bit();
 		/*
 		 * Flush the TLB for the given address.
 		 */
 		pmap_tlb_invalidate_addr(pmap, sva);
 #ifdef DEBUG
 		remove_stats.flushes++;
 #endif
+	}
 	return false;
+}
 void
 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva)
+{
 	struct vm_page *pg;
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_REMOVE|PDB_PROTECT))
 		printf("pmap_remove(%p, %#"PRIxVADDR", %#"PRIxVADDR")\n", pmap, sva, eva);
 	remove_stats.calls++;
 #endif
 	if (pmap == pmap_kernel()) {
 		/* remove entries from kernel pmap */
 		PMAP_COUNT(remove_kernel_calls);
 #ifdef PARANOIADIAG
-		if (sva < VM_MIN_KERNEL_ADDRESS || eva >= virtual_end)
+		if (sva < VM_MIN_KERNEL_ADDRESS || eva >= mips_virtual_end)
 			panic("pmap_remove: kva not in range");
 #endif
 		pt_entry_t *pte = kvtopte(sva);
 		for (; sva < eva; sva += NBPG, pte++) {
 			uint32_t pt_entry = pte->pt_entry;
 			if (!mips_pg_v(pt_entry))
 				continue;
 			PMAP_COUNT(remove_kernel_pages);
 			if (mips_pg_wired(pt_entry))
 				pmap->pm_stats.wired_count--;
 			pmap->pm_stats.resident_count--;
 			pg = PHYS_TO_VM_PAGE(mips_tlbpfn_to_paddr(pt_entry));
 			if (pg)
-				pmap_remove_pv(pmap, sva, pg);
+				pmap_remove_pv(pmap, sva, pg, false);
 			if (MIPS_HAS_R4K_MMU)
 				/* See above about G bit */
 				pte->pt_entry = MIPS3_PG_NV | MIPS3_PG_G;
 			else
 				pte->pt_entry = MIPS1_PG_NV;
 			/*
 			 * Flush the TLB for the given address.
 			 */
 			pmap_tlb_invalidate_addr(pmap, sva);
+		}
 		return;
+	}
 	PMAP_COUNT(remove_user_calls);
 #ifdef PARANOIADIAG
 	if (eva > VM_MAXUSER_ADDRESS)
 		panic("pmap_remove: uva not in range");
 	if (PMAP_IS_ACTIVE(pmap)) {
 		struct pmap_asid_info * const pai = PMAP_PAI(pmap, curcpu());
 		uint32_t asid;
 		__asm volatile("mfc0 %0,$10; nop" : "=r"(asid));
 		asid = (MIPS_HAS_R4K_MMU) ? (asid & 0xff) : (asid & 0xfc0) >> 6;
 		if (asid != pai->pai_asid) {
 			panic("inconsistency for active TLB flush: %d <-> %d",
 			    asid, pai->pai_asid);
+		}
 @@ -814,125 +900,146 @@ pmap_remove(pmap_t pmap, vaddr_t sva, va
+}
 /*
  *	pmap_page_protect:
+ *
  *	Lower the permission for all mappings to a given page.
  */
 void
 pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
+{
 	pv_entry_t pv;
 	vaddr_t va;
 	PMAP_COUNT(page_protect);
 #ifdef DEBUG
 	if ((pmapdebug & (PDB_FOLLOW|PDB_PROTECT)) ||
 	    (prot == VM_PROT_NONE && (pmapdebug & PDB_REMOVE)))
 		printf("pmap_page_protect(%#"PRIxPADDR", %x)\n",
 		    VM_PAGE_TO_PHYS(pg), prot);
 #endif
 	switch (prot) {
 	case VM_PROT_READ|VM_PROT_WRITE:
 	case VM_PROT_ALL:
 		break;
 	/* copy_on_write */
 	case VM_PROT_READ:
 	case VM_PROT_READ|VM_PROT_EXECUTE:
-		pv = pg->mdpage.pvh_list;
+		pv = &pg->mdpage.pvh_first;
 		/*
 		 * Loop over all current mappings setting/clearing as appropos.
 		 */
 		if (pv->pv_pmap != NULL) {
 			for (; pv; pv = pv->pv_next) {
 				va = pv->pv_va;
 				pmap_protect(pv->pv_pmap, va, va + PAGE_SIZE,
 				    prot);
 				pmap_update(pv->pv_pmap);
+			}
+		}
 		break;
 	/* remove_all */
 	default:
-		pv = pg->mdpage.pvh_list;
+		/*
 		 * Do this first so that for each unmapping, pmap_remove_pv
 		 * won't try to sync the icache.
 		 */
 		if (pmap_clear_page_attributes(pg, PG_MD_EXECPAGE)) {
 			PMAP_COUNT(exec_uncached_page_protect);
+		}
 		pv = &pg->mdpage.pvh_first;
 		while (pv->pv_pmap != NULL) {
-			pmap_remove(pv->pv_pmap, pv->pv_va,
+			va = pv->pv_va;
-			    pv->pv_va + PAGE_SIZE);
+			pmap_remove(pv->pv_pmap, va, va + PAGE_SIZE);
 			pmap_update(pv->pv_pmap);
+		}
 		pmap_update(pv->pv_pmap);
+	}
+}
 static bool
 pmap_pte_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *pte,
 	uintptr_t flags)
+{
 	const uint32_t pg_mask = ~(mips_pg_m_bit() | mips_pg_ro_bit());
-	const uint32_t p = flags;
+	const uint32_t p = (flags & VM_PROT_WRITE) ? mips_pg_rw_bit() : mips_pg_ro_bit();
 	/*
 	 * Change protection on every valid mapping within this segment.
 	 */
 	for (; sva < eva; sva += NBPG, pte++) {
 		uint32_t pt_entry = pte->pt_entry;
 		if (!mips_pg_v(pt_entry))
 			continue;
-		if (MIPS_HAS_R4K_MMU && (pt_entry & mips_pg_m_bit()))
+		struct vm_page *pg;
-			mips_dcache_wbinv_range_index(sva, PAGE_SIZE);
+		pg = PHYS_TO_VM_PAGE(mips_tlbpfn_to_paddr(pt_entry));
 		if (pg && (pt_entry & mips_pg_m_bit())) {
 			if (MIPS_HAS_R4K_MMU
 			    && MIPS_CACHE_VIRTUAL_ALIAS
 			    && PG_MD_CACHED_P(pg))
 				mips_dcache_wbinv_range_index(sva, PAGE_SIZE);
 			if (PG_MD_EXECPAGE_P(pg)) {
 				KASSERT(pg->mdpage.pvh_first.pv_pmap != NULL);
 				if (PG_MD_CACHED_P(pg)) {
 					pmap_page_syncicache(pg);
 					PMAP_COUNT(exec_synced_protect);
+				}
+			}
+		}
 		pt_entry = (pt_entry & pg_mask) | p;
 		pte->pt_entry = pt_entry;
 		/*
 		 * Update the TLB if needed.
 		 */
 		pmap_tlb_update(pmap, sva, pt_entry);
+	}
 	return false;
+}
 /*
  *	Set the physical protection on the
  *	specified range of this map as requested.
  */
 void
 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
+{
 	const uint32_t pg_mask = ~(mips_pg_m_bit() | mips_pg_ro_bit());
 	pt_entry_t *pte;
 	u_int p;
 	PMAP_COUNT(protect);
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_PROTECT))
 		printf("pmap_protect(%p, %#"PRIxVADDR", %#"PRIxVADDR", %x)\n",
 		    pmap, sva, eva, prot);
 #endif
 	if ((prot & VM_PROT_READ) == VM_PROT_NONE) {
 		pmap_remove(pmap, sva, eva);
 		return;
+	}
 	p = (prot & VM_PROT_WRITE) ? mips_pg_rw_bit() : mips_pg_ro_bit();
 	if (pmap == pmap_kernel()) {
 		/*
 		 * Change entries in kernel pmap.
 		 * This will trap if the page is writable (in order to set
 		 * the dirty bit) even if the dirty bit is already set. The
 		 * optimization isn't worth the effort since this code isn't
 		 * executed much. The common case is to make a user page
 		 * read-only.
 		 */
 #ifdef PARANOIADIAG
-		if (sva < VM_MIN_KERNEL_ADDRESS || eva >= virtual_end)
+		if (sva < VM_MIN_KERNEL_ADDRESS || eva >= mips_virtual_end)
 			panic("pmap_protect: kva not in range");
 #endif
 		pte = kvtopte(sva);
 		for (; sva < eva; sva += NBPG, pte++) {
 			uint32_t pt_entry = pte->pt_entry;
 			if (!mips_pg_v(pt_entry))
 				continue;
 			if (MIPS_HAS_R4K_MMU && (pt_entry & mips_pg_m_bit()))
 				mips_dcache_wb_range(sva, PAGE_SIZE);
 			pt_entry &= (pt_entry & pg_mask) | p;
 			pte->pt_entry = pt_entry;
 			pmap_tlb_update(pmap, sva, pt_entry);
+		}
 @@ -1006,237 +1113,250 @@ pmap_procwr(struct proc *p, vaddr_t va,
 		 * XXXJRT need to loop.
 		 */
 		mips_icache_sync_range(
 		    MIPS_PHYS_TO_KSEG0(mips1_tlbpfn_to_paddr(entry)
 		    + (va & PGOFSET)),
 		    len);
 #endif /* MIPS1 */
+	}
+}
 /*
  *	Return RO protection of page.
  */
-int
+bool
-pmap_is_page_ro(pmap_t pmap, vaddr_t va, int entry)
+pmap_is_page_ro_p(pmap_t pmap, vaddr_t va, uint32_t entry)
+{
-	return entry & mips_pg_ro_bit();
+	return (entry & mips_pg_ro_bit()) != 0;
+}
 #if defined(MIPS3_PLUS) && !defined(MIPS3_NO_PV_UNCACHED)	/* XXX mmu XXX */
 /*
  *	pmap_page_cache:
+ *
  *	Change all mappings of a managed page to cached/uncached.
  */
 static void
-pmap_page_cache(struct vm_page *pg, int mode)
+pmap_page_cache(struct vm_page *pg, bool cached)
+{
-	pv_entry_t pv;
+	const uint32_t newmode = cached ? MIPS3_PG_CACHED : MIPS3_PG_UNCACHED;
 	unsigned newmode;
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_ENTER))
 		printf("pmap_page_uncache(%#"PRIxPADDR")\n", VM_PAGE_TO_PHYS(pg));
 #endif
 	newmode = mode & PV_UNCACHED ? MIPS3_PG_UNCACHED : MIPS3_PG_CACHED;
-	pv = pg->mdpage.pvh_list;
+	if (cached) {
 		pmap_clear_page_attributes(pg, PG_MD_UNCACHED);
 		PMAP_COUNT(page_cache_restorations);
 	} else {
 		pmap_set_page_attributes(pg, PG_MD_UNCACHED);
 		PMAP_COUNT(page_cache_evictions);
+	}
-	while (pv) {
+	for (pv_entry_t pv = &pg->mdpage.pvh_first;
 	     pv != NULL;
 	     pv = pv->pv_next) {
 		pmap_t pmap = pv->pv_pmap;
 		pt_entry_t *pte;
-		uint32_t entry;
+		uint32_t pt_entry;
-		pv->pv_flags = (pv->pv_flags & ~PV_UNCACHED) | mode;
+		KASSERT(pv->pv_pmap != NULL);
 		if (pmap == pmap_kernel()) {
 			/*
 			 * Change entries in kernel pmap.
 			 */
 			pte = kvtopte(pv->pv_va);
 		} else {
 			pte = pmap_pte_lookup(pmap, pv->pv_va);
 			if (pte == NULL)
 				continue;
+		}
-		entry = pte->pt_entry;
+		pt_entry = pte->pt_entry;
-		if (entry & MIPS3_PG_V) {
+		if (pt_entry & MIPS3_PG_V) {
-			entry = (entry & ~MIPS3_PG_CACHEMODE) | newmode;
+			pt_entry = (pt_entry & ~MIPS3_PG_CACHEMODE) | newmode;
-			pte->pt_entry = entry;
+			pte->pt_entry = pt_entry;
-			pmap_tlb_update(pv->pv_pmap, pv->pv_va, entry);
+			pmap_tlb_update(pv->pv_pmap, pv->pv_va, pt_entry);
+		}
 		pv = pv->pv_next;
+	}
+}
 #endif	/* MIPS3_PLUS && !MIPS3_NO_PV_UNCACHED */
 /*
  *	Insert the given physical page (p) at
  *	the specified virtual address (v) in the
  *	target physical map with the protection requested.
+ *
  *	If specified, the page will be wired down, meaning
  *	that the related pte can not be reclaimed.
+ *
  *	NB:  This is the only routine which MAY NOT lazy-evaluate
  *	or lose information.  That is, this routine must actually
  *	insert this page into the given map NOW.
  */
 int
 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags)
+{
 	pt_entry_t *pte;
 	u_int npte;
 	struct vm_page *pg;
 	unsigned asid;
 #if defined(_MIPS_PADDR_T_64BIT) || defined(_LP64)
-	int cached = 1;
+	bool cached = true;
 #endif
 	bool wired = (flags & PMAP_WIRED) != 0;
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_ENTER))
 		printf("pmap_enter(%p, %#"PRIxVADDR", %#"PRIxPADDR", %x, %x)\n",
 		    pmap, va, pa, prot, wired);
 #endif
-#if defined(DEBUG) || defined(DIAGNOSTIC) || defined(PARANOIADIAG)
+	const bool good_color = PMAP_PAGE_COLOROK_P(pa, va);
 	if (pmap == pmap_kernel()) {
-#ifdef DEBUG
+		PMAP_COUNT(kernel_mappings);
-		enter_stats.kernel++;
+		if (!good_color)
-#endif
+			PMAP_COUNT(kernel_mappings_bad);
 #if defined(DEBUG) || defined(DIAGNOSTIC) || defined(PARANOIADIAG)
 		if (va < VM_MIN_KERNEL_ADDRESS || va >= mips_virtual_end)
-			panic("pmap_enter: kva too big");
+			panic("pmap_enter: kva %#"PRIxVADDR"too big", va);
 	} else {
 #ifdef DEBUG
 		enter_stats.user++;
 #endif
 	} else {
 		PMAP_COUNT(user_mappings);
 		if (!good_color)
 			PMAP_COUNT(user_mappings_bad);
 #if defined(DEBUG) || defined(DIAGNOSTIC) || defined(PARANOIADIAG)
 		if (va >= VM_MAXUSER_ADDRESS)
-			panic("pmap_enter: uva too big");
+			panic("pmap_enter: uva %#"PRIxVADDR" too big", va);
 #endif
+	}
 #ifdef PARANOIADIAG
 #if defined(cobalt) || defined(newsmips) || defined(pmax) /* otherwise ok */
 	if (pa & 0x80000000)	/* this is not error in general. */
 		panic("pmap_enter: pa");
 #endif
 #if defined(_MIPS_PADDR_T_64BIT) || defined(_LP64)
 	if (pa & PMAP_NOCACHE) {
-		cached = 0;
+		cached = false;
 		pa &= ~PMAP_NOCACHE;
+	}
 #endif
 	if (!(prot & VM_PROT_READ))
 		panic("pmap_enter: prot");
 #endif
 	pg = PHYS_TO_VM_PAGE(pa);
 	if (pg) {
 		/* Set page referenced/modified status based on flags */
 		if (flags & VM_PROT_WRITE)
-			pmap_set_page_attributes(pg, PV_MODIFIED|PV_REFERENCED);
+			pmap_set_page_attributes(pg, PG_MD_MODIFIED|PG_MD_REFERENCED);
 		else if (flags & VM_PROT_ALL)
-			pmap_set_page_attributes(pg, PV_REFERENCED);
+			pmap_set_page_attributes(pg, PG_MD_REFERENCED);
 		if (!(prot & VM_PROT_WRITE))
 			/*
 			 * If page is not yet referenced, we could emulate this
 			 * by not setting the page valid, and setting the
 			 * referenced status in the TLB fault handler, similar
 			 * to how page modified status is done for UTLBmod
 			 * exceptions.
 			 */
 			npte = mips_pg_ropage_bit();
 		else {
 #if defined(_MIPS_PADDR_T_64BIT) || defined(_LP64)
-			if (cached == 0) {
+			if (cached == false) {
-				if (pg->mdpage.pvh_attrs & PV_MODIFIED) {
+				if (PG_MD_MODIFIED_P(pg)) {
 					npte = mips_pg_rwncpage_bit();
 				} else {
 					npte = mips_pg_cwncpage_bit();
+				}
 				PMAP_COUNT(uncached_mappings);
 			} else {
 #endif
-				if (pg->mdpage.pvh_attrs & PV_MODIFIED) {
+				if (PG_MD_MODIFIED_P(pg)) {
 					npte = mips_pg_rwpage_bit();
 				} else {
 					npte = mips_pg_cwpage_bit();
+				}
 #if defined(_MIPS_PADDR_T_64BIT) || defined(_LP64)
+			}
 #endif
+		}
-#ifdef DEBUG
+		PMAP_COUNT(managed_mappings);
 		enter_stats.managed++;
 #endif
 	} else {
 		/*
 		 * Assumption: if it is not part of our managed memory
 		 * then it must be device memory which may be volatile.
 		 */
 #ifdef DEBUG
 		enter_stats.unmanaged++;
 #endif
 		if (MIPS_HAS_R4K_MMU) {
 			npte = MIPS3_PG_IOPAGE(PMAP_CCA_FOR_PA(pa)) &
 			    ~MIPS3_PG_G;
 			if ((prot & VM_PROT_WRITE) == 0) {
 				npte |= MIPS3_PG_RO;
 				npte &= ~MIPS3_PG_D;
+			}
 		} else {
 			npte = (prot & VM_PROT_WRITE) ?
 			    (MIPS1_PG_D | MIPS1_PG_N) :
 			    (MIPS1_PG_RO | MIPS1_PG_N);
+		}
 		PMAP_COUNT(unmanaged_mappings);
+	}
 #if 0
 	/*
 	 * The only time we need to flush the cache is if we
 	 * execute from a physical address and then change the data.
 	 * This is the best place to do this.
 	 * pmap_protect() and pmap_remove() are mostly used to switch
 	 * between R/W and R/O pages.
 	 * NOTE: we only support cache flush for read only text.
 	 */
 #ifdef MIPS1
-	if ((!MIPS_HAS_R4K_MMU) && prot == (VM_PROT_READ | VM_PROT_EXECUTE)) {
+	if (!MIPS_HAS_R4K_MMU
-		mips_icache_sync_range(MIPS_PHYS_TO_KSEG0(pa), PAGE_SIZE);
+	    && pg != NULL
 	    && prot == (VM_PROT_READ | VM_PROT_EXECUTE)) {
 		PMAP_COUNT(enter_exec_mapping);
 		if (!PG_MD_EXECPAGE_P(pg)) {
 			mips_icache_sync_range(MIPS_PHYS_TO_KSEG0(pa),
 			    PAGE_SIZE);
 			pmap_set_page_attributes(pg, PG_MD_EXECPAGE);
 			PMAP_COUNT(exec_syncicache_entry);
+		}
+	}
 #endif
 #endif
 	if (pmap == pmap_kernel()) {
 		if (pg)
 			pmap_enter_pv(pmap, va, pg, &npte);
 		/* enter entries into kernel pmap */
 		pte = kvtopte(va);
 		if (MIPS_HAS_R4K_MMU)
 			npte |= mips3_paddr_to_tlbpfn(pa) | MIPS3_PG_G;
 		else
 			npte |= mips1_paddr_to_tlbpfn(pa) |
 			    MIPS1_PG_V | MIPS1_PG_G;
 		if (wired) {
 			pmap->pm_stats.wired_count++;
 			npte |= mips_pg_wired_bit();
+		}
 		if (mips_pg_v(pte->pt_entry)
 		    && mips_tlbpfn_to_paddr(pte->pt_entry) != pa) {
 			pmap_remove(pmap, va, va + NBPG);
-#ifdef DEBUG
+			PMAP_COUNT(kernel_mappings_changed);
 			enter_stats.mchange++;
 #endif
+		}
 		if (!mips_pg_v(pte->pt_entry))
 			pmap->pm_stats.resident_count++;
 		pte->pt_entry = npte;
 		/*
 		 * Update the same virtual address entry.
 		 */
 		pmap_tlb_update(pmap, va, npte);
 		return 0;
+	}
 @@ -1254,169 +1374,161 @@ pmap_enter(pmap_t pmap, vaddr_t va, padd
 	 * Assume uniform modified and referenced status for all
 	 * MIPS pages in a MACH page.
 	 */
 	if (MIPS_HAS_R4K_MMU)
 		npte |= mips3_paddr_to_tlbpfn(pa);
 	else
 		npte |= mips1_paddr_to_tlbpfn(pa) | MIPS1_PG_V;
 	if (wired) {
 		pmap->pm_stats.wired_count++;
 		npte |= mips_pg_wired_bit();
+	}
 	struct pmap_asid_info * const pai = PMAP_PAI(pmap, curcpu());
 	bool needsupdate = PMAP_PAI_ASIDVALID_P(pai, curcpu());
 #if defined(DEBUG)
 	if (pmapdebug & PDB_ENTER) {
 		printf("pmap_enter: %p: %#"PRIxVADDR": new pte %#x (pa %#"PRIxPADDR")", pmap, va, npte, pa);
 		if (needsupdate)
 			printf(" asid %u (%#x)", pai->pai_asid, pai->pai_asid);
 		printf("\n");
+	}
 #endif
 #ifdef PARANOIADIAG
 	if (PMAP_IS_ACTIVE(pmap)) {
 		uint32_t asid;
 		__asm volatile("mfc0 %0,$10; nop" : "=r"(asid));
 		asid = (MIPS_HAS_R4K_MMU) ? (asid & 0xff) : (asid & 0xfc0) >> 6;
 		if (asid != pai->pai_asid) {
 			panic("inconsistency for active TLB update: %u <-> %u",
 			    asid, pai->pai_asid);
+		}
+	}
 #endif
 	asid = pai->pai_asid << MIPS_TLB_PID_SHIFT;
 	if (mips_pg_v(pte->pt_entry) &&
 	    mips_tlbpfn_to_paddr(pte->pt_entry) != pa) {
 		pmap_remove(pmap, va, va + NBPG);
-#ifdef DEBUG
+		PMAP_COUNT(user_mappings_changed);
 		enter_stats.mchange++;
 #endif
+	}
 	if (!mips_pg_v(pte->pt_entry))
 		pmap->pm_stats.resident_count++;
 	pte->pt_entry = npte;
-	if (needsupdate)
+	pmap_tlb_update(pmap, va, npte);
 		pmap_tlb_update(pmap, va, npte);
-#ifdef MIPS3_PLUS	/* XXX mmu XXX */
+	if (pg != NULL && (prot == (VM_PROT_READ | VM_PROT_EXECUTE))) {
 	if (MIPS_HAS_R4K_MMU && (prot == (VM_PROT_READ | VM_PROT_EXECUTE))) {
 #ifdef DEBUG
 		if (pmapdebug & PDB_ENTER)
 			printf("pmap_enter: flush I cache va %#"PRIxVADDR" (%#"PRIxPADDR")\n",
 			    va - NBPG, pa);
 #endif
-		/* XXXJRT */
+		PMAP_COUNT(exec_mappings);
-		mips_icache_sync_range_index(va, PAGE_SIZE);
+		if (!PG_MD_EXECPAGE_P(pg) && PG_MD_CACHED_P(pg)) {
 			pmap_page_syncicache(pg);
 			pmap_set_page_attributes(pg, PG_MD_EXECPAGE);
 			PMAP_COUNT(exec_synced_mappings);
+		}
+	}
 #endif
 	return 0;
+}
 void
 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot)
+{
 	const bool managed = PAGE_IS_MANAGED(pa);
 	pt_entry_t *pte;
 	u_int npte;
 	bool managed = PAGE_IS_MANAGED(pa);
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_ENTER))
 		printf("pmap_kenter_pa(%#"PRIxVADDR", %#"PRIxPADDR", %x)\n", va, pa, prot);
 #endif
 	PMAP_COUNT(kenter_pa);
 	if (!PMAP_PAGE_COLOROK_P(pa, va) && managed)
 		PMAP_COUNT(kenter_pa_bad);
 	if (!managed)
 		PMAP_COUNT(kenter_pa_unmanaged);
 	if (MIPS_HAS_R4K_MMU) {
-		npte = mips3_paddr_to_tlbpfn(pa) | MIPS3_PG_WIRED;
+		npte = mips3_paddr_to_tlbpfn(pa)
-		if (prot & VM_PROT_WRITE) {
+		    | ((prot & VM_PROT_WRITE) ? MIPS3_PG_D : MIPS3_PG_RO)
-			npte |= MIPS3_PG_D;
+		    | (managed ? MIPS3_PG_CACHED : MIPS3_PG_UNCACHED)
-		} else {
+		    | MIPS3_PG_WIRED | MIPS3_PG_V | MIPS3_PG_G;
 			npte |= MIPS3_PG_RO;
 		if (managed) {
 			npte |= MIPS3_PG_CACHED;
 		} else {
 			npte |= MIPS3_PG_UNCACHED;
 		npte |= MIPS3_PG_V | MIPS3_PG_G;
 	} else {
-		npte = mips1_paddr_to_tlbpfn(pa) | MIPS1_PG_WIRED;
+		npte = mips1_paddr_to_tlbpfn(pa)
-		if (prot & VM_PROT_WRITE) {
+		    | ((prot & VM_PROT_WRITE) ? MIPS1_PG_D : MIPS1_PG_RO)
-			npte |= MIPS1_PG_D;
+		    | (managed ? 0 : MIPS1_PG_N)
-		} else {
+		    | MIPS1_PG_WIRED | MIPS1_PG_V | MIPS1_PG_G;
 			npte |= MIPS1_PG_RO;
 		if (managed) {
 			npte |= 0;
 		} else {
 			npte |= MIPS1_PG_N;
 		npte |= MIPS1_PG_V | MIPS1_PG_G;
+	}
 	pte = kvtopte(va);
 	KASSERT(!mips_pg_v(pte->pt_entry));
 	pte->pt_entry = npte;
 	pmap_tlb_update(pmap_kernel(), va, npte);
+}
 void
 pmap_kremove(vaddr_t va, vsize_t len)
+{
 	pt_entry_t *pte;
 	vaddr_t eva;
 	u_int entry;
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_REMOVE))
 		printf("pmap_kremove(%#"PRIxVADDR", %#"PRIxVSIZE")\n", va, len);
 #endif
-	pte = kvtopte(va);
+	const uint32_t new_pt_entry =
-	eva = va + len;
+	    (MIPS_HAS_R4K_MMU ? MIPS3_PG_NV | MIPS3_PG_G : MIPS1_PG_NV);
 	for (; va < eva; va += PAGE_SIZE, pte++) {
-		entry = pte->pt_entry;
+	pt_entry_t *pte = kvtopte(va);
-		if (!mips_pg_v(entry)) {
+	for (vaddr_t eva = va + len; va < eva; va += PAGE_SIZE, pte++) {
 		uint32_t pt_entry = pte->pt_entry;
 		if (!mips_pg_v(pt_entry)) {
 			continue;
+		}
 		if (MIPS_HAS_R4K_MMU) {
-#ifndef sbmips	/* XXX XXX if (dcache_is_virtual) - should also check icache virtual && EXEC mapping */
+		PMAP_COUNT(kremove_pages);
-			mips_dcache_wbinv_range(va, PAGE_SIZE);
+		if (MIPS_HAS_R4K_MMU && MIPS_CACHE_VIRTUAL_ALIAS) {
-#endif
+			struct vm_page *pg =
-			pte->pt_entry = MIPS3_PG_NV | MIPS3_PG_G;
+			    PHYS_TO_VM_PAGE(mips_tlbpfn_to_paddr(pt_entry));
-		} else {
+			if (pg != NULL) {
-			pte->pt_entry = MIPS1_PG_NV;
+				pv_entry_t pv = &pg->mdpage.pvh_first;
 				if (pv->pv_pmap == NULL) {
 					pv->pv_va = va;
 				} else if (PG_MD_CACHED_P(pg)
 				    && mips_cache_badalias(pv->pv_va, va)) {
 					mips_dcache_wbinv_range(va, PAGE_SIZE);
+				}
+			}
+		}
 		pte->pt_entry = new_pt_entry;
 		pmap_tlb_invalidate_addr(pmap_kernel(), va);
+	}
+}
 /*
  *	Routine:	pmap_unwire
  *	Function:	Clear the wired attribute for a map/virtual-address
  *			pair.
  *	In/out conditions:
  *			The mapping must already exist in the pmap.
  */
 void
 pmap_unwire(pmap_t pmap, vaddr_t va)
+{
 	pt_entry_t *pte;
 	PMAP_COUNT(unwire);
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_WIRING))
 		printf("pmap_unwire(%p, %#"PRIxVADDR")\n", pmap, va);
 #endif
 	/*
 	 * Don't need to flush the TLB since PG_WIRED is only in software.
 	 */
 	if (pmap == pmap_kernel()) {
 		/* change entries in kernel pmap */
 #ifdef PARANOIADIAG
 		if (va < VM_MIN_KERNEL_ADDRESS || va >= virtual_end)
 			panic("pmap_unwire");
 #endif
 @@ -1511,222 +1623,159 @@ done:
 /*
  *	Copy the range specified by src_addr/len
  *	from the source map to the range dst_addr/len
  *	in the destination map.
+ *
  *	This routine is only advisory and need not do anything.
  */
 void
 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, vsize_t len,
     vaddr_t src_addr)
+{
 	PMAP_COUNT(copy);
 #ifdef DEBUG
 	if (pmapdebug & PDB_FOLLOW)
 		printf("pmap_copy(%p, %p, %#"PRIxVADDR", %#"PRIxVSIZE", %#"PRIxVADDR")\n",
 		    dst_pmap, src_pmap, dst_addr, len, src_addr);
 #endif
+}
 /*
  *	Routine:	pmap_collect
  *	Function:
  *		Garbage collects the physical map system for
  *		pages which are no longer used.
  *		Success need not be guaranteed -- that is, there
  *		may well be pages which are not referenced, but
  *		others may be collected.
  *	Usage:
  *		Called by the pageout daemon when pages are scarce.
  */
 void
 pmap_collect(pmap_t pmap)
+{
 	PMAP_COUNT(collect);
 #ifdef DEBUG
 	if (pmapdebug & PDB_FOLLOW)
 		printf("pmap_collect(%p)\n", pmap);
 #endif
+}
 /*
  *	pmap_zero_page zeros the specified page.
  */
 void
-pmap_zero_page(paddr_t phys)
+pmap_zero_page(paddr_t dst_pa)
+{
-	vaddr_t va;
+	vaddr_t dst_va;
-#if defined(MIPS3_PLUS)
+	pt_entry_t dst_tmp;
 	struct vm_page *pg;
 	pv_entry_t pv;
 #endif
 #ifdef DEBUG
 	if (pmapdebug & PDB_FOLLOW)
-		printf("pmap_zero_page(%#"PRIxPADDR")\n", phys);
+		printf("pmap_zero_page(%#"PRIxPADDR")\n", dst_pa);
 #endif
 #ifdef PARANOIADIAG
 	if (!(phys < MIPS_MAX_MEM_ADDR))
 		printf("pmap_zero_page(%#"PRIxPADDR") nonphys\n", phys);
 #endif
 #ifdef _LP64
 	KASSERT(mips_options.mips3_xkphys_cached);
 	va = MIPS_PHYS_TO_XKPHYS_CACHED(phys);
 #else
 	va = MIPS_PHYS_TO_KSEG0(phys);
 #endif
 	PMAP_COUNT(zeroed_pages);
-#if defined(MIPS3_PLUS)	/* XXX mmu XXX */
+	struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst_pa);
 	pg = PHYS_TO_VM_PAGE(phys);
 	if (mips_cache_info.mci_cache_virtual_alias) {
 		pv = pg->mdpage.pvh_list;
 		if ((pv->pv_flags & PV_UNCACHED) == 0 &&
 		    mips_cache_indexof(pv->pv_va) != mips_cache_indexof(va))
 			mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE);
 #endif
-	mips_pagezero((void *)va);
+	dst_va = pmap_map_ephemeral_page(dst_pg, VM_PROT_READ|VM_PROT_WRITE, &dst_tmp);
-#if defined(MIPS3_PLUS)	/* XXX mmu XXX */
+	mips_pagezero((void *)dst_va);
 	/*
-	 * If we have a virtually-indexed, physically-tagged WB cache,
+	pmap_unmap_ephemeral_page(dst_pg, dst_va, dst_tmp);
 	 * and no L2 cache to warn of aliased mappings,	we must force a
 	 * writeback of the destination out of the L1 cache.  If we don't,
 	 * later reads (from virtual addresses mapped to the destination PA)
 	 * might read old stale DRAM footprint, not the just-written data.
 	 * XXXJRT This is totally disgusting.
 	 */
 	if (MIPS_HAS_R4K_MMU)	/* XXX VCED on kernel stack is not allowed */
 		mips_dcache_wbinv_range(va, PAGE_SIZE);
 #endif	/* MIPS3_PLUS */
+}
 /*
  *	pmap_copy_page copies the specified page.
  */
 void
-pmap_copy_page(paddr_t src, paddr_t dst)
+pmap_copy_page(paddr_t src_pa, paddr_t dst_pa)
+{
 	vaddr_t src_va, dst_va;
 	pt_entry_t src_tmp, dst_tmp;
 #ifdef DEBUG
 	if (pmapdebug & PDB_FOLLOW)
-		printf("pmap_copy_page(%#"PRIxPADDR", %#"PRIxPADDR")\n", src, dst);
+		printf("pmap_copy_page(%#"PRIxPADDR", %#"PRIxPADDR")\n", src_pa, dst_pa);
 #endif
 #ifdef _LP64
 	KASSERT(mips_options.mips3_xkphys_cached);
 	src_va = MIPS_PHYS_TO_XKPHYS_CACHED(src);
 	dst_va = MIPS_PHYS_TO_XKPHYS_CACHED(dst);
 #else
 	src_va = MIPS_PHYS_TO_KSEG0(src);
 	dst_va = MIPS_PHYS_TO_KSEG0(dst);
 #endif
 #if !defined(_LP64) && defined(PARANOIADIAG)
 	if (!(src < MIPS_MAX_MEM_ADDR))
 		printf("pmap_copy_page(%#"PRIxPADDR") src nonphys\n", src);
 	if (!(dst < MIPS_MAX_MEM_ADDR))
 		printf("pmap_copy_page(%#"PRIxPADDR") dst nonphys\n", dst);
 #endif
 	struct vm_page *src_pg = PHYS_TO_VM_PAGE(src_pa);
 	struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst_pa);
-#if defined(MIPS3_PLUS) /* XXX mmu XXX */
+	PMAP_COUNT(copied_pages);
 	/*
-	 * If we have a virtually-indexed, physically-tagged cache,
+	src_va = pmap_map_ephemeral_page(src_pg, VM_PROT_READ, &src_tmp);
-	 * and no L2 cache to warn of aliased mappings, we must force an
+	dst_va = pmap_map_ephemeral_page(dst_pg, VM_PROT_READ|VM_PROT_WRITE, &dst_tmp);
 	 * write-back of all L1 cache lines of the source physical address,
 	 * irrespective of their virtual address (cache indexes).
 	 * If we don't, our copy loop might read and copy stale DRAM
 	 * footprint instead of the fresh (but dirty) data in a WB cache.
 	 * XXX invalidate any cached lines of the destination PA
 	 *     here also?
 	 * It would probably be better to map the destination as a
 	 * write-through no allocate to reduce cache thrash.
 	 */
 	if (mips_cache_info.mci_cache_virtual_alias) {
 		/*XXX FIXME Not very sophisticated */
 		mips_flushcache_allpvh(src);
 #if 0
 		mips_flushcache_allpvh(dst);
 #endif
 #endif	/* MIPS3_PLUS */
 	mips_pagecopy((void *)dst_va, (void *)src_va);
-#if defined(MIPS3_PLUS) /* XXX mmu XXX */
+	pmap_unmap_ephemeral_page(dst_pg, dst_va, dst_tmp);
-	/*
+	pmap_unmap_ephemeral_page(src_pg, src_va, src_tmp);
 	 * If we have a virtually-indexed, physically-tagged WB cache,
 	 * and no L2 cache to warn of aliased mappings,	we must force a
 	 * writeback of the destination out of the L1 cache.  If we don't,
 	 * later reads (from virtual addresses mapped to the destination PA)
 	 * might read old stale DRAM footprint, not the just-written data.
 	 * XXX  Do we need to also invalidate any cache lines matching
 	 *      the destination as well?
 	 * XXXJRT -- This is totally disgusting.
 	 */
 	if (mips_cache_info.mci_cache_virtual_alias) {
 		mips_dcache_wbinv_range(src_va, PAGE_SIZE);
 		mips_dcache_wbinv_range(dst_va, PAGE_SIZE);
 #endif	/* MIPS3_PLUS */
+}
 /*
  *	pmap_clear_reference:
+ *
  *	Clear the reference bit on the specified physical page.
  */
 bool
 pmap_clear_reference(struct vm_page *pg)
+{
 #ifdef DEBUG
 	if (pmapdebug & PDB_FOLLOW)
 		printf("pmap_clear_reference(%#"PRIxPADDR")\n",
 		    VM_PAGE_TO_PHYS(pg));
 #endif
-	return pmap_clear_page_attributes(pg, PV_REFERENCED);
+	return pmap_clear_page_attributes(pg, PG_MD_REFERENCED);
+}
 /*
  *	pmap_is_referenced:
+ *
  *	Return whether or not the specified physical page is referenced
  *	by any physical maps.
  */
 bool
 pmap_is_referenced(struct vm_page *pg)
+{
-	return pg->mdpage.pvh_attrs & PV_REFERENCED;
+	return PG_MD_REFERENCED_P(pg);
+}
 /*
  *	Clear the modify bits on the specified physical page.
  */
 bool
 pmap_clear_modify(struct vm_page *pg)
+{
-	struct pv_entry *pv;
+	struct pv_entry *pv = &pg->mdpage.pvh_first;
 	PMAP_COUNT(clear_modify);
 #ifdef DEBUG
 	if (pmapdebug & PDB_FOLLOW)
 		printf("pmap_clear_modify(%#"PRIxPADDR")\n", VM_PAGE_TO_PHYS(pg));
 #endif
-	if (!pmap_clear_page_attributes(pg, PV_MODIFIED))
+	if (PG_MD_EXECPAGE_P(pg)) {
 		if (pv->pv_pmap == NULL) {
 			pmap_clear_page_attributes(pg, PG_MD_EXECPAGE);
 			PMAP_COUNT(exec_uncached_clear_modify);
 		} else {
 			pmap_page_syncicache(pg);
 			PMAP_COUNT(exec_synced_clear_modify);
+		}
+	}
 	if (!pmap_clear_page_attributes(pg, PG_MD_MODIFIED))
 		return false;
 	pv = pg->mdpage.pvh_list;
 	if (pv->pv_pmap == NULL) {
 		return true;
+	}
 	/*
 	 * remove write access from any pages that are dirty
 	 * so we can tell if they are written to again later.
 	 * flush the VAC first if there is one.
 	 */
 	for (; pv; pv = pv->pv_next) {
 		pmap_t pmap = pv->pv_pmap;
 		vaddr_t va = pv->pv_va;
 		pt_entry_t *pte;
 @@ -1736,442 +1785,450 @@ pmap_clear_modify(struct vm_page *pg)
 		} else {
 			pte = pmap_pte_lookup(pmap, va);
 			KASSERT(pte);
+		}
 		pt_entry = pte->pt_entry & ~mips_pg_m_bit();
 		if (pte->pt_entry == pt_entry) {
 			continue;
+		}
 		KASSERT(pt_entry & MIPS3_PG_V);
 		/*
 		 * Why? Why?
 		 */
 		if (MIPS_HAS_R4K_MMU
-		    && mips_cache_info.mci_cache_virtual_alias) {
+		    && MIPS_CACHE_VIRTUAL_ALIAS) {
 			if (PMAP_IS_ACTIVE(pmap)) {
 				mips_dcache_wbinv_range(va, PAGE_SIZE);
 			} else {
 				mips_dcache_wbinv_range_index(va, PAGE_SIZE);
+			}
+		}
 		pte->pt_entry = pt_entry;
 		pmap_tlb_invalidate_addr(pmap, va);
+	}
 	return true;
+}
 /*
  *	pmap_is_modified:
+ *
  *	Return whether or not the specified physical page is modified
  *	by any physical maps.
  */
 bool
 pmap_is_modified(struct vm_page *pg)
+{
-	return pg->mdpage.pvh_attrs & PV_MODIFIED;
+	return PG_MD_MODIFIED_P(pg);
+}
 /*
  *	pmap_set_modified:
+ *
  *	Sets the page modified reference bit for the specified page.
  */
 void
 pmap_set_modified(paddr_t pa)
+{
 	struct vm_page *pg = PHYS_TO_VM_PAGE(pa);
-	pmap_set_page_attributes(pg, PV_MODIFIED | PV_REFERENCED);
+	pmap_set_page_attributes(pg, PG_MD_MODIFIED | PG_MD_REFERENCED);
+}
 /******************** pv_entry management ********************/
 /*
  * Enter the pmap and virtual address into the
  * physical to virtual map table.
  */
 void
 pmap_enter_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, u_int *npte)
+{
 	pv_entry_t pv, npv;
-	pv = pg->mdpage.pvh_list;
+	pv = &pg->mdpage.pvh_first;
 #ifdef DEBUG
 	if (pmapdebug & PDB_ENTER)
 		printf("pmap_enter: pv %p: was %#"PRIxVADDR"/%p/%p\n",
 		    pv, pv->pv_va, pv->pv_pmap, pv->pv_next);
 #endif
 #if defined(MIPS3_NO_PV_UNCACHED)
 again:
 #endif
 	if (pv->pv_pmap == NULL) {
 		KASSERT(pv->pv_next == NULL);
 		/*
 		 * No entries yet, use header as the first entry
 		 */
 #ifdef DEBUG
 		if (pmapdebug & PDB_PVENTRY)
 			printf("pmap_enter: first pv: pmap %p va %#"PRIxVADDR"\n",
 			    pmap, va);
 		enter_stats.firstpv++;
 #endif
 		PMAP_COUNT(primary_mappings);
 		PMAP_COUNT(mappings);
 		pmap_clear_page_attributes(pg, PG_MD_UNCACHED);
 		pv->pv_va = va;
 		pv->pv_flags &= ~PV_UNCACHED;
 		pv->pv_pmap = pmap;
 		pv->pv_next = NULL;
 	} else {
 #if defined(MIPS3_PLUS) /* XXX mmu XXX */
-		if (mips_cache_info.mci_cache_virtual_alias) {
+		if (MIPS_CACHE_VIRTUAL_ALIAS) {
 			/*
 			 * There is at least one other VA mapping this page.
 			 * Check if they are cache index compatible.
 			 */
 #if defined(MIPS3_NO_PV_UNCACHED)
 			/*
 			 * Instead of mapping uncached, which some platforms
 			 * cannot support, remove the mapping from the pmap.
 			 * When this address is touched again, the uvm will
 			 * fault it in.  Because of this, each page will only
 			 * be mapped with one index at any given time.
 			 */
-			for (npv = pv; npv; npv = npv->pv_next) {
+			if (mips_cache_badalias(pv->pv_va, va)) {
-				if (mips_cache_indexof(npv->pv_va) !=
+				for (npv = pv; npv; npv = npv->pv_next) {
 				    mips_cache_indexof(va)) {
 					pmap_remove(npv->pv_pmap, npv->pv_va,
 					    npv->pv_va + PAGE_SIZE);
 					pmap_update(npv->pv_pmap);
 					goto again;
+				}
+			}
 #else	/* !MIPS3_NO_PV_UNCACHED */
-			if (!(pv->pv_flags & PV_UNCACHED)) {
+			if (PG_MD_CACHED_P(pg)) {
-				for (npv = pv; npv; npv = npv->pv_next) {
+				/*
 				 * If this page is cached, then all mappings
-					/*
+				 * have the same cache alias so we only need
-					 * Check cache aliasing incompatibility.
+				 * to check the first page to see if it's
-					 * If one exists, re-map this page
+				 * incompatible with the new mapping.
 					 * uncached until all mappings have
-					 * the same index again.
+				 * If the mappings are incompatible, map this
-					 */
+				 * page as uncached and re-map all the current
-					if (mips_cache_indexof(npv->pv_va) !=
+				 * mapping as uncached until all pages can
-					    mips_cache_indexof(va)) {
+				 * share the same cache index again.
-						pmap_page_cache(pg,PV_UNCACHED);
+				 */
-						mips_dcache_wbinv_range_index(
+				if (mips_cache_badalias(pv->pv_va, va)) {
-						    pv->pv_va, PAGE_SIZE);
+					pmap_page_cache(pg, false);
-						*npte = (*npte &
+					mips_dcache_wbinv_range_index(
-						    ~MIPS3_PG_CACHEMODE) |
+					    pv->pv_va, PAGE_SIZE);
-						    MIPS3_PG_UNCACHED;
+					*npte = (*npte &
-#ifdef DEBUG
+					    ~MIPS3_PG_CACHEMODE) |
-						enter_stats.ci++;
+					    MIPS3_PG_UNCACHED;
-#endif
+					PMAP_COUNT(page_cache_evictions);
 						break;
+				}
 			} else {
 				*npte = (*npte & ~MIPS3_PG_CACHEMODE) |
 				    MIPS3_PG_UNCACHED;
 				PMAP_COUNT(page_cache_evictions);
+			}
 #endif	/* !MIPS3_NO_PV_UNCACHED */
+		}
 #endif /* MIPS3_PLUS */
 		/*
 		 * There is at least one other VA mapping this page.
 		 * Place this entry after the header.
+		 *
 		 * Note: the entry may already be in the table if
 		 * we are only changing the protection bits.
 		 */
 		for (npv = pv; npv; npv = npv->pv_next) {
 			if (pmap == npv->pv_pmap && va == npv->pv_va) {
 #ifdef PARANOIADIAG
 				pt_entry_t *pte;
-				unsigned pt_entry;
+				uint32_t pt_entry;
 				if (pmap == pmap_kernel()) {
 					pt_entry = kvtopte(va)->pt_entry;
 				} else {
 					pte = pmap_pte_lookup(pmap, va);
 					if (pte) {
 						pt_entry = pte->pt_entry;
 					} else
 						pt_entry = 0;
+				}
 				if (!mips_pg_v(pt_entry) ||
 				    mips_tlbpfn_to_paddr(pt_entry) !=
 				    VM_PAGE_TO_PHYS(pg))
 					printf(
 		"pmap_enter: found va %#"PRIxVADDR" pa %#"PRIxPADDR" in pv_table but != %x\n",
 					    va, VM_PAGE_TO_PHYS(pg),
 					    pt_entry);
 #endif
 				PMAP_COUNT(remappings);
 				return;
+			}
+		}
 #ifdef DEBUG
 		if (pmapdebug & PDB_PVENTRY)
 			printf("pmap_enter: new pv: pmap %p va %#"PRIxVADDR"\n",
 			    pmap, va);
 #endif
 		npv = (pv_entry_t)pmap_pv_alloc();
 		if (npv == NULL)
 			panic("pmap_enter_pv: pmap_pv_alloc() failed");
 		npv->pv_va = va;
 		npv->pv_pmap = pmap;
 		npv->pv_flags = pv->pv_flags;
 		npv->pv_next = pv->pv_next;
 		pv->pv_next = npv;
-#ifdef DEBUG
+		PMAP_COUNT(mappings);
 		if (!npv->pv_next)
 			enter_stats.secondpv++;
 #endif
+	}
+}
 /*
  * Remove a physical to virtual address translation.
  * If cache was inhibited on this page, and there are no more cache
  * conflicts, restore caching.
  * Flush the cache if the last page is removed (should always be cached
  * at this point).
  */
 void
-pmap_remove_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg)
+pmap_remove_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, bool dirty)
+{
 	pv_entry_t pv, npv;
-	int last;
+	bool last;
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_PVENTRY))
 		printf("pmap_remove_pv(%p, %#"PRIxVADDR", %#"PRIxPADDR")\n", pmap, va,
 		    VM_PAGE_TO_PHYS(pg));
 #endif
-	pv = pg->mdpage.pvh_list;
+	pv = &pg->mdpage.pvh_first;
 	/*
 	 * If it is the first entry on the list, it is actually
 	 * in the header and we must copy the following entry up
 	 * to the header.  Otherwise we must search the list for
 	 * the entry.  In either case we free the now unused entry.
 	 */
-	last = 0;
+	last = false;
 	if (pmap == pv->pv_pmap && va == pv->pv_va) {
 		npv = pv->pv_next;
 		if (npv) {
 			/*
 			 * Copy current modified and referenced status to
 			 * the following entry before copying.
 			 */
 			npv->pv_flags |=
 			    pv->pv_flags & (PV_MODIFIED | PV_REFERENCED);
 			*pv = *npv;
 			pmap_pv_free(npv);
 		} else {
 			pmap_clear_page_attributes(pg, PG_MD_UNCACHED);
 			pv->pv_pmap = NULL;
-			last = 1;	/* Last mapping removed */
+			last = true;	/* Last mapping removed */
+		}
-#ifdef DEBUG
+		PMAP_COUNT(remove_pvfirst);
 		remove_stats.pvfirst++;
 #endif
 	} else {
 		for (npv = pv->pv_next; npv; pv = npv, npv = npv->pv_next) {
-#ifdef DEBUG
+			PMAP_COUNT(remove_pvsearch);
 			remove_stats.pvsearch++;
 #endif
 			if (pmap == npv->pv_pmap && va == npv->pv_va)
 				break;
+		}
 		if (npv) {
 			pv->pv_next = npv->pv_next;
 			pmap_pv_free(npv);
+		}
+	}
 	if (PG_MD_EXECPAGE_P(pg) && dirty) {
 		if (last) {
 			/*
 			 * If this was the page's last mapping, we no longer
 			 * care about it execness.
 			 */
 			pmap_clear_page_attributes(pg, PG_MD_EXECPAGE);
 			PMAP_COUNT(exec_uncached_remove);
 		} else {
 			/*
 			 * Someone still has it mapped as an executable page
 			 * so we must sync it.
 			 */
 			pmap_page_syncicache(pg);
 			PMAP_COUNT(exec_synced_remove);
+		}
+	}
 #ifdef MIPS3_PLUS	/* XXX mmu XXX */
-#if !defined(MIPS3_NO_PV_UNCACHED)
+#ifndef MIPS3_NO_PV_UNCACHED
-	if (MIPS_HAS_R4K_MMU && pv->pv_flags & PV_UNCACHED) {
+	if (MIPS_HAS_R4K_MMU && PG_MD_UNCACHED_P(pg)) {
 		/*
 		 * Page is currently uncached, check if alias mapping has been
 		 * removed.  If it was, then reenable caching.
 		 */
-		pv = pg->mdpage.pvh_list;
+		pv = &pg->mdpage.pvh_first;
 		for (npv = pv->pv_next; npv; npv = npv->pv_next) {
-			if (mips_cache_indexof(pv->pv_va ^ npv->pv_va))
+			if (mips_cache_badalias(pv->pv_va, npv->pv_va))
 				break;
+		}
 		if (npv == NULL)
-			pmap_page_cache(pg, 0);
+			pmap_page_cache(pg, true);
+	}
 #endif
-	if (MIPS_HAS_R4K_MMU && last != 0)
+	if (MIPS_HAS_R4K_MMU && last)	/* XXX why */
 		mips_dcache_wbinv_range_index(va, PAGE_SIZE);
 #endif	/* MIPS3_PLUS */
+}
 /*
  * pmap_pv_page_alloc:
+ *
  *	Allocate a page for the pv_entry pool.
  */
 void *
 pmap_pv_page_alloc(struct pool *pp, int flags)
+{
-	const struct vm_page *pg = PMAP_ALLOC_POOLPAGE(UVM_PGA_USERESERVE);
+	struct vm_page *pg = PMAP_ALLOC_POOLPAGE(UVM_PGA_USERESERVE);
 	if (pg == NULL)
 		return NULL;
-	const paddr_t pa = VM_PAGE_TO_PHYS(pg);
+	return (void *)mips_pmap_map_poolpage(VM_PAGE_TO_PHYS(pg));
 #ifdef _LP64
 	KASSERT(mips_options.mips3_xkphys_cached);
 	const vaddr_t va = MIPS_PHYS_TO_XKPHYS_CACHED(pa);
 #else
 	const vaddr_t va = MIPS_PHYS_TO_KSEG0(pa);
 #endif
 #if defined(MIPS3_PLUS)
 	if (mips_cache_info.mci_cache_virtual_alias) {
 		pv_entry_t pv = pg->mdpage.pvh_list;
 		if ((pv->pv_flags & PV_UNCACHED) == 0 &&
 		    mips_cache_indexof(pv->pv_va) != mips_cache_indexof(va))
 			mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE);
 #endif
 	return (void *)va;
+}
 /*
  * pmap_pv_page_free:
+ *
  *	Free a pv_entry pool page.
  */
 void
 pmap_pv_page_free(struct pool *pp, void *v)
+{
-	paddr_t phys;
+	vaddr_t va = (vaddr_t)v;
 	paddr_t pa;
 #ifdef MIPS3_PLUS
 	if (mips_cache_info.mci_cache_virtual_alias)
 		mips_dcache_inv_range((vaddr_t)v, PAGE_SIZE);
 #endif
 #ifdef _LP64
-	KASSERT(MIPS_XKPHYS_P(v));
+	KASSERT(MIPS_XKPHYS_P(va));
-	phys = MIPS_XKPHYS_TO_PHYS((vaddr_t)v);
+	pa = MIPS_XKPHYS_TO_PHYS(va);
 #else
-	phys = MIPS_KSEG0_TO_PHYS((vaddr_t)v);
+	KASSERT(MIPS_KSEG0_P(va));
 	pa = MIPS_KSEG0_TO_PHYS(va);
 #endif
-	uvm_pagefree(PHYS_TO_VM_PAGE(phys));
+#ifdef MIPS3_PLUS
 	if (MIPS_CACHE_VIRTUAL_ALIAS)
 		mips_dcache_inv_range(va, PAGE_SIZE);
 #endif
 	struct vm_page *pg = PHYS_TO_VM_PAGE(pa);
 	pmap_clear_page_attributes(pg, PG_MD_POOLPAGE);
 	uvm_pagefree(pg);
+}
 pt_entry_t *
 pmap_pte(pmap_t pmap, vaddr_t va)
+{
 	pt_entry_t *pte;
 	if (pmap == pmap_kernel())
 		pte = kvtopte(va);
 	else
 		pte = pmap_pte_lookup(pmap, va);
 	return pte;
+}
 #ifdef MIPS3_PLUS	/* XXX mmu XXX */
 /*
  * Find first virtual address >= *vap that doesn't cause
  * a cache alias conflict.
  */
 void
-pmap_prefer(vaddr_t foff, vaddr_t *vap, int td)
+pmap_prefer(vaddr_t foff, vaddr_t *vap, vsize_t sz, int td)
+{
 	const struct mips_cache_info * const mci = &mips_cache_info;
 	vaddr_t	va;
 	vsize_t d;
 	vsize_t prefer_mask = ptoa(uvmexp.colormask);
 	PMAP_COUNT(prefer_requests);
 	if (MIPS_HAS_R4K_MMU) {
 		prefer_mask |= mci->mci_cache_prefer_mask;
+	}
 	if (prefer_mask) {
 		va = *vap;
 		d = foff - va;
-		d &= mci->mci_cache_prefer_mask;
+		d &= prefer_mask;
-		if (td && d)
+		if (d) {
-			d = -((-d) & mci->mci_cache_prefer_mask);
+			if (td)
-		*vap = va + d;
+				*vap = trunc_page(va -((-d) & prefer_mask));
 			else
 				*vap = round_page(va + d);
 			PMAP_COUNT(prefer_adjustments);
+		}
+	}
+}
 #endif	/* MIPS3_PLUS */
 struct vm_page *
 mips_pmap_alloc_poolpage(int flags)
+{
 	/*
 	 * On 32bit kernels, we must make sure that we only allocate pages that
 	 * can be mapped via KSEG0.  On 64bit kernels, try to allocated from
 	 * the first 4G.  If all memory is in KSEG0/4G, then we can just
 	 * use the default freelist otherwise we must use the pool page list.
 	 */
 	if (mips_poolpage_vmfreelist != VM_FREELIST_DEFAULT)
 		return uvm_pagealloc_strat(NULL, 0, NULL, flags,
 		    UVM_PGA_STRAT_ONLY, mips_poolpage_vmfreelist);
 	return uvm_pagealloc(NULL, 0, NULL, flags);
+}
 vaddr_t
 mips_pmap_map_poolpage(paddr_t pa)
+{
 	vaddr_t va;
 #if defined(MIPS3_PLUS)
 	struct vm_page *pg;
 	pv_entry_t pv;
 #endif
 #ifdef _LP64
 	KASSERT(mips_options.mips3_xkphys_cached);
 	va = MIPS_PHYS_TO_XKPHYS_CACHED(pa);
 #else
 	if (pa > MIPS_PHYS_MASK)
 		panic("mips_pmap_map_poolpage: "
 		    "pa #%"PRIxPADDR" can not be mapped into KSEG0", pa);
 	va = MIPS_PHYS_TO_KSEG0(pa);
 #endif
 	struct vm_page *pg = PHYS_TO_VM_PAGE(pa);
 	KASSERT(pg);
 	pmap_set_page_attributes(pg, PG_MD_POOLPAGE);
 #if defined(MIPS3_PLUS)
-	if (mips_cache_info.mci_cache_virtual_alias) {
+	if (MIPS_CACHE_VIRTUAL_ALIAS) {
-		pg = PHYS_TO_VM_PAGE(pa);
+		/*
-		pv = pg->mdpage.pvh_list;
+		 * If this page was last mapped with an address that might
-		if ((pv->pv_flags & PV_UNCACHED) == 0 &&
+		 * cause aliases, flush the page from the cache.
-		    mips_cache_indexof(pv->pv_va) != mips_cache_indexof(va))
+		 */
-			mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE);
+		pv_entry_t pv = &pg->mdpage.pvh_first;
 		KASSERT(pv->pv_pmap == NULL);
 		if (PG_MD_CACHED_P(pg) && mips_cache_badalias(pv->pv_va, va))
 			mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE);
 		pv->pv_va = va;
+	}
 #endif
 	return va;
+}
 paddr_t
 mips_pmap_unmap_poolpage(vaddr_t va)
+{
 	paddr_t pa;
 #ifdef _LP64
 	KASSERT(MIPS_XKPHYS_P(va));
 	pa = MIPS_XKPHYS_TO_PHYS(va);
 #else
 	KASSERT(MIPS_KSEG0_P(va));
 	pa = MIPS_KSEG0_TO_PHYS(va);
 #endif
 	struct vm_page *pg = PHYS_TO_VM_PAGE(pa);
 	pmap_clear_page_attributes(pg, PG_MD_POOLPAGE);
 #if defined(MIPS3_PLUS)
-	if (mips_cache_info.mci_cache_virtual_alias) {
+	if (MIPS_CACHE_VIRTUAL_ALIAS) {
 		mips_dcache_inv_range(va, PAGE_SIZE);
+	}
 #endif
 	return pa;
+}
 /******************** page table page management ********************/
 /* TO BE DONE */

 @@ -1,14 +1,14 @@
-#	$NetBSD: Makefile.mips,v 1.50.24.3 2009/09/16 03:22:03 matt Exp $
+#	$NetBSD: Makefile.mips,v 1.50.24.4 2010/01/26 21:19:25 matt Exp $
 # Makefile for NetBSD
+#
 # This makefile is constructed from a machine description:
 #	config machineid
 # Most changes should be made in the machine description
 #	/sys/arch/<machine>/conf/``machineid''
 # after which you should do
 #	config machineid
 # Machine generic makefile changes should be made in
 #	/sys/arch/mips/conf/Makefile.mips
 # after which config should be rerun for all machines of that type.
+#
 @@ -53,28 +53,28 @@ CFLAGS+=	-msym32 -mabi=64
 AFLAGS+=	-msym32 -mabi=64
 .endif
 .if ${MACHINE_ARCH} == "mips64eb"
 LDFLAGS+=	-m elf64btsmip
 LINKFORMAT+=	-m elf64btsmip
 SYSTEM_LD_TAIL_EXTRA+= \
 		;echo ${OBJCOPY} -O elf32-nbigmips $@ $@.elf32; \
 		${OBJCOPY} -O elf32-nbigmips $@ $@.elf32
 .endif
 .if ${MACHINE_ARCH} == "mips64el"
 LDFLAGS+=	-m elf64ltsmip
 LINKFORMAT+=	-m elf64ltsmip
 SYSTEM_LD_TAIL_EXTRA+= \
-		;echo ${OBJCOPY} -O elf32-nlittlemips $@ $@.elf32; \
+	;echo ${OBJCOPY} -O elf32-nlittlemips $@ $@.elf32; \
-		${OBJCOPY} -O elf32-nlittlemips $@ $@.elf32
+	${OBJCOPY} -O elf32-nlittlemips $@ $@.elf32
 .endif
 .endif # LP64=yes
 AFLAGS+=	-mno-abicalls -x assembler-with-cpp -traditional-cpp ${AOPTS}
 ##
 ## (3) libkern and compat
 ##
 .if !empty(IDENT:M-DLKM)
 KERN_AS=	obj
 .endif
 ##
 ## (4) local objects, compile rules, and dependencies

 @@ -1,14 +1,14 @@
-/*	$NetBSD: cache.h,v 1.9.96.1 2010/01/20 06:58:35 matt Exp $	*/
+/*	$NetBSD: cache.h,v 1.9.96.2 2010/01/26 21:19:25 matt Exp $	*/
 /*
  * Copyright 2001 Wasabi Systems, Inc.
  * All rights reserved.
+ *
  * Written by Jason R. Thorpe for Wasabi Systems, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
 @@ -196,37 +196,50 @@ struct mips_cache_info {
 	/*
 	 * These two variables inform the rest of the kernel about the
 	 * size of the largest D-cache line present in the system.  The
 	 * mask can be used to determine if a region of memory is cache
 	 * line size aligned.
+	 *
 	 * Whenever any code updates a data cache line size, it should
 	 * call mips_dcache_compute_align() to recompute these values.
 	 */
 	u_int mci_dcache_align;
 	u_int mci_dcache_align_mask;
 	u_int mci_cache_alias_mask;
 	u_int mci_cache_prefer_mask;
 #if defined(MIPS2) || defined(MIPS3) || defined(MIPS3_5900)
 	u_int mci_cache_alias_mask;
 	bool mci_cache_virtual_alias;
 #define	MIPS_CACHE_ALIAS_MASK		mips_cache_info.mci_cache_alias_mask
 #define	MIPS_CACHE_VIRTUAL_ALIAS	mips_cache_info.mci_cache_virtual_alias
 #elif defined(MIPS1) || defined(MIPS32) || defined(MIPS64)
 #define	MIPS_CACHE_ALIAS_MASK		0
 #define	MIPS_CACHE_VIRTUAL_ALIAS	false
 #else
 #error mci_cache screw up
 #endif
 };
 extern struct mips_cache_info mips_cache_info;
 /*
  * XXX XXX XXX THIS SHOULD NOT EXIST XXX XXX XXX
  */
-#define	mips_cache_indexof(x)	(((vaddr_t)(x)) & mips_cache_info.mci_cache_alias_mask)
+#define	mips_cache_indexof(x)	(((vaddr_t)(x)) & MIPS_CACHE_ALIAS_MASK)
 #define	mips_cache_badalias(x,y) (((vaddr_t)(x)^(vaddr_t)(y)) & MIPS_CACHE_ALIAS_MASK)
 #define	__mco_noargs(prefix, x)						\
 do {									\
 	(*mips_cache_ops.mco_ ## prefix ## p ## x )();			\
 	if (*mips_cache_ops.mco_ ## prefix ## s ## x )			\
 		(*mips_cache_ops.mco_ ## prefix ## s ## x )();		\
 } while (/*CONSTCOND*/0)
 #define	__mco_2args(prefix, x, a, b)					\
 do {									\
 	(*mips_cache_ops.mco_ ## prefix ## p ## x )((a), (b));		\
 	if (*mips_cache_ops.mco_ ## prefix ## s ## x )			\
 		(*mips_cache_ops.mco_ ## prefix ## s ## x )((a), (b));	\

 @@ -1,14 +1,14 @@
-/*	$NetBSD: cpu.h,v 1.90.16.14 2010/01/20 09:04:34 matt Exp $	*/
+/*	$NetBSD: cpu.h,v 1.90.16.15 2010/01/26 21:19:25 matt Exp $	*/
 /*-
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.  All rights reserved.
+ *
  * This code is derived from software contributed to Berkeley by
  * Ralph Campbell and Rick Macklem.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -132,26 +132,28 @@ struct cpu_info {
 	struct lwp *ci_fpcurlwp;	/* the current FPU owner */
 	int ci_want_resched;		/* user preemption pending */
 	int ci_mtx_count;		/* negative count of held mutexes */
 	int ci_mtx_oldspl;		/* saved SPL value */
 	int ci_idepth;			/* hardware interrupt depth */
 	device_t ci_dev;		/* owning device */
 	vaddr_t ci_ebase;		/* VA of exception base */
 	paddr_t ci_ebase_pa;		/* PA of exception base */
 	u_long ci_cctr_freq;		/* cycle counter frequency */
 	/*
 	 * Per-cpu pmap information
 	 */
 	struct segtab *ci_pmap_segbase;
 	vaddr_t ci_pmap_srcbase;	/* starting VA of ephemeral src space */
 	vaddr_t ci_pmap_dstbase;	/* starting VA of ephemeral dst space */
 	uint32_t ci_pmap_asid_next;	/* next asid to be assigned */
 	uint32_t ci_pmap_asid_generation; /* current asid generation */
 	uint32_t ci_pmap_asid_reserved;	/* base of ASID space */
 	uint32_t ci_pmap_asid_max;	/* max (exclusive) assignable asid */
 };
 #define	CPU_INFO_ITERATOR		int
 #define	CPU_INFO_FOREACH(cii, ci)	\
     (void)(cii), ci = &cpu_info_store; ci != NULL; ci = ci->ci_next
 #endif /* !_LOCORE */
 #endif /* _KERNEL */

 @@ -1,14 +1,14 @@
-/*	$NetBSD: mips3_pte.h,v 1.23.38.5 2010/01/20 06:58:35 matt Exp $	*/
+/*	$NetBSD: mips3_pte.h,v 1.23.38.6 2010/01/26 21:19:25 matt Exp $	*/
 /*
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.  All rights reserved.
+ *
  * This code is derived from software contributed to Berkeley by
  * the Systems Programming Group of the University of Utah Computer
  * Science Department and Ralph Campbell.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
 @@ -162,27 +162,27 @@ unsigned int 	pg_g:1,			/* HW: ignore as
 /* pte accessor macros */
 #define mips3_pfn_is_ext(x) ((x) & 0x3c000000)
 #define mips3_paddr_to_tlbpfn(x) \
     (((paddr_t)(x) >> MIPS3_PG_SHIFT) & MIPS3_PG_FRAME)
 #define mips3_tlbpfn_to_paddr(x) \
     ((paddr_t)((x) & MIPS3_PG_FRAME) << MIPS3_PG_SHIFT)
 #define mips3_vad_to_vpn(x) ((vaddr_t)(x) & MIPS3_PG_SVPN)
 #define mips3_vpn_to_vad(x) ((x) & MIPS3_PG_SVPN)
 #define MIPS3_PTE_TO_PADDR(pte) (mips3_tlbpfn_to_paddr(pte))
 #define MIPS3_PAGE_IS_RDONLY(pte,va) \
-    (pmap_is_page_ro(pmap_kernel(), mips_trunc_page(va), (pte)))
+    (pmap_is_page_ro_p(pmap_kernel(), mips_trunc_page(va), (pte)))
 #define	MIPS3_PG_SIZE_4K	0x00000000
 #define	MIPS3_PG_SIZE_16K	0x00006000
 #define	MIPS3_PG_SIZE_64K	0x0001e000
 #define	MIPS3_PG_SIZE_256K	0x0007e000
 #define	MIPS3_PG_SIZE_1M	0x001fe000
 #define	MIPS3_PG_SIZE_4M	0x007fe000
 #define	MIPS3_PG_SIZE_16M	0x01ffe000
 #define	MIPS3_PG_SIZE_64M	0x07ffe000
 #define	MIPS3_PG_SIZE_256M	0x1fffe000
 #define	MIPS3_PG_SIZE_MASK_TO_SIZE(pg_mask)	\

 @@ -1,14 +1,14 @@
-/*	$NetBSD: pmap.h,v 1.54.26.7 2010/01/22 07:41:10 matt Exp $	*/
+/*	$NetBSD: pmap.h,v 1.54.26.8 2010/01/26 21:19:25 matt Exp $	*/
 /*
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.  All rights reserved.
+ *
  * This code is derived from software contributed to Berkeley by
  * Ralph Campbell.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -159,43 +159,26 @@ struct pmap_asid_info {
  * Machine dependent pmap structure.
  */
 typedef struct pmap {
 	kmutex_t		pm_lock;	/* lock on pmap */
 	struct segtab		*pm_segtab;	/* pointers to pages of PTEs */
 #ifdef MULTIPROCESSOR
 	uint32_t		pm_cpus;	/* pmap was active on ... */
 #endif
 	int			pm_count;	/* pmap reference count */
 	struct pmap_statistics	pm_stats;	/* pmap statistics */
 	struct pmap_asid_info	pm_pai[1];
 } *pmap_t;
 /*
  * For each struct vm_page, there is a list of all currently valid virtual
  * mappings of that page.  An entry is a pv_entry_t, the list is pv_table.
  * XXX really should do this as a part of the higher level code.
  */
 typedef struct pv_entry {
 	struct pv_entry	*pv_next;	/* next pv_entry */
 	struct pmap	*pv_pmap;	/* pmap where mapping lies */
 	vaddr_t		pv_va;		/* virtual address for mapping */
 	int		pv_flags;	/* some flags for the mapping */
 } *pv_entry_t;
 #define	PV_UNCACHED	0x0001		/* page is mapped uncached */
 #define	PV_MODIFIED	0x0002		/* page has been modified */
 #define	PV_REFERENCED	0x0004		/* page has been recently referenced */
 #ifdef	_KERNEL
 struct pmap_kernel {
 	struct pmap kernel_pmap;
 #ifdef MULTIPROCESSOR
 	struct pmap_asid_info kernel_pai[MAXCPUS-1];
 #endif
 };
 extern struct pmap_kernel kernel_pmap_store;
 extern paddr_t mips_avail_start;
 extern paddr_t mips_avail_end;
 extern vaddr_t mips_virtual_end;
 @@ -221,28 +204,28 @@ void	pmap_set_modified(paddr_t);
 void	pmap_procwr(struct proc *, vaddr_t, size_t);
 #define	PMAP_NEED_PROCWR
 uint32_t pmap_tlb_asid_alloc(pmap_t pmap, struct cpu_info *ci);
 void	pmap_tlb_invalidate_asid(pmap_t pmap);
 int	pmap_tlb_update(pmap_t pmap, vaddr_t, uint32_t);
 void	pmap_tlb_invalidate_addr(pmap_t pmap, vaddr_t);
 /*
  * pmap_prefer() helps reduce virtual-coherency exceptions in
  * the virtually-indexed cache on mips3 CPUs.
  */
 #ifdef MIPS3_PLUS
-#define PMAP_PREFER(pa, va, sz, td)	pmap_prefer((pa), (va), (td))
+#define PMAP_PREFER(pa, va, sz, td)	pmap_prefer((pa), (va), (sz), (td))
-void	pmap_prefer(vaddr_t, vaddr_t *, int);
+void	pmap_prefer(vaddr_t, vaddr_t *, vsize_t, int);
 #endif /* MIPS3_PLUS */
 #define	PMAP_STEAL_MEMORY	/* enable pmap_steal_memory() */
 /*
  * Alternate mapping hooks for pool pages.  Avoids thrashing the TLB.
  */
 vaddr_t mips_pmap_map_poolpage(paddr_t);
 paddr_t mips_pmap_unmap_poolpage(vaddr_t);
 struct vm_page *mips_pmap_alloc_poolpage(int);
 #define	PMAP_ALLOC_POOLPAGE(flags)	mips_pmap_alloc_poolpage(flags)
 #define	PMAP_MAP_POOLPAGE(pa)		mips_pmap_map_poolpage(pa)
 #define	PMAP_UNMAP_POOLPAGE(va)		mips_pmap_unmap_poolpage(va)

 @@ -1,14 +1,14 @@
-/*	$NetBSD: pte.h,v 1.19.18.1 2009/12/30 04:51:26 matt Exp $	*/
+/*	$NetBSD: pte.h,v 1.19.18.2 2010/01/26 21:19:25 matt Exp $	*/
 /*-
  * Copyright (c) 1997 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
  * NASA Ames Research Center.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
 @@ -57,27 +57,27 @@
 typedef union pt_entry {
 	unsigned int	 pt_entry;	/* for copying, etc. */
 	struct mips1_pte pt_mips1_pte;	/* for getting to bits by name */
 	struct mips3_pte pt_mips3_pte;
 } pt_entry_t;
 /*
  * Macros/inline functions to hide PTE format differences.
  */
 #define	mips_pg_nv_bit()	(MIPS1_PG_NV)	/* same on mips1 and mips3 */
-int pmap_is_page_ro(struct pmap *pmap, vaddr_t, int);
+bool pmap_is_page_ro_p(struct pmap *pmap, vaddr_t, uint32_t);
 /* MIPS1-only */
 #if defined(MIPS1) && !defined(MIPS3_PLUS)
 #define	mips_pg_v(entry)	((entry) & MIPS1_PG_V)
 #define	mips_pg_wired(entry)	((entry) & MIPS1_PG_WIRED)
 #define	mips_pg_m_bit()		(MIPS1_PG_D)
 #define	mips_pg_rw_bit()	(MIPS1_PG_RW)	/* no RW bits for mips1 */
 #define	mips_pg_ro_bit()	(MIPS1_PG_RO)
 #define	mips_pg_ropage_bit()	(MIPS1_PG_RO)	/* XXX not MIPS1_PG_ROPAGE? */
 #define	mips_pg_rwpage_bit()	(MIPS1_PG_RWPAGE)
 #define	mips_pg_rwncpage_bit()	(MIPS1_PG_RWNCPAGE)
 @@ -110,63 +110,61 @@ int pmap_is_page_ro(struct pmap *pmap, v
 #define	mips_pg_global_bit()	(MIPS3_PG_G)
 #define	mips_pg_wired_bit()	(MIPS3_PG_WIRED)
 #define	PTE_TO_PADDR(pte)	MIPS3_PTE_TO_PADDR((pte))
 #define	PAGE_IS_RDONLY(pte, va)	MIPS3_PAGE_IS_RDONLY((pte), (va))
 #define	mips_tlbpfn_to_paddr(x)		mips3_tlbpfn_to_paddr((vaddr_t)(x))
 #define	mips_paddr_to_tlbpfn(x)		mips3_paddr_to_tlbpfn((x))
 #endif /* mips3 */
 /* MIPS1 and MIPS3 (or greater) */
 #if defined(MIPS1) && defined(MIPS3_PLUS)
-static __inline int
+static __inline bool
-    mips_pg_v(unsigned int entry),
+    mips_pg_v(uint32_t entry),
-    mips_pg_wired(unsigned int entry),
+    mips_pg_wired(uint32_t entry),
-    PAGE_IS_RDONLY(unsigned int pte, vaddr_t va);
+    PAGE_IS_RDONLY(uint32_t pte, vaddr_t va);
-static __inline unsigned int
+static __inline uint32_t
     mips_pg_wired_bit(void), mips_pg_m_bit(void),
     mips_pg_ro_bit(void), mips_pg_rw_bit(void),
     mips_pg_ropage_bit(void),
     mips_pg_cwpage_bit(void),
     mips_pg_rwpage_bit(void),
     mips_pg_global_bit(void);
-static __inline paddr_t PTE_TO_PADDR(unsigned int pte);
+static __inline paddr_t PTE_TO_PADDR(uint32_t pte);
-static __inline paddr_t mips_tlbpfn_to_paddr(unsigned int pfn);
+static __inline paddr_t mips_tlbpfn_to_paddr(uint32_t pfn);
-static __inline unsigned int mips_paddr_to_tlbpfn(paddr_t pa);
+static __inline uint32_t mips_paddr_to_tlbpfn(paddr_t pa);
-static __inline int
+static __inline bool
-mips_pg_v(entry)
+mips_pg_v(uint32_t entry)
 	unsigned int entry;
+{
 	if (MIPS_HAS_R4K_MMU)
-		return (entry & MIPS3_PG_V);
+		return (entry & MIPS3_PG_V) != 0;
-	return (entry & MIPS1_PG_V);
+	return (entry & MIPS1_PG_V) != 0;
+}
-static __inline int
+static __inline bool
-mips_pg_wired(entry)
+mips_pg_wired(uint32_t entry)
 	unsigned int entry;
+{
 	if (MIPS_HAS_R4K_MMU)
-		return (entry & MIPS3_PG_WIRED);
+		return (entry & MIPS3_PG_WIRED) != 0;
-	return (entry & MIPS1_PG_WIRED);
+	return (entry & MIPS1_PG_WIRED) != 0;
+}
-static __inline unsigned int
+static __inline uint32_t
 mips_pg_m_bit(void)
+{
 	if (MIPS_HAS_R4K_MMU)
 		return (MIPS3_PG_D);
 	return (MIPS1_PG_D);
+}
 static __inline unsigned int
 mips_pg_ro_bit(void)
+{
 	if (MIPS_HAS_R4K_MMU)
 		return (MIPS3_PG_RO);
 	return (MIPS1_PG_RO);
 @@ -212,56 +210,51 @@ mips_pg_global_bit(void)
 		return (MIPS3_PG_G);
 	return (MIPS1_PG_G);
+}
 static __inline unsigned int
 mips_pg_wired_bit(void)
+{
 	if (MIPS_HAS_R4K_MMU)
 		return (MIPS3_PG_WIRED);
 	return (MIPS1_PG_WIRED);
+}
 static __inline paddr_t
-PTE_TO_PADDR(pte)
+PTE_TO_PADDR(uint32_t pte)
 	unsigned int pte;
+{
 	if (MIPS_HAS_R4K_MMU)
 		return (MIPS3_PTE_TO_PADDR(pte));
 	return (MIPS1_PTE_TO_PADDR(pte));
+}
-static __inline int
+static __inline bool
-PAGE_IS_RDONLY(pte, va)
+PAGE_IS_RDONLY(uint32_t pte, vaddr_t va)
 	unsigned int pte;
 	vaddr_t va;
+{
 	if (MIPS_HAS_R4K_MMU)
 		return (MIPS3_PAGE_IS_RDONLY(pte, va));
 	return (MIPS1_PAGE_IS_RDONLY(pte, va));
+}
 static __inline paddr_t
-mips_tlbpfn_to_paddr(pfn)
+mips_tlbpfn_to_paddr(uint32_t pfn)
 	unsigned int pfn;
+{
 	if (MIPS_HAS_R4K_MMU)
 		return (mips3_tlbpfn_to_paddr(pfn));
 	return (mips1_tlbpfn_to_paddr(pfn));
+}
-static __inline unsigned int
+static __inline uint32_t
-mips_paddr_to_tlbpfn(pa)
+mips_paddr_to_tlbpfn(paddr_t pa)
 	paddr_t pa;
+{
 	if (MIPS_HAS_R4K_MMU)
 		return (mips3_paddr_to_tlbpfn(pa));
 	return (mips1_paddr_to_tlbpfn(pa));
+}
 #endif
 #endif /* ! _LOCORE */
 #if defined(_KERNEL) && !defined(_LOCORE)
 /*
  * Kernel virtual address to page table entry and visa versa.
  */

 @@ -1,14 +1,14 @@
-/*	$NetBSD: vmparam.h,v 1.41.28.9 2010/01/20 06:58:35 matt Exp $	*/
+/*	$NetBSD: vmparam.h,v 1.41.28.10 2010/01/26 21:19:25 matt Exp $	*/
 /*
  * Copyright (c) 1992, 1993
  *	The Regents of the University of California.  All rights reserved.
+ *
  * This code is derived from software contributed to Berkeley by
  * the Systems Programming Group of the University of Utah Computer
  * Science Department and Ralph Campbell.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
 @@ -199,40 +199,66 @@
 /* virtual sizes (bytes) for various kernel submaps */
 #define VM_PHYS_SIZE		(USRIOSIZE*PAGE_SIZE)
 /* VM_PHYSSEG_MAX defined by platform-dependent code. */
 #define	VM_PHYSSEG_STRAT	VM_PSTRAT_BSEARCH
 #define	VM_PHYSSEG_NOADD	/* can add RAM after vm_mem_init */
 #define	__HAVE_VM_PAGE_MD
 /*
  * pmap-specific data stored in the vm_page structure.
  */
 /*
  * For each struct vm_page, there is a list of all currently valid virtual
  * mappings of that page.  An entry is a pv_entry_t, the list is pv_table.
  * XXX really should do this as a part of the higher level code.
  */
 typedef struct pv_entry {
 	struct pv_entry	*pv_next;	/* next pv_entry */
 	struct pmap	*pv_pmap;	/* pmap where mapping lies */
 	vaddr_t		pv_va;		/* virtual address for mapping */
 } *pv_entry_t;
 #define	PG_MD_UNCACHED		0x0001	/* page is mapped uncached */
 #define	PG_MD_MODIFIED		0x0002	/* page has been modified */
 #define	PG_MD_REFERENCED	0x0004	/* page has been recently referenced */
 #define	PG_MD_POOLPAGE		0x0008	/* page is used as a poolpage */
 #define	PG_MD_EXECPAGE		0x0010	/* page is exec mapped */
 #define	PG_MD_CACHED_P(pg)	(((pg)->mdpage.pvh_attrs & PG_MD_UNCACHED) == 0)
 #define	PG_MD_UNCACHED_P(pg)	(((pg)->mdpage.pvh_attrs & PG_MD_UNCACHED) != 0)
 #define	PG_MD_MODIFIED_P(pg)	(((pg)->mdpage.pvh_attrs & PG_MD_MODIFIED) != 0)
 #define	PG_MD_REFERENCED_P(pg)	(((pg)->mdpage.pvh_attrs & PG_MD_REFERENCED) != 0)
 #define	PG_MD_POOLPAGE_P(pg)	(((pg)->mdpage.pvh_attrs & PG_MD_POOLPAGE) != 0)
 #define	PG_MD_EXECPAGE_P(pg)	(((pg)->mdpage.pvh_attrs & PG_MD_EXECPAGE) != 0)
 struct vm_page_md {
-	struct pv_entry *pvh_list;	/* pv_entry list */
+	struct pv_entry pvh_first;	/* pv_entry first */
 #ifdef MULTIPROCESSOR
 	volatile u_int pvh_attrs;	/* page attributes */
 	__cpu_simple_lock_t pvh_slock;	/* pv list lock */
 #define	VM_MDPAGE_SLOCK_INIT(pg)	\
 	__cpu_simple_lock_clear(&(pg)->mdpage.pvh_slock)
 #else
 	u_int pvh_attrs;		/* page attributes */
 #define	VM_MDPAGE_SLOCK_INIT(pg)	do { } while (/*CONSTCOND*/ 0)
 #endif
 };
 #define VM_MDPAGE_INIT(pg)						\
 do {									\
-	(pg)->mdpage.pvh_list = NULL;					\
+	(pg)->mdpage.pvh_first.pv_next = NULL;				\
 	(pg)->mdpage.pvh_first.pv_pmap = NULL;				\
 	(pg)->mdpage.pvh_first.pv_va = (pg)->phys_addr;			\
 	VM_MDPAGE_SLOCK_INIT(pg);					\
 	(pg)->mdpage.pvh_attrs = 0;					\
 } while (/* CONSTCOND */ 0)
 #ifndef VM_NFREELIST
 #define	VM_NFREELIST		16	/* 16 distinct memory segments */
 #define VM_FREELIST_DEFAULT	0
 #define VM_FREELIST_MAX		1
 #endif
 #endif /* ! _MIPS_VMPARAM_H_ */

 @@ -1,14 +1,14 @@
-/*	$NetBSD: cache.c,v 1.33.96.2 2010/01/20 09:04:34 matt Exp $	*/
+/*	$NetBSD: cache.c,v 1.33.96.3 2010/01/26 21:19:25 matt Exp $	*/
 /*
  * Copyright 2001, 2002 Wasabi Systems, Inc.
  * All rights reserved.
+ *
  * Written by Jason R. Thorpe and Simon Burge for Wasabi Systems, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
 @@ -58,27 +58,27 @@
  *    WARRANTIES, INCLUDING BUT NOT LIMITED TO, ANY IMPLIED WARRANTIES OF
  *    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR
  *    NON-INFRINGEMENT ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM BE LIABLE
  *    FOR ANY DAMAGES WHATSOEVER, AND IN PARTICULAR, BROADCOM SHALL NOT BE
  *    LIABLE FOR DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  *    CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  *    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  *    BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  *    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  *    OR OTHERWISE), EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cache.c,v 1.33.96.2 2010/01/20 09:04:34 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cache.c,v 1.33.96.3 2010/01/26 21:19:25 matt Exp $");
 #include "opt_cputype.h"
 #include "opt_mips_cache.h"
 #include <sys/param.h>
 #include <uvm/uvm_extern.h>
 #include <mips/cache.h>
 #include <mips/locore.h>
 #ifdef MIPS1
 #include <mips/cache_r3k.h>
 @@ -951,31 +951,32 @@ mips_config_cache_modern(void)
 	case MIPSNN_CFG1_DL_RSVD:
 		panic("reserved MIPS32/64 Dcache line size");
 		break;
 	default:
 		if (MIPSNN_GET(CFG1_DS, cfg1) == MIPSNN_CFG1_DS_RSVD)
 			panic("reserved MIPS32/64 Dcache sets per way");
 		mci->mci_pdcache_line_size = MIPSNN_CFG1_DL(cfg1);
 		mci->mci_pdcache_way_size =
 		    mci->mci_pdcache_line_size * MIPSNN_CFG1_DS(cfg1);
 		mci->mci_pdcache_ways = MIPSNN_CFG1_DA(cfg1) + 1;
 		/*
 		 * Compute the total size and "way mask" for the
-		 * primary Icache.
+		 * primary Dcache.
 		 */
 		mci->mci_pdcache_size =
 		    mci->mci_pdcache_way_size * mci->mci_pdcache_ways;
 		mci->mci_pdcache_way_mask = mci->mci_pdcache_way_size - 1;
 		uvmexp.ncolors = atop(mci->mci_pdcache_size) / mci->mci_pdcache_ways;
 		break;
+	}
 	/* figure out Icache params. */
 	switch (MIPSNN_GET(CFG1_IL, cfg1)) {
 	case MIPSNN_CFG1_IL_NONE:
 		mci->mci_picache_line_size = mci->mci_picache_way_size =
 		    mci->mci_picache_ways = 0;
 		break;
 	case MIPSNN_CFG1_IL_RSVD:
 		panic("reserved MIPS32/64 Icache line size");
 		break;
 	default:

 @@ -1,14 +1,14 @@
-/*	$NetBSD: mem.c,v 1.35.38.4 2010/01/20 06:58:36 matt Exp $	*/
+/*	$NetBSD: mem.c,v 1.35.38.5 2010/01/26 21:19:25 matt Exp $	*/
 /*
  * Copyright (c) 1982, 1986, 1990, 1993
  *	The Regents of the University of California.  All rights reserved.
+ *
  * This code is derived from software contributed to Berkeley by
  * the Systems Programming Group of the University of Utah Computer
  * Science Department and Ralph Campbell.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
 @@ -70,27 +70,27 @@
  * SUCH DAMAGE.
+ *
  *	@(#)mem.c	8.3 (Berkeley) 1/12/94
  */
 /*
  * Memory special file
  */
 #include "opt_cputype.h"
 #include "opt_mips_cache.h"
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: mem.c,v 1.35.38.4 2010/01/20 06:58:36 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: mem.c,v 1.35.38.5 2010/01/26 21:19:25 matt Exp $");
 #include <sys/param.h>
 #include <sys/conf.h>
 #include <sys/buf.h>
 #include <sys/systm.h>
 #include <sys/uio.h>
 #include <sys/malloc.h>
 #include <sys/msgbuf.h>
 #include <sys/event.h>
 #include <machine/cpu.h>
 #include <mips/cache.h>
 @@ -145,46 +145,46 @@ mmrw(dev, uio, flags)
 			c = iov->iov_len;
 			/*
 			 * XXX Broken; assumes contiguous physical memory.
 			 */
 			if (v + c > ctob(physmem))
 				return (EFAULT);
 #ifdef _LP64
 			v = MIPS_PHYS_TO_XKPHYS_CACHED(v);
 #else
 			v = MIPS_PHYS_TO_KSEG0(v);
 #endif
 			error = uiomove((void *)v, c, uio);
 #if defined(MIPS3_PLUS)
-			if (mips_cache_info.mci_cache_virtual_alias)
+			if (MIPS_CACHE_VIRTUAL_ALIAS)
 				mips_dcache_wbinv_range(v, c);
 #endif
 			continue;
 		case DEV_KMEM:
 			v = uio->uio_offset;
 			c = min(iov->iov_len, MAXPHYS);
 			if (v < MIPS_KSEG0_START)
 			if (v < MIPS_KSEG0_START)
 				return (EFAULT);
 			if (v > MIPS_PHYS_TO_KSEG0(mips_avail_end +
 					mips_round_page(MSGBUFSIZE) - c) &&
 			    (v < MIPS_KSEG2_START ||
 			    !uvm_kernacc((void *)v, c,
 			    uio->uio_rw == UIO_READ ? B_READ : B_WRITE)))
 				return (EFAULT);
 			error = uiomove((void *)v, c, uio);
 #if defined(MIPS3_PLUS)
-			if (mips_cache_info.mci_cache_virtual_alias)
+			if (MIPS_CACHE_VIRTUAL_ALIAS)
 				mips_dcache_wbinv_range(v, c);
 #endif
 			continue;
 		case DEV_NULL:
 			if (uio->uio_rw == UIO_WRITE)
 				uio->uio_resid = 0;
 			return (0);
 		case DEV_ZERO:
 			if (uio->uio_rw == UIO_WRITE) {
 				c = iov->iov_len;
 				break;