 @@ -1,14 +1,14 @@
-/*	$NetBSD: pmap.c,v 1.189 2010/07/06 20:50:34 cegger Exp $	*/
+/*	$NetBSD: pmap.c,v 1.190 2010/10/30 17:44:04 uebayasi Exp $	*/
 /*-
  * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
  * NASA Ames Research Center and by Chris G. Demetriou.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
 @@ -57,27 +57,27 @@
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  *	@(#)pmap.c	8.4 (Berkeley) 1/26/94
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.189 2010/07/06 20:50:34 cegger Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.190 2010/10/30 17:44:04 uebayasi Exp $");
 /*
  *	Manages physical address maps.
+ *
  *	In addition to hardware address maps, this
  *	module is called upon to provide software-use-only
  *	maps which may or may not be stored in the same
  *	form as hardware maps.  These pseudo-maps are
  *	used to store intermediate results from copy
  *	operations to and from address spaces.
+ *
  *	Since the information managed by this module is
  *	also stored by the logical address mapping module,
 @@ -124,26 +124,28 @@ __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.1
 #include <sys/mutex.h>
 #ifdef SYSVSHM
 #include <sys/shm.h>
 #endif
 #include <sys/socketvar.h>	/* XXX: for sock_loan_thresh */
 #include <uvm/uvm.h>
 #include <mips/cache.h>
 #include <mips/cpuregs.h>
 #include <mips/locore.h>
 #include <mips/pte.h>
 #define	VM_PAGE_TO_MD(pg)	(&(pg)->mdpage)
 CTASSERT(MIPS_KSEG0_START < 0);
 CTASSERT((intptr_t)MIPS_PHYS_TO_KSEG0(0x1000) < 0);
 CTASSERT(MIPS_KSEG1_START < 0);
 CTASSERT((intptr_t)MIPS_PHYS_TO_KSEG1(0x1000) < 0);
 CTASSERT(MIPS_KSEG2_START < 0);
 CTASSERT(MIPS_MAX_MEM_ADDR < 0);
 CTASSERT(MIPS_RESERVED_ADDR < 0);
 CTASSERT((uint32_t)MIPS_KSEG0_START == 0x80000000);
 CTASSERT((uint32_t)MIPS_KSEG1_START == 0xa0000000);
 CTASSERT((uint32_t)MIPS_KSEG2_START == 0xc0000000);
 CTASSERT((uint32_t)MIPS_MAX_MEM_ADDR == 0xbe000000);
 CTASSERT((uint32_t)MIPS_RESERVED_ADDR == 0xbfc80000);
 CTASSERT(MIPS_KSEG0_P(MIPS_PHYS_TO_KSEG0(0)));
 @@ -255,39 +257,41 @@ struct pool_allocator pmap_pv_page_alloc
  */
 #if defined(MIPS3_PLUS)	/* XXX mmu XXX */
 void mips_dump_segtab(struct proc *);
 static void mips_flushcache_allpvh(paddr_t);
 /*
  * Flush virtual addresses associated with a given physical address
  */
 static void
 mips_flushcache_allpvh(paddr_t pa)
+{
 	struct vm_page *pg;
 	struct vm_page_md *md;
 	struct pv_entry *pv;
 	pg = PHYS_TO_VM_PAGE(pa);
 	if (pg == NULL) {
 		/* page is unmanaged */
 #ifdef DIAGNOSTIC
 		printf("mips_flushcache_allpvh(): unmanaged pa = %#"PRIxPADDR"\n",
 		    pa);
 #endif
 		return;
+	}
-	pv = pg->mdpage.pvh_list;
+	md = VM_PAGE_TO_MD(pg);
 	pv = md->pvh_list;
 #if defined(MIPS3_NO_PV_UNCACHED)
 	/* No current mapping.  Cache was flushed by pmap_remove_pv() */
 	if (pv->pv_pmap == NULL)
 		return;
 	/* Only one index is allowed at a time */
 	if (mips_cache_indexof(pa) != mips_cache_indexof(pv->pv_va))
 		mips_dcache_wbinv_range_index(pv->pv_va, NBPG);
 #else
 	while (pv) {
 		mips_dcache_wbinv_range_index(pv->pv_va, NBPG);
 		pv = pv->pv_next;
 @@ -512,27 +516,27 @@ pmap_init(void)
 	if (pmapdebug & (PDB_FOLLOW|PDB_INIT))
 		printf("pmap_init()\n");
 #endif
 	/*
 	 * Memory for the pv entry heads has
 	 * already been allocated.  Initialize the physical memory
 	 * segments.
 	 */
 	pv = pv_table;
 	for (bank = 0; bank < vm_nphysseg; bank++) {
 		s = vm_physmem[bank].end - vm_physmem[bank].start;
 		for (i = 0; i < s; i++)
-			vm_physmem[bank].pgs[i].mdpage.pvh_list = pv++;
+			VM_PAGE_TO_MD(&vm_physmem[bank].pgs[i])->pvh_list = pv++;
+	}
 	/*
 	 * Set a low water mark on the pv_entry pool, so that we are
 	 * more likely to have these around even in extreme memory
 	 * starvation.
 	 */
 	pool_setlowat(&pmap_pv_pool, pmap_pv_lowat);
 	/*
 	 * Now it is safe to enable pv entry recording.
 	 */
 	pmap_initialized = true;
 @@ -851,60 +855,61 @@ pmap_remove(pmap_t pmap, vaddr_t sva, va
+			}
+		}
+	}
+}
 /*
  *	pmap_page_protect:
+ *
  *	Lower the permission for all mappings to a given page.
  */
 void
 pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	pv_entry_t pv;
 	vaddr_t va;
 #ifdef DEBUG
 	if ((pmapdebug & (PDB_FOLLOW|PDB_PROTECT)) ||
 	    (prot == VM_PROT_NONE && (pmapdebug & PDB_REMOVE)))
 		printf("pmap_page_protect(%#"PRIxPADDR", %x)\n",
 		    VM_PAGE_TO_PHYS(pg), prot);
 #endif
 	switch (prot) {
 	case VM_PROT_READ|VM_PROT_WRITE:
 	case VM_PROT_ALL:
 		break;
 	/* copy_on_write */
 	case VM_PROT_READ:
 	case VM_PROT_READ|VM_PROT_EXECUTE:
-		pv = pg->mdpage.pvh_list;
+		pv = md->pvh_list;
 		/*
 		 * Loop over all current mappings setting/clearing as appropos.
 		 */
 		if (pv->pv_pmap != NULL) {
 			for (; pv; pv = pv->pv_next) {
 				va = pv->pv_va;
 				pmap_protect(pv->pv_pmap, va, va + PAGE_SIZE,
 				    prot);
 				pmap_update(pv->pv_pmap);
+			}
+		}
 		break;
 	/* remove_all */
 	default:
-		pv = pg->mdpage.pvh_list;
+		pv = md->pvh_list;
 		while (pv->pv_pmap != NULL) {
 			pmap_remove(pv->pv_pmap, pv->pv_va,
 			    pv->pv_va + PAGE_SIZE);
+		}
 		pmap_update(pv->pv_pmap);
+	}
+}
 /*
  *	Set the physical protection on the
  *	specified range of this map as requested.
  */
 void
 @@ -1069,38 +1074,39 @@ pmap_is_page_ro(pmap_t pmap, vaddr_t va,
 	return entry & mips_pg_ro_bit();
+}
 #if defined(MIPS3_PLUS) && !defined(MIPS3_NO_PV_UNCACHED)	/* XXX mmu XXX */
 /*
  *	pmap_page_cache:
+ *
  *	Change all mappings of a managed page to cached/uncached.
  */
 static void
 pmap_page_cache(struct vm_page *pg, int mode)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	pv_entry_t pv;
 	pt_entry_t *pte;
 	unsigned entry;
 	unsigned newmode;
 	unsigned asid, needupdate;
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_ENTER))
 		printf("pmap_page_uncache(%#"PRIxPADDR")\n", VM_PAGE_TO_PHYS(pg));
 #endif
 	newmode = mode & PV_UNCACHED ? MIPS3_PG_UNCACHED : MIPS3_PG_CACHED;
-	pv = pg->mdpage.pvh_list;
+	pv = md->pvh_list;
 	asid = pv->pv_pmap->pm_asid;
 	needupdate = (pv->pv_pmap->pm_asidgen == pmap_asid_generation);
 	while (pv) {
 		pv->pv_flags = (pv->pv_flags & ~PV_UNCACHED) | mode;
 		if (pv->pv_pmap == pmap_kernel()) {
 		/*
 		 * Change entries in kernel pmap.
 		 */
 			pte = kvtopte(pv->pv_va);
 			entry = pte->pt_entry;
 			if (entry & MIPS3_PG_V) {
 				entry = (entry & ~MIPS3_PG_CACHEMODE) | newmode;
 @@ -1182,27 +1188,28 @@ pmap_enter(pmap_t pmap, vaddr_t va, padd
 		pa &= ~PGC_NOCACHE;
 	} else {
 		cached = 1;
 		pa |= PGC_NOCACHE;
+	}
 #endif
 	if (!(prot & VM_PROT_READ))
 		panic("pmap_enter: prot");
 #endif
 	pg = PHYS_TO_VM_PAGE(pa);
 	if (pg) {
-		int *attrs = &pg->mdpage.pvh_attrs;
+		struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 		int *attrs = &md->pvh_attrs;
 		/* Set page referenced/modified status based on flags */
 		if (flags & VM_PROT_WRITE)
 			*attrs |= PGA_MODIFIED | PGA_REFERENCED;
 		else if (flags & VM_PROT_ALL)
 			*attrs |= PGA_REFERENCED;
 		if (!(prot & VM_PROT_WRITE))
 			/*
 			 * If page is not yet referenced, we could emulate this
 			 * by not setting the page valid, and setting the
 			 * referenced status in the TLB fault handler, similar
 			 * to how page modified status is done for UTLBmod
 			 * exceptions.
 @@ -1636,27 +1643,28 @@ pmap_zero_page(paddr_t phys)
 	if (!(phys < MIPS_MAX_MEM_ADDR))
 		printf("pmap_zero_page(%#"PRIxPADDR") nonphys\n", phys);
 #endif
 #ifdef _LP64
 	KASSERT(mips3_xkphys_cached);
 	va = MIPS_PHYS_TO_XKPHYS_CACHED(phys);
 #else
 	va = MIPS_PHYS_TO_KSEG0(phys);
 #endif
 #if defined(MIPS3_PLUS)	/* XXX mmu XXX */
 	pg = PHYS_TO_VM_PAGE(phys);
 	if (mips_cache_virtual_alias) {
-		pv = pg->mdpage.pvh_list;
+		struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 		pv = md->pvh_list;
 		if ((pv->pv_flags & PV_UNCACHED) == 0 &&
 		    mips_cache_indexof(pv->pv_va) != mips_cache_indexof(va))
 			mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE);
+	}
 #endif
 	mips_pagezero((void *)va);
 #if defined(MIPS3_PLUS)	/* XXX mmu XXX */
 	/*
 	 * If we have a virtually-indexed, physically-tagged WB cache,
 	 * and no L2 cache to warn of aliased mappings,	we must force a
 	 * writeback of the destination out of the L1 cache.  If we don't,
 @@ -1738,78 +1746,81 @@ pmap_copy_page(paddr_t src, paddr_t dst)
 		mips_dcache_wbinv_range(dst_va, PAGE_SIZE);
+	}
 #endif	/* MIPS3_PLUS */
+}
 /*
  *	pmap_clear_reference:
+ *
  *	Clear the reference bit on the specified physical page.
  */
 bool
 pmap_clear_reference(struct vm_page *pg)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	int *attrp;
 	bool rv;
 #ifdef DEBUG
 	if (pmapdebug & PDB_FOLLOW)
 		printf("pmap_clear_reference(%#"PRIxPADDR")\n",
 		    VM_PAGE_TO_PHYS(pg));
 #endif
-	attrp = &pg->mdpage.pvh_attrs;
+	attrp = &md->pvh_attrs;
 	rv = *attrp & PGA_REFERENCED;
 	*attrp &= ~PGA_REFERENCED;
 	return rv;
+}
 /*
  *	pmap_is_referenced:
+ *
  *	Return whether or not the specified physical page is referenced
  *	by any physical maps.
  */
 bool
 pmap_is_referenced(struct vm_page *pg)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
-	return pg->mdpage.pvh_attrs & PGA_REFERENCED;
+	return md->pvh_attrs & PGA_REFERENCED;
+}
 /*
  *	Clear the modify bits on the specified physical page.
  */
 bool
 pmap_clear_modify(struct vm_page *pg)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	struct pmap *pmap;
 	struct pv_entry *pv;
 	pt_entry_t *pte;
 	int *attrp;
 	vaddr_t va;
 	unsigned asid;
 	bool rv;
 #ifdef DEBUG
 	if (pmapdebug & PDB_FOLLOW)
 		printf("pmap_clear_modify(%#"PRIxPADDR")\n", VM_PAGE_TO_PHYS(pg));
 #endif
-	attrp = &pg->mdpage.pvh_attrs;
+	attrp = &md->pvh_attrs;
 	rv = *attrp & PGA_MODIFIED;
 	*attrp &= ~PGA_MODIFIED;
 	if (!rv) {
 		return rv;
+	}
-	pv = pg->mdpage.pvh_list;
+	pv = md->pvh_list;
 	if (pv->pv_pmap == NULL) {
 		return true;
+	}
 	/*
 	 * remove write access from any pages that are dirty
 	 * so we can tell if they are written to again later.
 	 * flush the VAC first if there is one.
 	 */
 	for (; pv; pv = pv->pv_next) {
 		pmap = pv->pv_pmap;
 		va = pv->pv_va;
 @@ -1839,42 +1850,45 @@ pmap_clear_modify(struct vm_page *pg)
+	}
 	return true;
+}
 /*
  *	pmap_is_modified:
+ *
  *	Return whether or not the specified physical page is modified
  *	by any physical maps.
  */
 bool
 pmap_is_modified(struct vm_page *pg)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
-	return pg->mdpage.pvh_attrs & PGA_MODIFIED;
+	return md->pvh_attrs & PGA_MODIFIED;
+}
 /*
  *	pmap_set_modified:
+ *
  *	Sets the page modified reference bit for the specified page.
  */
 void
 pmap_set_modified(paddr_t pa)
+{
 	struct vm_page *pg;
 	struct vm_page_md *md;
 	pg = PHYS_TO_VM_PAGE(pa);
-	pg->mdpage.pvh_attrs |= PGA_MODIFIED | PGA_REFERENCED;
+	md = VM_PAGE_TO_MD(pg);
 	md->pvh_attrs |= PGA_MODIFIED | PGA_REFERENCED;
+}
 /******************** misc. functions ********************/
 /*
  * Allocate TLB address space tag (called ASID or TLBPID) and return it.
  * It takes almost as much or more time to search the TLB for a
  * specific ASID and flush those entries as it does to flush the entire TLB.
  * Therefore, when we allocate a new ASID, we just take the next number. When
  * we run out of numbers, we flush the TLB, increment the generation count
  * and start over. ASID zero is reserved for kernel use.
  */
 void
 @@ -1904,29 +1918,30 @@ pmap_asid_alloc(pmap_t pmap)
+	}
 #endif
+}
 /******************** pv_entry management ********************/
 /*
  * Enter the pmap and virtual address into the
  * physical to virtual map table.
  */
 void
 pmap_enter_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, u_int *npte)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	pv_entry_t pv, npv;
-	pv = pg->mdpage.pvh_list;
+	pv = md->pvh_list;
 #ifdef DEBUG
 	if (pmapdebug & PDB_ENTER)
 		printf("pmap_enter: pv %p: was %#"PRIxVADDR"/%p/%p\n",
 		    pv, pv->pv_va, pv->pv_pmap, pv->pv_next);
 #endif
 #if defined(MIPS3_NO_PV_UNCACHED)
 again:
 #endif
 	if (pv->pv_pmap == NULL) {
 		/*
 		 * No entries yet, use header as the first entry
 		 */
 @@ -2056,36 +2071,37 @@ again:
+	}
+}
 /*
  * Remove a physical to virtual address translation.
  * If cache was inhibited on this page, and there are no more cache
  * conflicts, restore caching.
  * Flush the cache if the last page is removed (should always be cached
  * at this point).
  */
 void
 pmap_remove_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	pv_entry_t pv, npv;
 	int last;
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_PVENTRY))
 		printf("pmap_remove_pv(%p, %#"PRIxVADDR", %#"PRIxPADDR")\n", pmap, va,
 		    VM_PAGE_TO_PHYS(pg));
 #endif
-	pv = pg->mdpage.pvh_list;
+	pv = md->pvh_list;
 	/*
 	 * If it is the first entry on the list, it is actually
 	 * in the header and we must copy the following entry up
 	 * to the header.  Otherwise we must search the list for
 	 * the entry.  In either case we free the now unused entry.
 	 */
 	last = 0;
 	if (pmap == pv->pv_pmap && va == pv->pv_va) {
 		npv = pv->pv_next;
 		if (npv) {
 			*pv = *npv;
 @@ -2109,70 +2125,72 @@ pmap_remove_pv(pmap_t pmap, vaddr_t va,
 			pv->pv_next = npv->pv_next;
 			pmap_pv_free(npv);
+		}
+	}
 #ifdef MIPS3_PLUS	/* XXX mmu XXX */
 #if !defined(MIPS3_NO_PV_UNCACHED)
 	if (MIPS_HAS_R4K_MMU && pv->pv_flags & PV_UNCACHED) {
 		/*
 		 * Page is currently uncached, check if alias mapping has been
 		 * removed.  If it was, then reenable caching.
 		 */
-		pv = pg->mdpage.pvh_list;
+		pv = md->pvh_list;
 		for (npv = pv->pv_next; npv; npv = npv->pv_next) {
 			if (mips_cache_indexof(pv->pv_va ^ npv->pv_va))
 				break;
+		}
 		if (npv == NULL)
 			pmap_page_cache(pg, 0);
+	}
 #endif
 	if (MIPS_HAS_R4K_MMU && last != 0)
 		mips_dcache_wbinv_range_index(va, PAGE_SIZE);
 #endif	/* MIPS3_PLUS */
+}
 /*
  * pmap_pv_page_alloc:
+ *
  *	Allocate a page for the pv_entry pool.
  */
 void *
 pmap_pv_page_alloc(struct pool *pp, int flags)
+{
 	struct vm_page *pg;
 	struct vm_page_md *md;
 	paddr_t phys;
 #if defined(MIPS3_PLUS)
 	pv_entry_t pv;
 #endif
 	vaddr_t va;
 	pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
 	if (pg == NULL)
 		return NULL;
 	md = VM_PAGE_TO_MD(pg);
 	phys = VM_PAGE_TO_PHYS(pg);
 #ifdef _LP64
 	KASSERT(mips3_xkphys_cached);
 	va = MIPS_PHYS_TO_XKPHYS_CACHED(phys);
 #else
 	va = MIPS_PHYS_TO_KSEG0(phys);
 #endif
 #if defined(MIPS3_PLUS)
 	if (mips_cache_virtual_alias) {
 		pg = PHYS_TO_VM_PAGE(phys);
-		pv = pg->mdpage.pvh_list;
+		pv = md->pvh_list;
 		if ((pv->pv_flags & PV_UNCACHED) == 0 &&
 		    mips_cache_indexof(pv->pv_va) != mips_cache_indexof(va))
 			mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE);
+	}
 #endif
 	return (void *)va;
+}
 /*
  * pmap_pv_page_free:
+ *
  *	Free a pv_entry pool page.
  */
 @@ -2241,27 +2259,28 @@ mips_pmap_map_poolpage(paddr_t pa)
 #ifdef _LP64
 	KASSERT(mips3_xkphys_cached);
 	va = MIPS_PHYS_TO_XKPHYS_CACHED(pa);
 #else
 	if (pa <= MIPS_PHYS_MASK)
 		va = MIPS_PHYS_TO_KSEG0(pa);
 	else
 		panic("mips_pmap_map_poolpage: "
 		    "pa #%"PRIxPADDR" can not be mapped into KSEG0", pa);
 #endif
 #if defined(MIPS3_PLUS)
 	if (mips_cache_virtual_alias) {
 		pg = PHYS_TO_VM_PAGE(pa);
-		pv = pg->mdpage.pvh_list;
+		struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 		pv = md->pvh_list;
 		if ((pv->pv_flags & PV_UNCACHED) == 0 &&
 		    mips_cache_indexof(pv->pv_va) != mips_cache_indexof(va))
 			mips_dcache_wbinv_range_index(pv->pv_va, PAGE_SIZE);
+	}
 #endif
 	return va;
+}
 paddr_t
 mips_pmap_unmap_poolpage(vaddr_t va)
+{
 	paddr_t pa;