 @@ -1,14 +1,14 @@
-/*	$NetBSD: pmap.c,v 1.265 2010/07/29 10:54:50 hannken Exp $	*/
+/*	$NetBSD: pmap.c,v 1.266 2010/10/30 18:30:55 uebayasi Exp $	*/
 /*
+ *
  * Copyright (C) 1996-1999 Eduardo Horvath.
  * All rights reserved.
+ *
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR  ``AS IS'' AND
 @@ -16,27 +16,27 @@
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR  BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.265 2010/07/29 10:54:50 hannken Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.266 2010/10/30 18:30:55 uebayasi Exp $");
 #undef	NO_VCACHE /* Don't forget the locked TLB in dostart */
 #define	HWREF
 #include "opt_ddb.h"
 #include "opt_multiprocessor.h"
 #include <sys/param.h>
 #include <sys/malloc.h>
 #include <sys/queue.h>
 #include <sys/systm.h>
 #include <sys/msgbuf.h>
 #include <sys/pool.h>
 @@ -63,26 +63,28 @@ __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.2
 #ifdef DDB
 #include <machine/db_machdep.h>
 #include <ddb/db_command.h>
 #include <ddb/db_sym.h>
 #include <ddb/db_variables.h>
 #include <ddb/db_extern.h>
 #include <ddb/db_access.h>
 #include <ddb/db_output.h>
 #else
 #define Debugger()
 #define db_printf	printf
 #endif
 #define	VM_PAGE_TO_MD(pg)	(&(pg)->mdpage)
 #define	MEG		(1<<20) /* 1MB */
 #define	KB		(1<<10)	/* 1KB */
 paddr_t cpu0paddr;		/* contigious phys memory preallocated for cpus */
 /* These routines are in assembly to allow access thru physical mappings */
 extern int64_t pseg_get_real(struct pmap *, vaddr_t);
 extern int pseg_set_real(struct pmap *, vaddr_t, int64_t, paddr_t);
 /*
  * Diatribe on ref/mod counting:
+ *
  * First of all, ref/mod info must be non-volatile.  Hence we need to keep it
 @@ -1437,27 +1439,27 @@ pmap_destroy(struct pmap *pm)
 #else
 	if (pmap_ctx(pm)) {
 		mutex_enter(&curcpu()->ci_ctx_lock);
 		ctx_free(pm, curcpu());
 		mutex_exit(&curcpu()->ci_ctx_lock);
+	}
 #endif
 	/* we could be a little smarter and leave pages zeroed */
 	for (pg = TAILQ_FIRST(&pm->pm_obj.memq); pg != NULL; pg = nextpg) {
 		KASSERT((pg->flags & PG_MARKER) == 0);
 		nextpg = TAILQ_NEXT(pg, listq.queue);
 		TAILQ_REMOVE(&pm->pm_obj.memq, pg, listq.queue);
-		KASSERT(pg->mdpage.mdpg_pvh.pv_pmap == NULL);
+		KASSERT(md->mdpg_pvh.pv_pmap == NULL);
 		dcache_flush_page_cpuset(VM_PAGE_TO_PHYS(pg), pmap_cpus_active);
 		uvm_pagefree(pg);
+	}
 	pmap_free_page((paddr_t)(u_long)pm->pm_segs, pmap_cpus_active);
 	UVM_OBJ_DESTROY(&pm->pm_obj);
 	pool_cache_put(&pmap_cache, pm);
+}
 /*
  * Copy the range specified by src_addr/len
  * from the source map to the range dst_addr/len
  * in the destination map.
+ *
 @@ -1704,27 +1706,29 @@ pmap_enter(struct pmap *pm, vaddr_t va,
 		if (opa != pa) {
 			opg = PHYS_TO_VM_PAGE(opa);
 			if (opg != NULL) {
 				npv = pmap_remove_pv(pm, va, opg);
+			}
+		}
+	}
 	/*
 	 * Construct the TTE.
 	 */
 	pg = PHYS_TO_VM_PAGE(pa);
 	if (pg) {
-		pvh = &pg->mdpage.mdpg_pvh;
+		struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 		pvh = &md->mdpg_pvh;
 		uncached = (pvh->pv_va & (PV_ALIAS|PV_NVC));
 #ifdef DIAGNOSTIC
 		if ((flags & VM_PROT_ALL) & ~prot)
 			panic("pmap_enter: access_type exceeds prot");
 #endif
 		/*
 		 * If we don't have the traphandler do it,
 		 * set the ref/mod bits now.
 		 */
 		if (flags & VM_PROT_ALL)
 			pvh->pv_va |= PV_REF;
 		if (flags & VM_PROT_WRITE)
 			pvh->pv_va |= PV_MOD;
 @@ -2122,28 +2126,30 @@ pmap_protect(struct pmap *pm, vaddr_t sv
 		if ((data & TLB_V) == 0) {
 			continue;
+		}
 		pa = data & TLB_PA_MASK;
 		DPRINTF(PDB_CHANGEPROT|PDB_REF,
 			("pmap_protect: va=%08x data=%08llx "
 			 "seg=%08x pte=%08x\n",
 			 (u_int)sva, (long long)pa, (int)va_to_seg(sva),
 			 (int)va_to_pte(sva)));
 		pg = PHYS_TO_VM_PAGE(pa);
 		if (pg) {
 			struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 			/* Save REF/MOD info */
-			pv = &pg->mdpage.mdpg_pvh;
+			pv = &md->mdpg_pvh;
 			if (data & TLB_ACCESS)
 				pv->pv_va |= PV_REF;
 			if (data & TLB_MODIFY)
 				pv->pv_va |= PV_MOD;
+		}
 		/* Just do the pmap and TSB, not the pv_list */
 		if ((prot & VM_PROT_WRITE) == 0)
 			data &= ~(TLB_W|TLB_REAL_W);
 		if ((prot & VM_PROT_EXECUTE) == 0)
 			data &= ~(TLB_EXEC);
 		rv = pseg_set(pm, sva, data, 0);
 @@ -2462,39 +2468,40 @@ ptelookup_va(vaddr_t va)
 #define TSBBASEMASK	(0xffffffffffffe000LL << tsbsize)
 	tsbptr = (((va >> 9) & 0xfffffffffffffff0LL) & ~TSBBASEMASK);
 	return (tsbptr / sizeof(pte_t));
+}
 /*
  * Do whatever is needed to sync the MOD/REF flags
  */
 bool
 pmap_clear_modify(struct vm_page *pg)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	pv_entry_t pv;
 	int rv;
 	int changed = 0;
 #ifdef DEBUG
 	int modified = 0;
 	DPRINTF(PDB_CHANGEPROT|PDB_REF, ("pmap_clear_modify(%p)\n", pg));
 	modified = pmap_is_modified(pg);
 #endif
 	mutex_enter(&pmap_lock);
 	/* Clear all mappings */
-	pv = &pg->mdpage.mdpg_pvh;
+	pv = &md->mdpg_pvh;
 #ifdef DEBUG
 	if (pv->pv_va & PV_MOD)
 		pv->pv_va |= PV_WE;	/* Remember this was modified */
 #endif
 	if (pv->pv_va & PV_MOD) {
 		changed |= 1;
 		pv->pv_va &= ~PV_MOD;
+	}
 #ifdef DEBUG
 	if (pv->pv_next && !pv->pv_pmap) {
 		printf("pmap_clear_modify: npv but no pmap for pv %p\n", pv);
 		Debugger();
+	}
 @@ -2543,40 +2550,41 @@ pmap_clear_modify(struct vm_page *pg)
 	    (changed ? "was modified" : "was not modified")));
 	if (modified != changed) {
 		printf("pmap_clear_modify: modified %d changed %d\n",
 		       modified, changed);
 		Debugger();
 	} else return (modified);
 #endif
 	return (changed);
+}
 bool
 pmap_clear_reference(struct vm_page *pg)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	pv_entry_t pv;
 	int rv;
 	int changed = 0;
 #ifdef DEBUG
 	int referenced = 0;
 #endif
 	mutex_enter(&pmap_lock);
 #ifdef DEBUG
 	DPRINTF(PDB_CHANGEPROT|PDB_REF, ("pmap_clear_reference(%p)\n", pg));
 	referenced = pmap_is_referenced_locked(pg);
 #endif
 	/* Clear all references */
-	pv = &pg->mdpage.mdpg_pvh;
+	pv = &md->mdpg_pvh;
 	if (pv->pv_va & PV_REF) {
 		changed |= 1;
 		pv->pv_va &= ~PV_REF;
+	}
 #ifdef DEBUG
 	if (pv->pv_next && !pv->pv_pmap) {
 		printf("pmap_clear_reference: npv but no pmap for pv %p\n", pv);
 		Debugger();
+	}
 #endif
 	if (pv->pv_pmap != NULL) {
 		for (; pv; pv = pv->pv_next) {
 			int64_t data;
 @@ -2609,56 +2617,57 @@ pmap_clear_reference(struct vm_page *pg)
 				tsb_invalidate(va, pmap);
 				tlb_flush_pte(va, pmap);
+			}
 			if (pv->pv_va & PV_REF) {
 				changed |= 1;
 				pv->pv_va &= ~PV_REF;
+			}
+		}
+	}
 	dcache_flush_page_all(VM_PAGE_TO_PHYS(pg));
 	pv_check();
 #ifdef DEBUG
 	if (pmap_is_referenced_locked(pg)) {
-		pv = &pg->mdpage.mdpg_pvh;
+		pv = &md->mdpg_pvh;
 		printf("pmap_clear_reference(): %p still referenced "
 			"(pmap = %p, ctx = %d)\n", pg, pv->pv_pmap,
 			pv->pv_pmap ? pmap_ctx(pv->pv_pmap) : 0);
 		Debugger();
+	}
 	DPRINTF(PDB_CHANGEPROT|PDB_REF,
 	    ("pmap_clear_reference: pg %p %s\n", pg,
 	     (changed ? "was referenced" : "was not referenced")));
 	if (referenced != changed) {
 		printf("pmap_clear_reference: referenced %d changed %d\n",
 		       referenced, changed);
 		Debugger();
 	} else {
 		mutex_exit(&pmap_lock);
 		return (referenced);
+	}
 #endif
 	mutex_exit(&pmap_lock);
 	return (changed);
+}
 bool
 pmap_is_modified(struct vm_page *pg)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	pv_entry_t pv, npv;
 	bool res = false;
 	/* Check if any mapping has been modified */
-	pv = &pg->mdpage.mdpg_pvh;
+	pv = &md->mdpg_pvh;
 	if (pv->pv_va & PV_MOD)
 		res = true;
 #ifdef HWREF
 #ifdef DEBUG
 	if (pv->pv_next && !pv->pv_pmap) {
 		printf("pmap_is_modified: npv but no pmap for pv %p\n", pv);
 		Debugger();
+	}
 #endif
 	if (!res && pv->pv_pmap != NULL) {
 		mutex_enter(&pmap_lock);
 		for (npv = pv; !res && npv && npv->pv_pmap;
 		     npv = npv->pv_next) {
 @@ -2688,33 +2697,34 @@ pmap_is_modified(struct vm_page *pg)
 	DPRINTF(PDB_CHANGEPROT|PDB_REF, ("pmap_is_modified(%p) = %d\n", pg,
 	    res));
 	pv_check();
 	return res;
+}
 /*
  * Variant of pmap_is_reference() where caller already holds pmap_lock
  */
 static bool
 pmap_is_referenced_locked(struct vm_page *pg)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	pv_entry_t pv, npv;
 	bool res = false;
 	KASSERT(mutex_owned(&pmap_lock));
 	/* Check if any mapping has been referenced */
-	pv = &pg->mdpage.mdpg_pvh;
+	pv = &md->mdpg_pvh;
 	if (pv->pv_va & PV_REF)
 		return true;
 #ifdef HWREF
 #ifdef DEBUG
 	if (pv->pv_next && !pv->pv_pmap) {
 		printf("pmap_is_referenced: npv but no pmap for pv %p\n", pv);
 		Debugger();
+	}
 #endif
 	if (pv->pv_pmap == NULL)
 		return false;
 @@ -2736,31 +2746,32 @@ pmap_is_referenced_locked(struct vm_page
 	if (res)
 		pv->pv_va |= PV_REF;
 #endif
 	DPRINTF(PDB_CHANGEPROT|PDB_REF,
 		("pmap_is_referenced(%p) = %d\n", pg, res));
 	pv_check();
 	return res;
+}
 bool
 pmap_is_referenced(struct vm_page *pg)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	pv_entry_t pv;
 	bool res = false;
 	/* Check if any mapping has been referenced */
-	pv = &pg->mdpage.mdpg_pvh;
+	pv = &md->mdpg_pvh;
 	if (pv->pv_va & PV_REF)
 		return true;
 #ifdef HWREF
 #ifdef DEBUG
 	if (pv->pv_next && !pv->pv_pmap) {
 		printf("pmap_is_referenced: npv but no pmap for pv %p\n", pv);
 		Debugger();
+	}
 #endif
 	if (pv->pv_pmap != NULL) {
 		mutex_enter(&pmap_lock);
 		res = pmap_is_referenced_locked(pg);
 @@ -2812,39 +2823,40 @@ pmap_unwire(pmap_t pmap, vaddr_t va)
 	pv_check();
 	mutex_exit(&pmap_lock);
+}
 /*
  * Lower the protection on the specified physical page.
+ *
  * Never enable writing as it will break COW
  */
 void
 pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	int64_t clear, set;
 	int64_t data = 0;
 	int rv;
 	pv_entry_t pv, npv, freepv = NULL;
 	struct pmap *pmap;
 	vaddr_t va;
 	bool needflush = FALSE;
 	DPRINTF(PDB_CHANGEPROT,
 	    ("pmap_page_protect: pg %p prot %x\n", pg, prot));
 	mutex_enter(&pmap_lock);
-	pv = &pg->mdpage.mdpg_pvh;
+	pv = &md->mdpg_pvh;
 	if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) {
 		/* copy_on_write */
 		set = TLB_V;
 		clear = TLB_REAL_W|TLB_W;
 		if (VM_PROT_EXECUTE & prot)
 			set |= TLB_EXEC;
 		else
 			clear |= TLB_EXEC;
 		if (VM_PROT_EXECUTE == prot)
 			set |= TLB_EXEC_ONLY;
 #ifdef DEBUG
 @@ -3182,31 +3194,32 @@ ctx_free(struct pmap *pm, struct cpu_inf
+}
 /*
  * Enter the pmap and virtual address into the
  * physical to virtual map table.
+ *
  * We enter here with the pmap locked.
  */
 void
 pmap_enter_pv(struct pmap *pmap, vaddr_t va, paddr_t pa, struct vm_page *pg,
 	      pv_entry_t npv)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	pv_entry_t pvh;
 	KASSERT(mutex_owned(&pmap_lock));
-	pvh = &pg->mdpage.mdpg_pvh;
+	pvh = &md->mdpg_pvh;
 	DPRINTF(PDB_ENTER, ("pmap_enter: pvh %p: was %lx/%p/%p\n",
 	    pvh, pvh->pv_va, pvh->pv_pmap, pvh->pv_next));
 	if (pvh->pv_pmap == NULL) {
 		/*
 		 * No entries yet, use header as the first entry
 		 */
 		DPRINTF(PDB_ENTER, ("pmap_enter: first pv: pmap %p va %lx\n",
 		    pmap, va));
 		ENTER_STAT(firstpv);
 		PV_SETVA(pvh, va);
 		pvh->pv_pmap = pmap;
 		pvh->pv_next = NULL;
 @@ -3243,32 +3256,33 @@ pmap_enter_pv(struct pmap *pmap, vaddr_t
 		if (!npv->pv_next) {
 			ENTER_STAT(secondpv);
+		}
+	}
+}
 /*
  * Remove a physical to virtual address translation.
  */
 pv_entry_t
 pmap_remove_pv(struct pmap *pmap, vaddr_t va, struct vm_page *pg)
+{
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	pv_entry_t pvh, npv, pv;
 	int64_t data = 0;
 	KASSERT(mutex_owned(&pmap_lock));
-	pvh = &pg->mdpage.mdpg_pvh;
+	pvh = &md->mdpg_pvh;
 	DPRINTF(PDB_REMOVE, ("pmap_remove_pv(pm=%p, va=%p, pg=%p)\n", pmap,
 	    (void *)(u_long)va, pg));
 	pv_check();
 	/*
 	 * Remove page from the PV table.
 	 * If it is the first entry on the list, it is actually
 	 * in the header and we must copy the following entry up
 	 * to the header.  Otherwise we must search the list for
 	 * the entry.  In either case we free the now unused entry.
 	 */
 	if (pmap == pvh->pv_pmap && PV_MATCH(pvh, va)) {
 @@ -3319,36 +3333,38 @@ pmap_remove_pv(struct pmap *pmap, vaddr_
 	pv_check();
 	return npv;
+}
 /*
  *	pmap_page_cache:
+ *
  *	Change all mappings of a page to cached/uncached.
  */
 void
 pmap_page_cache(struct pmap *pm, paddr_t pa, int mode)
+{
 	struct vm_page *pg;
 	struct vm_page_md *md;
 	pv_entry_t pv;
 	vaddr_t va;
 	int rv;
 	KASSERT(mutex_owned(&pmap_lock));
 	DPRINTF(PDB_ENTER, ("pmap_page_uncache(%llx)\n",
 	    (unsigned long long)pa));
 	pg = PHYS_TO_VM_PAGE(pa);
-	pv = &pg->mdpage.mdpg_pvh;
+	md = VM_PAGE_TO_MD(pg);
 	pv = &md->mdpg_pvh;
 	while (pv) {
 		va = pv->pv_va & PV_VAMASK;
 		if (pv->pv_va & PV_NC) {
 			int64_t data;
 			/* Non-cached -- I/O mapping */
 			data = pseg_get(pv->pv_pmap, va);
 			KASSERT(data & TLB_V);
 			rv = pseg_set(pv->pv_pmap, va,
 				     data & ~(TLB_CV|TLB_CP), 0);
 			if (rv & 1)
 				panic("pmap_page_cache: pseg_set needs"
 				     " spare! rv=%d\n", rv);
 @@ -3421,39 +3437,41 @@ pmap_free_page_noflush(paddr_t pa)
+{
 	struct vm_page *pg = PHYS_TO_VM_PAGE(pa);
 	uvm_pagefree(pg);
+}
 #ifdef DDB
 void db_dump_pv(db_expr_t, int, db_expr_t, const char *);
 void
 db_dump_pv(db_expr_t addr, int have_addr, db_expr_t count, const char *modif)
+{
 	struct vm_page *pg;
 	struct vm_page_md *md;
 	struct pv_entry *pv;
 	if (!have_addr) {
 		db_printf("Need addr for pv\n");
 		return;
+	}
 	pg = PHYS_TO_VM_PAGE((paddr_t)addr);
 	if (pg == NULL) {
 		db_printf("page is not managed\n");
 		return;
+	}
-	for (pv = &pg->mdpage.mdpg_pvh; pv; pv = pv->pv_next)
+	md = VM_PAGE_TO_MD(pg);
 	for (pv = &md->mdpg_pvh; pv; pv = pv->pv_next)
 		db_printf("pv@%p: next=%p pmap=%p va=0x%llx\n",
 			  pv, pv->pv_next, pv->pv_pmap,
 			  (unsigned long long)pv->pv_va);
+}
 #endif
 #ifdef DEBUG
 /*
  * Test ref/modify handling.  */
 void pmap_testout(void);
 void
 pmap_testout(void)