 @@ -1,14 +1,14 @@
-/*	$NetBSD: init_main.c,v 1.436.2.1 2011/11/02 21:53:59 yamt Exp $	*/
+/*	$NetBSD: init_main.c,v 1.436.2.2 2011/11/20 10:52:33 yamt Exp $	*/
 /*-
  * Copyright (c) 2008, 2009 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -87,27 +87,27 @@
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  *	@(#)init_main.c	8.16 (Berkeley) 5/14/95
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.436.2.1 2011/11/02 21:53:59 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: init_main.c,v 1.436.2.2 2011/11/20 10:52:33 yamt Exp $");
 #include "opt_ddb.h"
 #include "opt_ipsec.h"
 #include "opt_modular.h"
 #include "opt_ntp.h"
 #include "opt_pipe.h"
 #include "opt_sa.h"
 #include "opt_syscall_debug.h"
 #include "opt_sysv.h"
 #include "opt_fileassoc.h"
 #include "opt_ktrace.h"
 #include "opt_pax.h"
 #include "opt_compat_netbsd.h"
 @@ -314,30 +314,26 @@ main(void)
 	uvm_init();
 	kcpuset_sysinit();
 	prop_kern_init();
 #if ((NKSYMS > 0) || (NDDB > 0) || (NMODULAR > 0))
 	ksyms_init();
 #endif
 	kprintf_init();
 	percpu_init();
 	/* Initialize lock caches. */
 	mutex_obj_init();
 	rw_obj_init();
 	/* Passive serialization. */
 	pserialize_init();
 	/* Initialize the extent manager. */
 	extent_init();
 	/* Do machine-dependent initialization. */
 	cpu_startup();
 	/* Initialize the sysctl subsystem. */
 	sysctl_init();
 	/* Initialize callouts, part 1. */

 @@ -1,14 +1,14 @@
-/*	$NetBSD: uvm.h,v 1.62.4.2 2011/11/12 02:54:04 yamt Exp $	*/
+/*	$NetBSD: uvm.h,v 1.62.4.3 2011/11/20 10:52:33 yamt Exp $	*/
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -78,26 +78,39 @@ struct uvm_cpu {
 	int page_free_nextcolor;	/* next color to allocate from */
 	int page_idlezero_next;		/* which color to zero next */
 	bool page_idle_zero;		/* TRUE if we should try to zero
 					   pages in the idle loop */
 	int pages[PGFL_NQUEUES];	/* total of pages in page_free */
 	u_int emap_gen;			/* emap generation number */
 	/*
 	 * pagestate
 	 * 	[0] non-anonymous
 	 * 	[1] anonymous (PQ_SWAPBACKED)
 	 */
 	int64_t pagestate[2][UVM_PAGE_NUM_STATUS];
 	int64_t loan_obj;	/* O->K loan */
 	int64_t unloan_obj;	/* O->K unloan */
 	int64_t loanbreak_obj;	/* O->K loan resolved on write */
 	int64_t loanfree_obj;	/* O->K loan resolved on free */
 	int64_t loan_anon;	/* A->K loan */
 	int64_t unloan_anon;	/* A->K unloan */
 	int64_t loanbreak_anon;	/* A->K loan resolved on write */
 	int64_t loanfree_anon;	/* A->K loan resolved on free */
 	int64_t loan_zero;	/* O->K loan (zero) */
 	int64_t unloan_zero;	/* O->K unloan (zero) */
 };
 /*
  * uvm structure (vm global state: collected in one structure for ease
  * of reference...)
  */
 struct uvm {
 	/* vm_page related parameters */
 		/* vm_page queues */
 	struct pgfreelist page_free[VM_NFREELIST]; /* unallocated pages */
 	bool page_init_done;		/* TRUE if uvm_page_init() finished */

 @@ -1,14 +1,14 @@
-/*	$NetBSD: uvm_extern.h,v 1.176.2.3 2011/11/14 14:24:54 yamt Exp $	*/
+/*	$NetBSD: uvm_extern.h,v 1.176.2.4 2011/11/20 10:52:33 yamt Exp $	*/
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -446,32 +446,46 @@ struct uvmexp_sysctl {
 	int64_t	pdanscan;
 	int64_t	pdobscan;
 	int64_t	pdreact;
 	int64_t	pdbusy;
 	int64_t	pdpageouts;
 	int64_t	pdpending;
 	int64_t	pddeact;
 	int64_t	anonpages;
 	int64_t	filepages;
 	int64_t	execpages;
 	int64_t colorhit;
 	int64_t colormiss;
 	int64_t ncolors;
 	int64_t possiblydirtypages;
 	int64_t cleanpages;
 	int64_t dirtypages;
 	int64_t possiblydirtyanonpages;
 	int64_t cleananonpages;
 	int64_t dirtyanonpages;
 	int64_t loan_obj;	/* O->K loan */
 	int64_t unloan_obj;	/* O->K unloan */
 	int64_t loanbreak_obj;	/* O->K loan resolved on write */
 	int64_t loanfree_obj;	/* O->K loan resolved on free */
 	int64_t loan_anon;	/* A->K loan */
 	int64_t unloan_anon;	/* A->K unloan */
 	int64_t loanbreak_anon;	/* A->K loan resolved on write */
 	int64_t loanfree_anon;	/* A->K loan resolved on free */
 	int64_t loan_zero;	/* O->K loan (zero) */
 	int64_t unloan_zero;	/* O->K unloan (zero) */
 };
 #ifdef _KERNEL
 /* we need this before including uvm_page.h on some platforms */
 extern struct uvmexp uvmexp;
 /* MD code needs this without including <uvm/uvm.h> */
 extern bool vm_page_zero_enable;
 #endif
 /*
  * Finally, bring in standard UVM headers.
  */
 #include <sys/vmmeter.h>

 @@ -1,14 +1,14 @@
-/*	$NetBSD: uvm_init.c,v 1.41 2011/04/24 03:56:50 rmind Exp $	*/
+/*	$NetBSD: uvm_init.c,v 1.41.4.1 2011/11/20 10:52:33 yamt Exp $	*/
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -22,36 +22,38 @@
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
  * from: Id: uvm_init.c,v 1.1.2.3 1998/02/06 05:15:27 chs Exp
  */
 /*
  * uvm_init.c: init the vm system.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_init.c,v 1.41 2011/04/24 03:56:50 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_init.c,v 1.41.4.1 2011/11/20 10:52:33 yamt Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/debug.h>
 #include <sys/file.h>
 #include <sys/filedesc.h>
 #include <sys/resourcevar.h>
 #include <sys/kmem.h>
 #include <sys/mman.h>
 #include <sys/mutex.h>
 #include <sys/rwlock.h>
 #include <sys/vnode.h>
 #include <uvm/uvm.h>
 #include <uvm/uvm_pdpolicy.h>
 #include <uvm/uvm_readahead.h>
 /*
  * struct uvm: we store most global vars in this structure to make them
  * easier to spot...
  */
 struct uvm uvm;		/* decl */
 struct uvmexp uvmexp;	/* decl */
 @@ -147,27 +149,35 @@ uvm_init(void)
 	 * Initialize pools.  This must be done before anyone manipulates
 	 * any vm_maps because we use a pool for some map entry structures.
 	 */
 	pool_subsystem_init();
 	/*
 	 * init slab memory allocator kmem(9).
 	 */
 	kmem_init();
 	/*
 	 * Initialize lock caches.
 	 */
 	mutex_obj_init();
 	rw_obj_init();
 	/*
 	 * Initialize the uvm_loan() facility.
 	 * REQUIRE: mutex_obj_init
 	 */
 	uvm_loan_init();
 	/*
 	 * init emap subsystem.
 	 */
 	uvm_emap_sysinit();
 	/*
 	 * the VM system is now up!  now that kmem is up we can resize the
 	 * <obj,off> => <page> hash table for general use and enable paging

 @@ -1,14 +1,14 @@
-/*	$NetBSD: uvm_loan.c,v 1.81.2.3 2011/11/18 00:57:33 yamt Exp $	*/
+/*	$NetBSD: uvm_loan.c,v 1.81.2.4 2011/11/20 10:52:33 yamt Exp $	*/
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -22,27 +22,27 @@
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
  * from: Id: uvm_loan.c,v 1.1.6.4 1998/02/06 05:08:43 chs Exp
  */
 /*
  * uvm_loan.c: page loanout handler
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.81.2.3 2011/11/18 00:57:33 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.81.2.4 2011/11/20 10:52:33 yamt Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/mman.h>
 #include <uvm/uvm.h>
 /*
  * "loaned" pages are pages which are (read-only, copy-on-write) loaned
  * from the VM system to other parts of the kernel.   this allows page
  * copying to be avoided (e.g. you can loan pages from objs/anons to
  * the mbuf system).
 @@ -330,26 +330,27 @@ fail:
+ *
  * => called with map, amap, uobj locked
  * => return value:
  *	-1 = fatal error, everything is unlocked, abort.
  *	 0 = lookup in ufi went stale, everything unlocked, relookup and
  *		try again
  *	 1 = got it, everything still locked
  */
 int
 uvm_loananon(struct uvm_faultinfo *ufi, void ***output, int flags,
     struct vm_anon *anon)
+{
 	struct uvm_cpu *ucpu;
 	struct vm_page *pg;
 	int error;
 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
 	/*
 	 * if we are loaning to "another" anon then it is easy, we just
 	 * bump the reference count on the current anon and return a
 	 * pointer to it (it becomes copy-on-write shared).
 	 */
 	if (flags & UVM_LOAN_TOANON) {
 		KASSERT(mutex_owned(anon->an_lock));
 @@ -418,42 +419,48 @@ uvm_loananon(struct uvm_faultinfo *ufi,
+	}
 	if (pg->loan_count == 0) {
 		pmap_page_protect(pg, VM_PROT_READ);
+	}
 	pg->loan_count++;
 	uvm_pageactivate(pg);
 	mutex_exit(&uvm_pageqlock);
 	**output = pg;
 	(*output)++;
 	/* unlock and return success */
 	if (pg->uobject)
 		mutex_exit(pg->uobject->vmobjlock);
 	ucpu = uvm_cpu_get();
 	ucpu->loan_anon++;
 	uvm_cpu_put(ucpu);
 	UVMHIST_LOG(loanhist, "->K done", 0,0,0,0);
 	return (1);
+}
 /*
  * uvm_loanpage: loan out pages to kernel (->K)
+ *
  * => pages should be object-owned and the object should be locked.
  * => in the case of error, the object might be unlocked and relocked.
  * => caller should busy the pages beforehand.
  * => pages will be unbusied.
  * => fail with EBUSY if meet a wired page.
  */
 static int
 uvm_loanpage(struct vm_page **pgpp, int npages)
+{
 	struct uvm_cpu *ucpu;
 	int i;
 	int error = 0;
 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
 	for (i = 0; i < npages; i++) {
 		struct vm_page *pg = pgpp[i];
 		KASSERT(pg->uobject != NULL);
 		KASSERT(pg->uobject == pgpp[0]->uobject);
 		KASSERT(!(pg->flags & (PG_RELEASED|PG_PAGEOUT)));
 		KASSERT(mutex_owned(pg->uobject->vmobjlock));
 		KASSERT(pg->flags & PG_BUSY);
 @@ -465,35 +472,40 @@ uvm_loanpage(struct vm_page **pgpp, int
 			error = EBUSY;
 			break;
+		}
 		if (pg->loan_count == 0) {
 			pmap_page_protect(pg, VM_PROT_READ);
+		}
 		pg->loan_count++;
 		uvm_pageactivate(pg);
 		mutex_exit(&uvm_pageqlock);
+	}
 	uvm_page_unbusy(pgpp, npages);
-	if (error) {
+	if (i > 0) {
-		/*
+		ucpu = uvm_cpu_get();
-		 * backout what we've done
+		ucpu->loan_obj += i;
-		 */
+		uvm_cpu_put(ucpu);
-		kmutex_t *slock = pgpp[0]->uobject->vmobjlock;
+		if (error) {
 			/*
 			 * backout what we've done
 			 */
 			kmutex_t *slock = pgpp[0]->uobject->vmobjlock;
-		mutex_exit(slock);
+			mutex_exit(slock);
-		uvm_unloan(pgpp, i, UVM_LOAN_TOPAGE);
+			uvm_unloan(pgpp, i, UVM_LOAN_TOPAGE);
-		mutex_enter(slock);
+			mutex_enter(slock);
+		}
+	}
 	UVMHIST_LOG(loanhist, "done %d", error,0,0,0);
 	return error;
+}
 /*
  * XXX UBC temp limit
  * number of pages to get at once.
  * should be <= MAX_READ_AHEAD in genfs_vnops.c
  */
 #define	UVM_LOAN_GET_CHUNK	16
 @@ -819,27 +831,26 @@ fail:
 /*
  * uvm_loanzero: loan a zero-fill page out
+ *
  * => called with map, amap, uobj locked
  * => return value:
  *	-1 = fatal error, everything is unlocked, abort.
  *	 0 = lookup in ufi went stale, everything unlocked, relookup and
  *		try again
  *	 1 = got it, everything still locked
  */
 static struct uvm_object uvm_loanzero_object;
 static kmutex_t uvm_loanzero_lock;
 static int
 uvm_loanzero(struct uvm_faultinfo *ufi, void ***output, int flags)
+{
 	struct vm_page *pg;
 	struct vm_amap *amap = ufi->entry->aref.ar_amap;
 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(loanhist);
 again:
 	mutex_enter(uvm_loanzero_object.vmobjlock);
 	/*
 	 * first, get ahold of our single zero page.
 @@ -861,32 +872,37 @@ again:
 			goto again;
+		}
 		/* got a zero'd page. */
 		pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
 		pg->flags |= PG_RDONLY;
 		mutex_enter(&uvm_pageqlock);
 		uvm_pageactivate(pg);
 		mutex_exit(&uvm_pageqlock);
 		UVM_PAGE_OWN(pg, NULL);
+	}
 	if ((flags & UVM_LOAN_TOANON) == 0) {	/* loaning to kernel-page */
 		struct uvm_cpu *ucpu;
 		mutex_enter(&uvm_pageqlock);
 		pg->loan_count++;
 		mutex_exit(&uvm_pageqlock);
 		mutex_exit(uvm_loanzero_object.vmobjlock);
 		**output = pg;
 		(*output)++;
 		ucpu = uvm_cpu_get();
 		ucpu->loan_zero++;
 		uvm_cpu_put(ucpu);
 		return (1);
+	}
 #ifdef notdef
 	/*
 	 * loaning to an anon.  check to see if there is already an anon
 	 * associated with this page.  if so, then just return a reference
 	 * to this object.
 	 */
 	if (pg->uanon) {
 		anon = pg->uanon;
 		mutex_enter(&anon->an_lock);
 @@ -955,26 +971,30 @@ uvm_unloananon(struct vm_anon **aloans,
  * uvm_unloanpage: kill loans on pages loaned out to the kernel
+ *
  * => we expect all our resources to be unlocked
  */
 static void
 uvm_unloanpage(struct vm_page **ploans, int npages)
+{
 	struct vm_page *pg;
 	kmutex_t *slock;
 	mutex_enter(&uvm_pageqlock);
 	while (npages-- > 0) {
 		struct uvm_object *obj;
 		struct vm_anon *anon;
 		struct uvm_cpu *ucpu;
 		pg = *ploans++;
 		/*
 		 * do a little dance to acquire the object or anon lock
 		 * as appropriate.  we are locking in the wrong order,
 		 * so we have to do a try-lock here.
 		 */
 		slock = NULL;
 		while (pg->uobject != NULL || pg->uanon != NULL) {
 			if (pg->uobject != NULL) {
 				slock = pg->uobject->vmobjlock;
 			} else {
 @@ -988,42 +1008,55 @@ uvm_unloanpage(struct vm_page **ploans,
+		}
 		/*
 		 * drop our loan.  if page is owned by an anon but
 		 * PQ_ANON is not set, the page was loaned to the anon
 		 * from an object which dropped ownership, so resolve
 		 * this by turning the anon's loan into real ownership
 		 * (ie. decrement loan_count again and set PQ_ANON).
 		 * after all this, if there are no loans left, put the
 		 * page back a paging queue (if the page is owned by
 		 * an anon) or free it (if the page is now unowned).
 		 */
 		obj = pg->uobject;
 		anon = pg->uanon;
 		KASSERT(pg->loan_count > 0);
 		pg->loan_count--;
-		if (pg->uobject == NULL && pg->uanon != NULL &&
+		if (obj == NULL && anon != NULL &&
 		    (pg->pqflags & PQ_ANON) == 0) {
 			KASSERT(pg->loan_count > 0);
 			pg->loan_count--;
 			pg->pqflags |= PQ_ANON;
+		}
-		if (pg->loan_count == 0 && pg->uobject == NULL &&
+		if (pg->loan_count == 0 && obj == NULL && anon == NULL) {
 		    pg->uanon == NULL) {
 			KASSERT((pg->flags & PG_BUSY) == 0);
 			uvm_pagefree(pg);
+		}
 		if (slock != NULL) {
 			mutex_exit(slock);
+		}
 		ucpu = uvm_cpu_get();
 		if (obj != NULL) {
 			KASSERT(anon == NULL); /* XXX no O->A loan */
 			if (obj == &uvm_loanzero_object) {
 				ucpu->unloan_zero++;
 			} else {
 				ucpu->unloan_obj++;
+			}
 		} else if (anon != NULL) {
 			ucpu->unloan_anon++;
+		}
 		uvm_cpu_put(ucpu);
+	}
 	mutex_exit(&uvm_pageqlock);
+}
 /*
  * uvm_unloan: kill loans on pages or anons.
  */
 void
 uvm_unloan(void *v, int npages, int flags)
+{
 	if (flags & UVM_LOAN_TOANON) {
 		uvm_unloananon(v, npages);
 @@ -1077,146 +1110,147 @@ ulz_put(struct uvm_object *uobj, voff_t
 static const struct uvm_pagerops ulz_pager = {
 	.pgo_put = ulz_put,
 };
 /*
  * uvm_loan_init(): initialize the uvm_loan() facility.
  */
 void
 uvm_loan_init(void)
+{
-	mutex_init(&uvm_loanzero_lock, MUTEX_DEFAULT, IPL_NONE);
+	uvm_obj_init(&uvm_loanzero_object, &ulz_pager, true, 0);
 	uvm_obj_init(&uvm_loanzero_object, &ulz_pager, false, 0);
 	uvm_obj_setlock(&uvm_loanzero_object, &uvm_loanzero_lock);
 	UVMHIST_INIT(loanhist, 300);
+}
 /*
  * uvm_loanbreak: break loan on a uobj page
+ *
  * => called with uobj locked
  * => the page should be busy
  * => return value:
  *	newly allocated page if succeeded
  */
 struct vm_page *
 uvm_loanbreak(struct vm_page *uobjpage)
+{
 	struct uvm_cpu *ucpu;
 	struct vm_page *pg;
 #ifdef DIAGNOSTIC
 	struct uvm_object *uobj = uobjpage->uobject;
 #endif
 	const unsigned int count = uobjpage->loan_count;
 	KASSERT(uobj != NULL);
 	KASSERT(mutex_owned(uobj->vmobjlock));
 	KASSERT(uobjpage->flags & PG_BUSY);
 	KASSERT(count > 0);
 	/* alloc new un-owned page */
 	pg = uvm_pagealloc(NULL, 0, NULL, 0);
 	if (pg == NULL)
 		return NULL;
 	/*
 	 * copy the data from the old page to the new
 	 * one and clear the fake flags on the new page (keep it busy).
 	 * force a reload of the old page by clearing it from all
 	 * pmaps.
 	 * transfer dirtiness of the old page to the new page.
 	 * then lock the page queues to rename the pages.
 	 */
 	uvm_pagecopy(uobjpage, pg);	/* old -> new */
 	pg->flags &= ~PG_FAKE;
 	KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_DIRTY);
 	pmap_page_protect(uobjpage, VM_PROT_NONE);
 	if (uvm_pagegetdirty(uobjpage) == UVM_PAGE_STATUS_UNKNOWN &&
 	    !pmap_clear_modify(uobjpage)) {
 		uvm_pagemarkdirty(uobjpage, UVM_PAGE_STATUS_CLEAN);
 	if (uvm_pagegetdirty(uobjpage) == UVM_PAGE_STATUS_CLEAN) {
 		uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_CLEAN);
 	} else {
 		/* uvm_pagecopy marked it dirty */
 		KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_DIRTY);
 		/* a object with a dirty page should be dirty. */
 		KASSERT(!UVM_OBJ_IS_CLEAN(uobj));
 	if (uobjpage->flags & PG_WANTED)
 		wakeup(uobjpage);
 	/* uobj still locked */
 	uobjpage->flags &= ~(PG_WANTED|PG_BUSY);
 	UVM_PAGE_OWN(uobjpage, NULL);
 	mutex_enter(&uvm_pageqlock);
 	/*
 	 * replace uobjpage with new page.
+	 *
 	 * this will update the page dirtiness statistics.
 	 */
 	uvm_pagereplace(uobjpage, pg);
 	mutex_enter(&uvm_pageqlock);
 	KASSERT(uobjpage->uanon == NULL); /* XXX no O->A loan */
 	/*
 	 * if the page is no longer referenced by
 	 * an anon (i.e. we are breaking an O->K
 	 * loan), then remove it from any pageq's.
 	 */
 	if (uobjpage->uanon == NULL)
 		uvm_pagedequeue(uobjpage);
 	/*
 	 * at this point we have absolutely no
 	 * control over uobjpage
 	 */
 	/* install new page */
 	uvm_pageactivate(pg);
 	mutex_exit(&uvm_pageqlock);
 	/*
 	 * done!  loan is broken and "pg" is
 	 * PG_BUSY.   it can now replace uobjpage.
 	 */
 	ucpu = uvm_cpu_get();
 	ucpu->loanbreak_obj += count;
 	uvm_cpu_put(ucpu);
 	return pg;
+}
 int
 uvm_loanbreak_anon(struct vm_anon *anon, struct uvm_object *uobj)
+{
 	struct uvm_cpu *ucpu;
 	struct vm_page *pg;
 	unsigned int oldstatus;
 	const unsigned int count = anon->an_page->loan_count;
 	KASSERT(mutex_owned(anon->an_lock));
 	KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
 	KASSERT(count > 0);
 	/* get new un-owned replacement page */
 	pg = uvm_pagealloc(NULL, 0, NULL, 0);
 	if (pg == NULL) {
 		return ENOMEM;
+	}
 	/* copy old -> new */
 	uvm_pagecopy(anon->an_page, pg);
 	KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_DIRTY);
 	/* force reload */
 	pmap_page_protect(anon->an_page, VM_PROT_NONE);
 	oldstatus = uvm_pagegetdirty(anon->an_page);
 	mutex_enter(&uvm_pageqlock);	  /* KILL loan */
 	anon->an_page->uanon = NULL;
 	/* in case we owned */
 	anon->an_page->pqflags &= ~PQ_ANON;
 	KASSERT(uobj == NULL); /* XXX O->A loan is currently broken */
 	if (uobj) {
 		/* if we were receiver of loan */
 		anon->an_page->loan_count--;
 	} else {
 		/*
 		 * we were the lender (A->K); need to remove the page from
 		 * pageq's.
 		 */
 		uvm_pagedequeue(anon->an_page);
+	}
 	if (uobj) {
 		mutex_exit(uobj->vmobjlock);
 @@ -1224,16 +1258,22 @@ uvm_loanbreak_anon(struct vm_anon *anon,
 	/* install new page in anon */
 	anon->an_page = pg;
 	pg->uanon = anon;
 	pg->pqflags |= PQ_ANON;
 	uvm_pageactivate(pg);
 	mutex_exit(&uvm_pageqlock);
 	pg->flags &= ~(PG_BUSY|PG_FAKE);
 	UVM_PAGE_OWN(pg, NULL);
 	/* done! */
 	if (uobj == NULL) {
 		ucpu = uvm_cpu_get();
 		ucpu->loanbreak_anon += count;
 		ucpu->pagestate[1][oldstatus]--;
 		ucpu->pagestate[1][UVM_PAGE_STATUS_DIRTY]++;
 		uvm_cpu_put(ucpu);
+	}
 	return 0;
+}

 @@ -1,14 +1,14 @@
-/*	$NetBSD: uvm_meter.c,v 1.56.4.4 2011/11/14 14:24:54 yamt Exp $	*/
+/*	$NetBSD: uvm_meter.c,v 1.56.4.5 2011/11/20 10:52:34 yamt Exp $	*/
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
  * Copyright (c) 1982, 1986, 1989, 1993
  *      The Regents of the University of California.
+ *
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -26,27 +26,27 @@
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  *      @(#)vm_meter.c  8.4 (Berkeley) 1/4/94
  * from: Id: uvm_meter.c,v 1.1.2.1 1997/08/14 19:10:35 chuck Exp
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_meter.c,v 1.56.4.4 2011/11/14 14:24:54 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_meter.c,v 1.56.4.5 2011/11/20 10:52:34 yamt Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/cpu.h>
 #include <sys/proc.h>
 #include <sys/kernel.h>
 #include <sys/sysctl.h>
 #include <uvm/uvm.h>
 #include <uvm/uvm_pdpolicy.h>
 /*
  * maxslp: ???? XXXCDC
 @@ -166,37 +166,54 @@ sysctl_vm_uvmexp2(SYSCTLFN_ARGS)
 	u.pdobscan = uvmexp.pdobscan;
 	u.pdreact = uvmexp.pdreact;
 	u.pdbusy = uvmexp.pdbusy;
 	u.pdpageouts = uvmexp.pdpageouts;
 	u.pdpending = uvmexp.pdpending;
 	u.pddeact = uvmexp.pddeact;
 	u.anonpages = uvmexp.anonpages;
 	u.filepages = uvmexp.filepages;
 	u.execpages = uvmexp.execpages;
 	u.colorhit = uvmexp.colorhit;
 	u.colormiss = uvmexp.colormiss;
 	u.cpuhit = uvmexp.cpuhit;
 	u.cpumiss = uvmexp.cpumiss;
 	/*
 	 * XXX should use xcall
 	 * XXX should be an array
 	 */
 	for (CPU_INFO_FOREACH(cii, ci)) {
 		struct uvm_cpu *ucpu = ci->ci_data.cpu_uvm;
 		u.possiblydirtypages +=
 		    ucpu->pagestate[0][UVM_PAGE_STATUS_UNKNOWN];
 		u.cleanpages += ucpu->pagestate[0][UVM_PAGE_STATUS_CLEAN];
 		u.dirtypages += ucpu->pagestate[0][UVM_PAGE_STATUS_DIRTY];
 		u.possiblydirtyanonpages +=
 		    ucpu->pagestate[1][UVM_PAGE_STATUS_UNKNOWN];
 		u.cleananonpages += ucpu->pagestate[1][UVM_PAGE_STATUS_CLEAN];
 		u.dirtyanonpages += ucpu->pagestate[1][UVM_PAGE_STATUS_DIRTY];
 		u.loan_obj += ucpu->loan_obj;
 		u.unloan_obj += ucpu->unloan_obj;
 		u.loanbreak_obj += ucpu->loanbreak_obj;
 		u.loanfree_obj += ucpu->loanfree_obj;
 		u.loan_anon += ucpu->loan_anon;
 		u.unloan_anon += ucpu->unloan_anon;
 		u.loanbreak_anon += ucpu->loanbreak_anon;
 		u.loanfree_anon += ucpu->loanfree_anon;
 		u.loan_zero += ucpu->loan_zero;
 		u.unloan_zero += ucpu->unloan_zero;
+	}
 	node = *rnode;
 	node.sysctl_data = &u;
 	node.sysctl_size = sizeof(u);
 	if (oldp)
 		node.sysctl_size = min(*oldlenp, node.sysctl_size);
 	return (sysctl_lookup(SYSCTLFN_CALL(&node)));
+}
 /*
  * sysctl helper routine for uvm_pctparam.
  */
 static int

 @@ -1,14 +1,14 @@
-/*	$NetBSD: uvm_page.c,v 1.178.2.6 2011/11/18 00:57:33 yamt Exp $	*/
+/*	$NetBSD: uvm_page.c,v 1.178.2.7 2011/11/20 10:52:34 yamt Exp $	*/
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
  * Copyright (c) 1991, 1993, The Regents of the University of California.
+ *
  * All rights reserved.
+ *
  * This code is derived from software contributed to Berkeley by
  * The Mach Operating System project at Carnegie-Mellon University.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
 @@ -56,27 +56,27 @@
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890
+ *
  * any improvements or extensions that they make and grant Carnegie the
  * rights to redistribute these changes.
  */
 /*
  * uvm_page.c: page ops.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.178.2.6 2011/11/18 00:57:33 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.178.2.7 2011/11/20 10:52:34 yamt Exp $");
 #include "opt_ddb.h"
 #include "opt_uvmhist.h"
 #include "opt_readahead.h"
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/malloc.h>
 #include <sys/sched.h>
 #include <sys/kernel.h>
 #include <sys/vnode.h>
 #include <sys/proc.h>
 #include <sys/radixtree.h>
 @@ -156,45 +156,41 @@ static void uvm_pageremove(struct uvm_ob
 /*
  * inline functions
  */
 /*
  * uvm_pageinsert: insert a page in the object.
+ *
  * => caller must lock object
  * => call should have already set pg's object and offset pointers
  */
 static inline void
-uvm_pageinsert_list(struct uvm_object *uobj, struct vm_page *pg,
+uvm_pageinsert_list(struct uvm_object *uobj, struct vm_page *pg)
     struct vm_page *where)
+{
 	KASSERT(uobj == pg->uobject);
 	KASSERT(mutex_owned(uobj->vmobjlock));
 	KASSERT((pg->flags & PG_TABLED) == 0);
 	KASSERT(where == NULL || (where->flags & PG_TABLED));
 	KASSERT(where == NULL || (where->uobject == uobj));
 	if ((pg->pqflags & PQ_STAT) != 0) {
 		struct uvm_cpu *ucpu;
 		const unsigned int status = uvm_pagegetdirty(pg);
 		const bool isaobj = (pg->pqflags & PQ_AOBJ) != 0;
-		kpreempt_disable();
+		ucpu = uvm_cpu_get();
 		ucpu = curcpu()->ci_data.cpu_uvm;
 		ucpu->pagestate[isaobj][status]++;
-		kpreempt_enable();
+		uvm_cpu_put(ucpu);
 		if (!isaobj) {
 			KASSERT((pg->pqflags & PQ_FILE) != 0);
 			if (uobj->uo_npages == 0) {
 				struct vnode *vp = (struct vnode *)uobj;
 				vholdl(vp);
+			}
 			if (UVM_OBJ_IS_VTEXT(uobj)) {
 				atomic_inc_uint(&uvmexp.execpages);
 			} else {
 				atomic_inc_uint(&uvmexp.filepages);
+			}
 		} else {
 @@ -225,53 +221,52 @@ uvm_pageinsert_tree(struct uvm_object *u
 static inline int
 uvm_pageinsert(struct uvm_object *uobj, struct vm_page *pg)
+{
 	int error;
 	KDASSERT(uobj != NULL);
 	KASSERT(uobj == pg->uobject);
 	error = uvm_pageinsert_tree(uobj, pg);
 	if (error != 0) {
 		KASSERT(error == ENOMEM);
 		return error;
+	}
-	uvm_pageinsert_list(uobj, pg, NULL);
+	uvm_pageinsert_list(uobj, pg);
 	return 0;
+}
 /*
  * uvm_page_remove: remove page from object.
+ *
  * => caller must lock object
  */
 static inline void
 uvm_pageremove_list(struct uvm_object *uobj, struct vm_page *pg)
+{
 	KASSERT(uobj == pg->uobject);
 	KASSERT(mutex_owned(uobj->vmobjlock));
 	KASSERT(pg->flags & PG_TABLED);
 	if ((pg->pqflags & PQ_STAT) != 0) {
 		struct uvm_cpu *ucpu;
 		const unsigned int status = uvm_pagegetdirty(pg);
 		const bool isaobj = (pg->pqflags & PQ_AOBJ) != 0;
-		kpreempt_disable();
+		ucpu = uvm_cpu_get();
 		ucpu = curcpu()->ci_data.cpu_uvm;
 		ucpu->pagestate[isaobj][status]--;
-		kpreempt_enable();
+		uvm_cpu_put(ucpu);
 		if (!isaobj) {
 			KASSERT((pg->pqflags & PQ_FILE) != 0);
 			if (uobj->uo_npages == 1) {
 				struct vnode *vp = (struct vnode *)uobj;
 				holdrelel(vp);
+			}
 			if (UVM_OBJ_IS_VTEXT(uobj)) {
 				atomic_dec_uint(&uvmexp.execpages);
 			} else {
 				atomic_dec_uint(&uvmexp.filepages);
+			}
 		} else {
 @@ -1319,30 +1314,29 @@ uvm_pagealloc_strat(struct uvm_object *o
 	pg->uobject = obj;
 	pg->uanon = anon;
 	pg->flags = PG_BUSY|PG_CLEAN|PG_FAKE;
 	/*
 	 * clear PQ_FREE before releasing uvm_fpageqlock.
 	 * otherwise we race with uvm_pglistalloc.
 	 */
 	pg->pqflags = 0;
 	mutex_spin_exit(&uvm_fpageqlock);
 	if (anon) {
 		anon->an_page = pg;
 		pg->pqflags = PQ_ANON;
 		atomic_inc_uint(&uvmexp.anonpages);
-		kpreempt_disable();
+		ucpu = uvm_cpu_get();
 		ucpu = curcpu()->ci_data.cpu_uvm;
 		ucpu->pagestate[1][UVM_PAGE_STATUS_CLEAN]++;
-		kpreempt_enable();
+		uvm_cpu_put(ucpu);
 	} else {
 		if (obj) {
 			int error;
 			/*
 			 * set PQ_FILE|PQ_AOBJ before the first uvm_pageinsert.
 			 */
 			if (UVM_OBJ_IS_VNODE(obj)) {
 				pg->pqflags |= PQ_FILE;
 			} else {
 				pg->pqflags |= PQ_AOBJ;
+			}
 			error = uvm_pageinsert(obj, pg);
 @@ -1404,27 +1398,32 @@ uvm_pagereplace(struct vm_page *oldpg, s
 	newpg->offset = oldpg->offset;
 	idx = newpg->offset >> PAGE_SHIFT;
 	pg = radix_tree_replace_node(&uobj->uo_pages, idx, newpg);
 	KASSERT(pg == oldpg);
 	if (((oldpg->flags ^ newpg->flags) & PG_CLEAN) != 0) {
 		if ((newpg->flags & PG_CLEAN) != 0) {
 			radix_tree_clear_tag(&uobj->uo_pages, idx,
 			    UVM_PAGE_DIRTY_TAG);
 		} else {
 			radix_tree_set_tag(&uobj->uo_pages, idx,
 			    UVM_PAGE_DIRTY_TAG);
+		}
+	}
-	uvm_pageinsert_list(uobj, newpg, oldpg);
+	/*
 	 * oldpg->pqflags is stable.  newpg is not reachable by others yet.
 	 */
 	newpg->pqflags =
 	    (newpg->pqflags & ~PQ_STAT) | (oldpg->pqflags & PQ_STAT);
 	uvm_pageinsert_list(uobj, newpg);
 	uvm_pageremove_list(uobj, oldpg);
+}
 /*
  * uvm_pagerealloc: reallocate a page from one object to another
+ *
  * => both objects must be locked
  */
 void
 uvm_pagerealloc(struct vm_page *pg, struct uvm_object *newobj, voff_t newoff)
+{
 	/*
 @@ -1512,87 +1511,94 @@ uvm_pagefree(struct vm_page *pg)
 	KASSERT((pg->flags & PG_PAGEOUT) == 0);
 	KASSERT(!(pg->pqflags & PQ_FREE));
 	KASSERT(mutex_owned(&uvm_pageqlock) || !uvmpdpol_pageisqueued_p(pg));
 	KASSERT(pg->uobject == NULL || mutex_owned(pg->uobject->vmobjlock));
 	KASSERT(pg->uobject != NULL || pg->uanon == NULL ||
 		mutex_owned(pg->uanon->an_lock));
 	/*
 	 * if the page is loaned, resolve the loan instead of freeing.
 	 */
 	if (pg->loan_count) {
 		struct uvm_object * const obj = pg->uobject;
 		KASSERT(pg->wire_count == 0);
 		/*
 		 * if the page is owned by an anon then we just want to
 		 * drop anon ownership.  the kernel will free the page when
 		 * it is done with it.  if the page is owned by an object,
 		 * remove it from the object and mark it dirty for the benefit
 		 * of possible anon owners.
+		 *
 		 * regardless of previous ownership, wakeup any waiters,
 		 * unbusy the page, and we're done.
 		 */
-		if (pg->uobject != NULL) {
+		if (obj != NULL) {
-			uvm_pageremove(pg->uobject, pg);
+			uvm_pageremove(obj, pg);
 			uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
 		} else if (pg->uanon != NULL) {
 			if ((pg->pqflags & PQ_ANON) == 0) {
 				pg->loan_count--;
 			} else {
 				pg->pqflags &= ~PQ_ANON;
 				atomic_dec_uint(&uvmexp.anonpages);
 				status = uvm_pagegetdirty(pg);
-				kpreempt_disable();
+				ucpu = uvm_cpu_get();
 				ucpu = curcpu()->ci_data.cpu_uvm;
 				ucpu->pagestate[1][status]--;
-				kpreempt_enable();
+				uvm_cpu_put(ucpu);
+			}
 			pg->uanon->an_page = NULL;
 			pg->uanon = NULL;
+		}
 		if (pg->flags & PG_WANTED) {
 			wakeup(pg);
+		}
 		pg->flags &= ~(PG_WANTED|PG_BUSY|PG_RELEASED|PG_PAGER1);
 #ifdef UVM_PAGE_TRKOWN
 		pg->owner_tag = NULL;
 #endif
 		if (pg->loan_count) {
 			KASSERT(pg->uobject == NULL);
 			if (pg->uanon == NULL) {
 				uvm_pagedequeue(pg);
+			}
 			ucpu = uvm_cpu_get();
 			if (obj != NULL) {
 				ucpu->loanfree_obj += pg->loan_count;
 			} else {
 				ucpu->loanfree_anon += pg->loan_count;
+			}
 			uvm_cpu_put(ucpu);
 			return;
+		}
+	}
 	/*
 	 * remove page from its object or anon.
 	 */
 	if (pg->uobject != NULL) {
 		uvm_pageremove(pg->uobject, pg);
 	} else if (pg->uanon != NULL) {
 		pg->uanon->an_page = NULL;
 		atomic_dec_uint(&uvmexp.anonpages);
 		status = uvm_pagegetdirty(pg);
-		kpreempt_disable();
+		ucpu = uvm_cpu_get();
 		ucpu = curcpu()->ci_data.cpu_uvm;
 		ucpu->pagestate[1][status]--;
-		kpreempt_enable();
+		uvm_cpu_put(ucpu);
+	}
 	/*
 	 * now remove the page from the queues.
 	 */
 	uvm_pagedequeue(pg);
 	/*
 	 * if the page was wired, unwire it now.
 	 */
 	if (pg->wire_count) {

 @@ -1,14 +1,14 @@
-/*	$NetBSD: uvm_page.h,v 1.73.2.6 2011/11/18 00:57:34 yamt Exp $	*/
+/*	$NetBSD: uvm_page.h,v 1.73.2.7 2011/11/20 10:52:34 yamt Exp $	*/
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
  * Copyright (c) 1991, 1993, The Regents of the University of California.
+ *
  * All rights reserved.
+ *
  * This code is derived from software contributed to Berkeley by
  * The Mach Operating System project at Carnegie-Mellon University.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
 @@ -290,26 +290,30 @@ void uvm_page_unbusy(struct vm_page **,
 struct vm_page *uvm_pagelookup(struct uvm_object *, voff_t);
 void uvm_pageunwire(struct vm_page *);
 void uvm_pagewire(struct vm_page *);
 void uvm_pagezero(struct vm_page *);
 bool uvm_pageismanaged(paddr_t);
 unsigned int uvm_pagegetdirty(struct vm_page *);
 void uvm_pagemarkdirty(struct vm_page *, unsigned int);
 bool uvm_pagecheckdirty(struct vm_page *, bool);
 bool uvm_pagereadonly_p(struct vm_page *);
 bool uvm_page_locked_p(struct vm_page *);
 kmutex_t *uvm_page_getlock(struct vm_page *);
 bool uvm_page_samelock_p(struct vm_page *, struct vm_page *);
 struct uvm_cpu;
 struct uvm_cpu *uvm_cpu_get(void);
 void uvm_cpu_put(struct uvm_cpu *);
 /*
  * page dirtiness status for uvm_pagegetdirty and uvm_pagemarkdirty
+ *
  * UNKNOWN means that we need to consult pmap to know if the page is
  * dirty or not.
  * basically, UVM_PAGE_STATUS_CLEAN implies that the page has no writable
  * mapping.
+ *
  * if you want to renumber these, check __CTASSERTs in
  * uvm_page_status.c first.
  */
 #define	UVM_PAGE_STATUS_UNKNOWN	0
 #define	UVM_PAGE_STATUS_CLEAN	1

 @@ -1,14 +1,14 @@
-/*	$NetBSD: uvm_page_status.c,v 1.1.2.4 2011/11/13 01:18:02 yamt Exp $	*/
+/*	$NetBSD: uvm_page_status.c,v 1.1.2.5 2011/11/20 10:52:35 yamt Exp $	*/
 /*-
  * Copyright (c)2011 YAMAMOTO Takashi,
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -17,27 +17,27 @@
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_page_status.c,v 1.1.2.4 2011/11/13 01:18:02 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_page_status.c,v 1.1.2.5 2011/11/20 10:52:35 yamt Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <uvm/uvm.h>
 /*
  * page dirtiness status tracking
+ *
  * separated from uvm_page.c mainly for rump
  */
 /*
 @@ -63,46 +63,44 @@ uvm_pagegetdirty(struct vm_page *pg)
 	KASSERT((~pg->flags & (PG_CLEAN|PG_DIRTY)) != 0);
 	KASSERT(uvm_page_locked_p(pg));
 	KASSERT(uobj == NULL || ((pg->flags & PG_CLEAN) == 0) ==
 	    radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
 	return pg->flags & (PG_CLEAN|PG_DIRTY);
+}
 static void
 stat_update(bool isanon, unsigned int oldstatus, unsigned int newstatus)
+{
 	struct uvm_cpu *ucpu;
 	KASSERT(oldstatus != newstatus);
-	kpreempt_disable();
+	ucpu = uvm_cpu_get();
 	ucpu = curcpu()->ci_data.cpu_uvm;
 	ucpu->pagestate[isanon][oldstatus]--;
 	ucpu->pagestate[isanon][newstatus]++;
-	kpreempt_enable();
+	uvm_cpu_put(ucpu);
+}
 /*
  * uvm_pagemarkdirty: set the dirtiness status (one of UVM_PAGE_STATUS_ values)
  * of the page.
  */
 void
 uvm_pagemarkdirty(struct vm_page *pg, unsigned int newstatus)
+{
 	struct uvm_object * const uobj = pg->uobject;
 	const uint64_t idx = pg->offset >> PAGE_SHIFT;
 	const unsigned int oldstatus = uvm_pagegetdirty(pg);
 	KASSERT(uobj != NULL || pg->uanon != NULL);
 	KASSERT((~newstatus & (PG_CLEAN|PG_DIRTY)) != 0);
 	KASSERT((newstatus & ~(PG_CLEAN|PG_DIRTY)) == 0);
 	KASSERT(uvm_page_locked_p(pg));
 	KASSERT(uobj == NULL || ((pg->flags & PG_CLEAN) == 0) ==
 	    radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
 	if (oldstatus == newstatus) {
 		return;
+	}
 	/*
 	 * set UVM_PAGE_DIRTY_TAG tag unless known CLEAN so that putpages can
 	 * find possibly-dirty pages quickly.
 	 */
 @@ -171,13 +169,30 @@ uvm_pagecheckdirty(struct vm_page *pg, b
 		if (oldstatus == UVM_PAGE_STATUS_DIRTY) {
 			modified = true;
 			if (newstatus == UVM_PAGE_STATUS_UNKNOWN) {
 				pmap_clear_modify(pg);
+			}
 		} else {
 			modified = pmap_clear_modify(pg);
+		}
 		uvm_pagemarkdirty(pg, newstatus);
+	}
 	return modified;
+}
 struct uvm_cpu *
 uvm_cpu_get(void)
+{
 	kpreempt_disable();
 	return curcpu()->ci_data.cpu_uvm;
+}
 void
 uvm_cpu_put(struct uvm_cpu *ucpu)
+{
 	KASSERT(kpreempt_disabled());
 	KASSERT(curcpu()->ci_data.cpu_uvm == ucpu);
 	kpreempt_enable();
+}