 @@ -1,14 +1,14 @@
-/*	$NetBSD: uvm_page.c,v 1.157 2010/11/06 15:42:43 uebayasi Exp $	*/
+/*	$NetBSD: uvm_page.c,v 1.158 2010/11/11 14:50:54 uebayasi Exp $	*/
 /*
  * Copyright (c) 2010 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -87,27 +87,27 @@
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890
+ *
  * any improvements or extensions that they make and grant Carnegie the
  * rights to redistribute these changes.
  */
 /*
  * uvm_page.c: page ops.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.157 2010/11/06 15:42:43 uebayasi Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.158 2010/11/11 14:50:54 uebayasi Exp $");
 #include "opt_ddb.h"
 #include "opt_uvmhist.h"
 #include "opt_readahead.h"
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/malloc.h>
 #include <sys/sched.h>
 #include <sys/kernel.h>
 #include <sys/vnode.h>
 #include <sys/proc.h>
 #include <sys/atomic.h>
 @@ -117,26 +117,27 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v
 #include <uvm/uvm_ddb.h>
 #include <uvm/uvm_pdpolicy.h>
 /*
  * global vars... XXXCDC: move to uvm. structure.
  */
 /*
  * physical memory config is stored in vm_physmem.
  */
 struct vm_physseg vm_physmem[VM_PHYSSEG_MAX];	/* XXXCDC: uvm.physmem */
 int vm_nphysseg = 0;				/* XXXCDC: uvm.nphysseg */
 #define	vm_nphysmem	vm_nphysseg
 /*
  * Some supported CPUs in a given architecture don't support all
  * of the things necessary to do idle page zero'ing efficiently.
  * We therefore provide a way to enable it from machdep code here.
  */
 bool vm_page_zero_enable = false;
 /*
  * number of pages per-CPU to reserve for the kernel.
  */
 int vm_page_reserve_kernel = 5;
 @@ -357,26 +358,27 @@ uvm_page_init_buckets(struct pgfreelist
 /*
  * uvm_page_init: init the page system.   called from uvm_init().
+ *
  * => we return the range of kernel virtual memory in kvm_startp/kvm_endp
  */
 void
 uvm_page_init(vaddr_t *kvm_startp, vaddr_t *kvm_endp)
+{
 	static struct uvm_cpu boot_cpu;
 	psize_t freepages, pagecount, bucketcount, n;
 	struct pgflbucket *bucketarray, *cpuarray;
 	struct vm_page *pagearray;
 	struct vm_physseg *seg;
 	int lcv;
 	u_int i;
 	paddr_t paddr;
 	KASSERT(ncpu <= 1);
 	CTASSERT(sizeof(pagearray->offset) >= sizeof(struct uvm_cpu *));
 	/*
 	 * init the page queues and page queue locks, except the free
 	 * list; we allocate that later (with the initial vm_page
 	 * structures).
 	 */
 @@ -388,40 +390,42 @@ uvm_page_init(vaddr_t *kvm_startp, vaddr
 	mutex_init(&uvm_fpageqlock, MUTEX_DRIVER, IPL_VM);
 	/*
 	 * allocate vm_page structures.
 	 */
 	/*
 	 * sanity check:
 	 * before calling this function the MD code is expected to register
 	 * some free RAM with the uvm_page_physload() function.   our job
 	 * now is to allocate vm_page structures for this memory.
 	 */
-	if (vm_nphysseg == 0)
+	if (vm_nphysmem == 0)
 		panic("uvm_page_bootstrap: no memory pre-allocated");
 	/*
 	 * first calculate the number of free pages...
+	 *
 	 * note that we use start/end rather than avail_start/avail_end.
 	 * this allows us to allocate extra vm_page structures in case we
 	 * want to return some memory to the pool after booting.
 	 */
 	freepages = 0;
-	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
+	for (lcv = 0 ; lcv < vm_nphysmem ; lcv++) {
-		freepages += (vm_physmem[lcv].end - vm_physmem[lcv].start);
+		seg = VM_PHYSMEM_PTR(lcv);
 		freepages += (seg->end - seg->start);
+	}
 	/*
 	 * Let MD code initialize the number of colors, or default
 	 * to 1 color if MD code doesn't care.
 	 */
 	if (uvmexp.ncolors == 0)
 		uvmexp.ncolors = 1;
 	uvmexp.colormask = uvmexp.ncolors - 1;
 	/*
 	 * we now know we have (PAGE_SIZE * freepages) bytes of memory we can
 	 * use.   for each page of memory we use we need a vm_page structure.
 	 * thus, the total number of pages we can use is the total size of
 @@ -445,47 +449,48 @@ uvm_page_init(vaddr_t *kvm_startp, vaddr
 		uvm.page_free[lcv].pgfl_buckets =
 		    (bucketarray + (lcv * uvmexp.ncolors));
 		uvm_page_init_buckets(&uvm.page_free[lcv]);
 		uvm.cpus[0]->page_free[lcv].pgfl_buckets =
 		    (cpuarray + (lcv * uvmexp.ncolors));
 		uvm_page_init_buckets(&uvm.cpus[0]->page_free[lcv]);
+	}
 	memset(pagearray, 0, pagecount * sizeof(struct vm_page));
 	/*
 	 * init the vm_page structures and put them in the correct place.
 	 */
-	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) {
+	for (lcv = 0 ; lcv < vm_nphysmem ; lcv++) {
-		n = vm_physmem[lcv].end - vm_physmem[lcv].start;
+		seg = VM_PHYSMEM_PTR(lcv);
 		n = seg->end - seg->start;
 		/* set up page array pointers */
-		vm_physmem[lcv].pgs = pagearray;
+		seg->pgs = pagearray;
 		pagearray += n;
 		pagecount -= n;
-		vm_physmem[lcv].lastpg = vm_physmem[lcv].pgs + (n - 1);
+		seg->lastpg = seg->pgs + (n - 1);
 		/* init and free vm_pages (we've already zeroed them) */
-		paddr = ctob(vm_physmem[lcv].start);
+		paddr = ctob(seg->start);
 		for (i = 0 ; i < n ; i++, paddr += PAGE_SIZE) {
-			vm_physmem[lcv].pgs[i].phys_addr = paddr;
+			seg->pgs[i].phys_addr = paddr;
 #ifdef __HAVE_VM_PAGE_MD
-			VM_MDPAGE_INIT(&vm_physmem[lcv].pgs[i]);
+			VM_MDPAGE_INIT(&seg->pgs[i]);
 #endif
-			if (atop(paddr) >= vm_physmem[lcv].avail_start &&
+			if (atop(paddr) >= seg->avail_start &&
-			    atop(paddr) <= vm_physmem[lcv].avail_end) {
+			    atop(paddr) <= seg->avail_end) {
 				uvmexp.npages++;
 				/* add page to free pool */
-				uvm_pagefree(&vm_physmem[lcv].pgs[i]);
+				uvm_pagefree(&seg->pgs[i]);
+			}
+		}
+	}
 	/*
 	 * pass up the values of virtual_space_start and
 	 * virtual_space_end (obtained by uvm_pageboot_alloc) to the upper
 	 * layers of the VM.
 	 */
 	*kvm_startp = round_page(virtual_space_start);
 	*kvm_endp = trunc_page(virtual_space_end);
 #ifdef DEBUG
 @@ -643,102 +648,102 @@ uvm_pageboot_alloc(vsize_t size)
  * => return false if out of memory.
  */
 /* subroutine: try to allocate from memory chunks on the specified freelist */
 static bool uvm_page_physget_freelist(paddr_t *, int);
 static bool
 uvm_page_physget_freelist(paddr_t *paddrp, int freelist)
+{
 	int lcv, x;
 	/* pass 1: try allocating from a matching end */
 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
-	for (lcv = vm_nphysseg - 1 ; lcv >= 0 ; lcv--)
+	for (lcv = vm_nphysmem - 1 ; lcv >= 0 ; lcv--)
 #else
-	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
+	for (lcv = 0 ; lcv < vm_nphysmem ; lcv++)
 #endif
+	{
 		if (uvm.page_init_done == true)
 			panic("uvm_page_physget: called _after_ bootstrap");
 		if (vm_physmem[lcv].free_list != freelist)
 			continue;
 		/* try from front */
 		if (vm_physmem[lcv].avail_start == vm_physmem[lcv].start &&
 		    vm_physmem[lcv].avail_start < vm_physmem[lcv].avail_end) {
 			*paddrp = ctob(vm_physmem[lcv].avail_start);
 			vm_physmem[lcv].avail_start++;
 			vm_physmem[lcv].start++;
 			/* nothing left?   nuke it */
 			if (vm_physmem[lcv].avail_start ==
 			    vm_physmem[lcv].end) {
-				if (vm_nphysseg == 1)
+				if (vm_nphysmem == 1)
 				    panic("uvm_page_physget: out of memory!");
-				vm_nphysseg--;
+				vm_nphysmem--;
-				for (x = lcv ; x < vm_nphysseg ; x++)
+				for (x = lcv ; x < vm_nphysmem ; x++)
 					/* structure copy */
-					vm_physmem[x] = vm_physmem[x+1];
+					VM_PHYSMEM_PTR_SWAP(x, x + 1);
+			}
 			return (true);
+		}
 		/* try from rear */
 		if (vm_physmem[lcv].avail_end == vm_physmem[lcv].end &&
 		    vm_physmem[lcv].avail_start < vm_physmem[lcv].avail_end) {
 			*paddrp = ctob(vm_physmem[lcv].avail_end - 1);
 			vm_physmem[lcv].avail_end--;
 			vm_physmem[lcv].end--;
 			/* nothing left?   nuke it */
 			if (vm_physmem[lcv].avail_end ==
 			    vm_physmem[lcv].start) {
-				if (vm_nphysseg == 1)
+				if (vm_nphysmem == 1)
 				    panic("uvm_page_physget: out of memory!");
-				vm_nphysseg--;
+				vm_nphysmem--;
-				for (x = lcv ; x < vm_nphysseg ; x++)
+				for (x = lcv ; x < vm_nphysmem ; x++)
 					/* structure copy */
-					vm_physmem[x] = vm_physmem[x+1];
+					VM_PHYSMEM_PTR_SWAP(x, x + 1);
+			}
 			return (true);
+		}
+	}
 	/* pass2: forget about matching ends, just allocate something */
 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
-	for (lcv = vm_nphysseg - 1 ; lcv >= 0 ; lcv--)
+	for (lcv = vm_nphysmem - 1 ; lcv >= 0 ; lcv--)
 #else
-	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
+	for (lcv = 0 ; lcv < vm_nphysmem ; lcv++)
 #endif
+	{
 		/* any room in this bank? */
 		if (vm_physmem[lcv].avail_start >= vm_physmem[lcv].avail_end)
 			continue;  /* nope */
 		*paddrp = ctob(vm_physmem[lcv].avail_start);
 		vm_physmem[lcv].avail_start++;
 		/* truncate! */
 		vm_physmem[lcv].start = vm_physmem[lcv].avail_start;
 		/* nothing left?   nuke it */
 		if (vm_physmem[lcv].avail_start == vm_physmem[lcv].end) {
-			if (vm_nphysseg == 1)
+			if (vm_nphysmem == 1)
 				panic("uvm_page_physget: out of memory!");
-			vm_nphysseg--;
+			vm_nphysmem--;
-			for (x = lcv ; x < vm_nphysseg ; x++)
+			for (x = lcv ; x < vm_nphysmem ; x++)
 				/* structure copy */
-				vm_physmem[x] = vm_physmem[x+1];
+				VM_PHYSMEM_PTR_SWAP(x, x + 1);
+		}
 		return (true);
+	}
 	return (false);        /* whoops! */
+}
 bool
 uvm_page_physget(paddr_t *paddrp)
+{
 	int i;
 	/* try in the order of freelist preference */
 @@ -768,107 +773,107 @@ uvm_page_physload(paddr_t start, paddr_t
 	struct vm_physseg *ps;
 	if (uvmexp.pagesize == 0)
 		panic("uvm_page_physload: page size not set!");
 	if (free_list >= VM_NFREELIST || free_list < VM_FREELIST_DEFAULT)
 		panic("uvm_page_physload: bad free list %d", free_list);
 	if (start >= end)
 		panic("uvm_page_physload: start >= end");
 	/*
 	 * do we have room?
 	 */
-	if (vm_nphysseg == VM_PHYSSEG_MAX) {
+	if (vm_nphysmem == VM_PHYSSEG_MAX) {
 		printf("uvm_page_physload: unable to load physical memory "
 		    "segment\n");
 		printf("\t%d segments allocated, ignoring 0x%llx -> 0x%llx\n",
 		    VM_PHYSSEG_MAX, (long long)start, (long long)end);
 		printf("\tincrease VM_PHYSSEG_MAX\n");
 		return;
+	}
 	/*
 	 * check to see if this is a "preload" (i.e. uvm_mem_init hasn't been
 	 * called yet, so malloc is not available).
 	 */
-	for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) {
+	for (lcv = 0 ; lcv < vm_nphysmem ; lcv++) {
 		if (vm_physmem[lcv].pgs)
 			break;
+	}
-	preload = (lcv == vm_nphysseg);
+	preload = (lcv == vm_nphysmem);
 	/*
 	 * if VM is already running, attempt to malloc() vm_page structures
 	 */
 	if (!preload) {
 		panic("uvm_page_physload: tried to add RAM after vm_mem_init");
 	} else {
 		pgs = NULL;
 		npages = 0;
+	}
 	/*
 	 * now insert us in the proper place in vm_physmem[]
 	 */
 #if (VM_PHYSSEG_STRAT == VM_PSTRAT_RANDOM)
 	/* random: put it at the end (easy!) */
-	ps = &vm_physmem[vm_nphysseg];
+	ps = &vm_physmem[vm_nphysmem];
 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH)
+	{
 		int x;
 		/* sort by address for binary search */
-		for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
+		for (lcv = 0 ; lcv < vm_nphysmem ; lcv++)
 			if (start < vm_physmem[lcv].start)
 				break;
 		ps = &vm_physmem[lcv];
 		/* move back other entries, if necessary ... */
-		for (x = vm_nphysseg ; x > lcv ; x--)
+		for (x = vm_nphysmem ; x > lcv ; x--)
 			/* structure copy */
-			vm_physmem[x] = vm_physmem[x - 1];
+			VM_PHYSMEM_PTR_SWAP(x, x - 1);
+	}
 #elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST)
+	{
 		int x;
 		/* sort by largest segment first */
-		for (lcv = 0 ; lcv < vm_nphysseg ; lcv++)
+		for (lcv = 0 ; lcv < vm_nphysmem ; lcv++)
 			if ((end - start) >
 			    (vm_physmem[lcv].end - vm_physmem[lcv].start))
 				break;
 		ps = &vm_physmem[lcv];
 		/* move back other entries, if necessary ... */
-		for (x = vm_nphysseg ; x > lcv ; x--)
+		for (x = vm_nphysmem ; x > lcv ; x--)
 			/* structure copy */
-			vm_physmem[x] = vm_physmem[x - 1];
+			VM_PHYSMEM_PTR_SWAP(x, x - 1);
+	}
 #else
 	panic("uvm_page_physload: unknown physseg strategy selected!");
 #endif
 	ps->start = start;
 	ps->end = end;
 	ps->avail_start = avail_start;
 	ps->avail_end = avail_end;
 	if (preload) {
 		ps->pgs = NULL;
 	} else {
 		ps->pgs = pgs;
 		ps->lastpg = pgs + npages - 1;
+	}
 	ps->free_list = free_list;
-	vm_nphysseg++;
+	vm_nphysmem++;
 	if (!preload) {
 		uvmpdpol_reinit();
+	}
+}
 /*
  * uvm_page_recolor: Recolor the pages if the new bucket count is
  * larger than the old one.
  */
 void
 uvm_page_recolor(int newncolors)
 @@ -1970,27 +1975,27 @@ uvm_page_printit(struct vm_page *pg, boo
  */
 void
 uvm_page_printall(void (*pr)(const char *, ...))
+{
 	unsigned i;
 	struct vm_page *pg;
 	(*pr)("%18s %4s %4s %18s %18s"
 #ifdef UVM_PAGE_TRKOWN
 	    " OWNER"
 #endif
 	    "\n", "PAGE", "FLAG", "PQ", "UOBJECT", "UANON");
-	for (i = 0; i < vm_nphysseg; i++) {
+	for (i = 0; i < vm_nphysmem; i++) {
 		for (pg = vm_physmem[i].pgs; pg <= vm_physmem[i].lastpg; pg++) {
 			(*pr)("%18p %04x %04x %18p %18p",
 			    pg, pg->flags, pg->pqflags, pg->uobject,
 			    pg->uanon);
 #ifdef UVM_PAGE_TRKOWN
 			if (pg->flags & PG_BUSY)
 				(*pr)(" %d [%s]", pg->owner, pg->owner_tag);
 #endif
 			(*pr)("\n");
+		}
+	}
+}