 @@ -1,14 +1,14 @@
-/*	$NetBSD: subr_pool.c,v 1.200 2013/03/11 21:37:54 pooka Exp $	*/
+/*	$NetBSD: subr_pool.c,v 1.201 2014/02/17 20:40:06 para Exp $	*/
 /*-
  * Copyright (c) 1997, 1999, 2000, 2002, 2007, 2008, 2010
  *     The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Paul Kranenburg; by Jason R. Thorpe of the Numerical Aerospace
  * Simulation Facility, NASA Ames Research Center, and by Andrew Doran.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
 @@ -22,27 +22,27 @@
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.200 2013/03/11 21:37:54 pooka Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_pool.c,v 1.201 2014/02/17 20:40:06 para Exp $");
 #include "opt_ddb.h"
 #include "opt_lockdebug.h"
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bitops.h>
 #include <sys/proc.h>
 #include <sys/errno.h>
 #include <sys/kernel.h>
 #include <sys/vmem.h>
 #include <sys/pool.h>
 #include <sys/syslog.h>
 @@ -551,29 +551,30 @@ pool_init(struct pool *pp, size_t size,
 	 * a returned item with its header based on the page address.
 	 * We use 1/16 of the page size and about 8 times of the item
 	 * size as the threshold (XXX: tune)
+	 *
 	 * However, we'll put the header into the page if we can put
 	 * it without wasting any items.
+	 *
 	 * Silently enforce `0 <= ioff < align'.
 	 */
 	pp->pr_itemoffset = ioff %= align;
 	/* See the comment below about reserved bytes. */
 	trysize = palloc->pa_pagesz - ((align - ioff) % align);
 	phsize = ALIGN(sizeof(struct pool_item_header));
-	if ((pp->pr_roflags & (PR_NOTOUCH | PR_NOALIGN)) == 0 &&
+	if (pp->pr_roflags & PR_PHINPAGE ||
 	    ((pp->pr_roflags & (PR_NOTOUCH | PR_NOALIGN)) == 0 &&
 	    (pp->pr_size < MIN(palloc->pa_pagesz / 16, phsize << 3) ||
-	    trysize / pp->pr_size == (trysize - phsize) / pp->pr_size)) {
+	    trysize / pp->pr_size == (trysize - phsize) / pp->pr_size))) {
 		/* Use the end of the page for the page header */
 		pp->pr_roflags |= PR_PHINPAGE;
 		pp->pr_phoffset = off = palloc->pa_pagesz - phsize;
 	} else {
 		/* The page header will be taken from our page header pool */
 		pp->pr_phoffset = 0;
 		off = palloc->pa_pagesz;
 		SPLAY_INIT(&pp->pr_phtree);
+	}
 	/*
 	 * Alignment is to take place at `ioff' within the item. This means
 	 * we must reserve up to `align - 1' bytes on the page to allow

 @@ -1,14 +1,14 @@
-/*	$NetBSD: subr_vmem.c,v 1.87 2013/11/22 21:04:11 christos Exp $	*/
+/*	$NetBSD: subr_vmem.c,v 1.88 2014/02/17 20:40:06 para Exp $	*/
 /*-
  * Copyright (c)2006,2007,2008,2009 YAMAMOTO Takashi,
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -21,30 +21,42 @@
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 /*
  * reference:
  * -	Magazines and Vmem: Extending the Slab Allocator
  *	to Many CPUs and Arbitrary Resources
  *	http://www.usenix.org/event/usenix01/bonwick.html
+ *
  * locking & the boundary tag pool:
  * - 	A pool(9) is used for vmem boundary tags
  * - 	During a pool get call the global vmem_btag_refill_lock is taken,
  *	to serialize access to the allocation reserve, but no other
  *	vmem arena locks.
  * -	During pool_put calls no vmem mutexes are locked.
  * - 	pool_drain doesn't hold the pool's mutex while releasing memory to
  * 	its backing therefore no interferance with any vmem mutexes.
  * -	The boundary tag pool is forced to put page headers into pool pages
  *  	(PR_PHINPAGE) and not off page to avoid pool recursion.
  *  	(due to sizeof(bt_t) it should be the case anyway)
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_vmem.c,v 1.87 2013/11/22 21:04:11 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_vmem.c,v 1.88 2014/02/17 20:40:06 para Exp $");
 #if defined(_KERNEL)
 #include "opt_ddb.h"
 #endif /* defined(_KERNEL) */
 #include <sys/param.h>
 #include <sys/hash.h>
 #include <sys/queue.h>
 #include <sys/bitops.h>
 #if defined(_KERNEL)
 #include <sys/systm.h>
 #include <sys/kernel.h>	/* hz */
 @@ -65,34 +77,33 @@ __KERNEL_RCSID(0, "$NetBSD: subr_vmem.c,
 #include <errno.h>
 #include <assert.h>
 #include <stdlib.h>
 #include <string.h>
 #include "../sys/vmem.h"
 #include "../sys/vmem_impl.h"
 #endif /* defined(_KERNEL) */
 #if defined(_KERNEL)
 #include <sys/evcnt.h>
 #define VMEM_EVCNT_DEFINE(name) \
 struct evcnt vmem_evcnt_##name = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, \
-    "vmemev", #name); \
+    "vmem", #name); \
 EVCNT_ATTACH_STATIC(vmem_evcnt_##name);
 #define VMEM_EVCNT_INCR(ev)	vmem_evcnt_##ev.ev_count++
 #define VMEM_EVCNT_DECR(ev)	vmem_evcnt_##ev.ev_count--
-VMEM_EVCNT_DEFINE(bt_pages)
+VMEM_EVCNT_DEFINE(static_bt_count)
-VMEM_EVCNT_DEFINE(bt_count)
+VMEM_EVCNT_DEFINE(static_bt_inuse)
 VMEM_EVCNT_DEFINE(bt_inuse)
 #define	VMEM_CONDVAR_INIT(vm, wchan)	cv_init(&vm->vm_cv, wchan)
 #define	VMEM_CONDVAR_DESTROY(vm)	cv_destroy(&vm->vm_cv)
 #define	VMEM_CONDVAR_WAIT(vm)		cv_wait(&vm->vm_cv, &vm->vm_lock)
 #define	VMEM_CONDVAR_BROADCAST(vm)	cv_broadcast(&vm->vm_cv)
 #else /* defined(_KERNEL) */
 #define VMEM_EVCNT_INCR(ev)	/* nothing */
 #define VMEM_EVCNT_DECR(ev)	/* nothing */
 #define	VMEM_CONDVAR_INIT(vm, wchan)	/* nothing */
 #define	VMEM_CONDVAR_DESTROY(vm)	/* nothing */
 @@ -165,164 +176,182 @@ static LIST_HEAD(, vmem) vmem_list = LIS
  * This reserve is 4 for each arena involved in allocating vmems memory.
  * BT_MAXFREE: don't cache excessive counts of bts in arenas
  */
 #define STATIC_BT_COUNT 200
 #define BT_MINRESERVE 4
 #define BT_MAXFREE 64
 static struct vmem_btag static_bts[STATIC_BT_COUNT];
 static int static_bt_count = STATIC_BT_COUNT;
 static struct vmem kmem_va_meta_arena_store;
 vmem_t *kmem_va_meta_arena;
 static struct vmem kmem_meta_arena_store;
-vmem_t *kmem_meta_arena;
+vmem_t *kmem_meta_arena = NULL;
-static kmutex_t vmem_refill_lock;
+static kmutex_t vmem_btag_refill_lock;
 static kmutex_t vmem_btag_lock;
 static LIST_HEAD(, vmem_btag) vmem_btag_freelist;
 static size_t vmem_btag_freelist_count = 0;
-static size_t vmem_btag_count = STATIC_BT_COUNT;
+static struct pool vmem_btag_pool;
 /* ---- boundary tag */
 #define	BT_PER_PAGE	(PAGE_SIZE / sizeof(bt_t))
 static int bt_refill(vmem_t *vm, vm_flag_t flags);
-static int
+static void *
-bt_refillglobal(vm_flag_t flags)
+pool_page_alloc_vmem_meta(struct pool *pp, int flags)
+{
 	const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP;
 	vmem_addr_t va;
-	bt_t *btp;
+	int ret;
 	bt_t *bt;
 	int i;
 	mutex_enter(&vmem_refill_lock);
-	mutex_enter(&vmem_btag_lock);
+	ret = vmem_alloc(kmem_meta_arena, pp->pr_alloc->pa_pagesz,
-	if (vmem_btag_freelist_count > 0) {
+	    (vflags & ~VM_FITMASK) | VM_INSTANTFIT | VM_POPULATING, &va);
 		mutex_exit(&vmem_btag_lock);
 		mutex_exit(&vmem_refill_lock);
 		return 0;
 	mutex_exit(&vmem_btag_lock);
 	if (vmem_alloc(kmem_meta_arena, PAGE_SIZE,
 	    (flags & ~VM_FITMASK) | VM_INSTANTFIT | VM_POPULATING, &va) != 0) {
 		mutex_exit(&vmem_refill_lock);
 		return ENOMEM;
 	VMEM_EVCNT_INCR(bt_pages);
 	mutex_enter(&vmem_btag_lock);
 	btp = (void *) va;
 	for (i = 0; i < (BT_PER_PAGE); i++) {
 		bt = btp;
 		memset(bt, 0, sizeof(*bt));
 		LIST_INSERT_HEAD(&vmem_btag_freelist, bt,
 		    bt_freelist);
 		vmem_btag_freelist_count++;
 		vmem_btag_count++;
 		VMEM_EVCNT_INCR(bt_count);
 		btp++;
 	mutex_exit(&vmem_btag_lock);
-	bt_refill(kmem_arena, (flags & ~VM_FITMASK)
+	return ret ? NULL : (void *)va;
 	    | VM_INSTANTFIT | VM_POPULATING);
 	bt_refill(kmem_va_meta_arena, (flags & ~VM_FITMASK)
 	    | VM_INSTANTFIT | VM_POPULATING);
 	bt_refill(kmem_meta_arena, (flags & ~VM_FITMASK)
 	    | VM_INSTANTFIT | VM_POPULATING);
-	mutex_exit(&vmem_refill_lock);
+static void
 pool_page_free_vmem_meta(struct pool *pp, void *v)
+{
-	return 0;
+	vmem_free(kmem_meta_arena, (vmem_addr_t)v, pp->pr_alloc->pa_pagesz);
+}
 /* allocator for vmem-pool metadata */
 struct pool_allocator pool_allocator_vmem_meta = {
 	.pa_alloc = pool_page_alloc_vmem_meta,
 	.pa_free = pool_page_free_vmem_meta,
 	.pa_pagesz = 0
 };
 static int
 bt_refill(vmem_t *vm, vm_flag_t flags)
+{
 	bt_t *bt;
-	if (!(flags & VM_POPULATING)) {
+	VMEM_LOCK(vm);
-		bt_refillglobal(flags);
+	if (vm->vm_nfreetags > BT_MINRESERVE) {
 		VMEM_UNLOCK(vm);
 		return 0;
+	}
 	VMEM_LOCK(vm);
 	mutex_enter(&vmem_btag_lock);
 	while (!LIST_EMPTY(&vmem_btag_freelist) &&
 	    vm->vm_nfreetags <= BT_MINRESERVE) {
 		bt = LIST_FIRST(&vmem_btag_freelist);
 		LIST_REMOVE(bt, bt_freelist);
 		LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist);
 		vm->vm_nfreetags++;
 		vmem_btag_freelist_count--;
 		VMEM_EVCNT_INCR(static_bt_inuse);
+	}
 	mutex_exit(&vmem_btag_lock);
-	if (vm->vm_nfreetags == 0) {
+	while (vm->vm_nfreetags <= BT_MINRESERVE) {
 		VMEM_UNLOCK(vm);
-		return ENOMEM;
+		mutex_enter(&vmem_btag_refill_lock);
 		bt = pool_get(&vmem_btag_pool,
 		    (flags & VM_SLEEP) ? PR_WAITOK: PR_NOWAIT);
 		mutex_exit(&vmem_btag_refill_lock);
 		VMEM_LOCK(vm);
 		if (bt == NULL && (flags & VM_SLEEP) == 0)
 			break;
 		LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist);
 		vm->vm_nfreetags++;
+	}
 	VMEM_UNLOCK(vm);
 	if (vm->vm_nfreetags == 0) {
 		return ENOMEM;
+	}
 	if (kmem_meta_arena != NULL) {
 		bt_refill(kmem_arena, (flags & ~VM_FITMASK)
 		    | VM_INSTANTFIT | VM_POPULATING);
 		bt_refill(kmem_va_meta_arena, (flags & ~VM_FITMASK)
 		    | VM_INSTANTFIT | VM_POPULATING);
 		bt_refill(kmem_meta_arena, (flags & ~VM_FITMASK)
 		    | VM_INSTANTFIT | VM_POPULATING);
+	}
 	return 0;
+}
-static inline bt_t *
+static bt_t *
 bt_alloc(vmem_t *vm, vm_flag_t flags)
+{
 	bt_t *bt;
 again:
 	VMEM_LOCK(vm);
-	if (vm->vm_nfreetags <= BT_MINRESERVE &&
+	while (vm->vm_nfreetags <= BT_MINRESERVE && (flags & VM_POPULATING) == 0) {
 	    (flags & VM_POPULATING) == 0) {
 		VMEM_UNLOCK(vm);
 		if (bt_refill(vm, VM_NOSLEEP | VM_INSTANTFIT)) {
 			return NULL;
+		}
-		goto again;
+		VMEM_LOCK(vm);
+	}
 	bt = LIST_FIRST(&vm->vm_freetags);
 	LIST_REMOVE(bt, bt_freelist);
 	vm->vm_nfreetags--;
 	VMEM_UNLOCK(vm);
 	VMEM_EVCNT_INCR(bt_inuse);
 	return bt;
+}
-static inline void
+static void
 bt_free(vmem_t *vm, bt_t *bt)
+{
 	VMEM_LOCK(vm);
 	LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist);
 	vm->vm_nfreetags++;
-	while (vm->vm_nfreetags > BT_MAXFREE) {
+	VMEM_UNLOCK(vm);
 		bt = LIST_FIRST(&vm->vm_freetags);
 static void
 bt_freetrim(vmem_t *vm, int freelimit)
+{
 	bt_t *t;
 	LIST_HEAD(, vmem_btag) tofree;
 	LIST_INIT(&tofree);
 	VMEM_LOCK(vm);
 	while (vm->vm_nfreetags > freelimit) {
 		bt_t *bt = LIST_FIRST(&vm->vm_freetags);
 		LIST_REMOVE(bt, bt_freelist);
 		vm->vm_nfreetags--;
-		mutex_enter(&vmem_btag_lock);
+		if (bt >= static_bts
-		LIST_INSERT_HEAD(&vmem_btag_freelist, bt, bt_freelist);
+		    && bt < static_bts + sizeof(static_bts)) {
-		vmem_btag_freelist_count++;
+			mutex_enter(&vmem_btag_lock);
-		mutex_exit(&vmem_btag_lock);
+			LIST_INSERT_HEAD(&vmem_btag_freelist, bt, bt_freelist);
 			vmem_btag_freelist_count++;
 			mutex_exit(&vmem_btag_lock);
 			VMEM_EVCNT_DECR(static_bt_inuse);
 		} else {
 			LIST_INSERT_HEAD(&tofree, bt, bt_freelist);
+		}
+	}
 	VMEM_UNLOCK(vm);
-	VMEM_EVCNT_DECR(bt_inuse);
+	while (!LIST_EMPTY(&tofree)) {
 		t = LIST_FIRST(&tofree);
 		LIST_REMOVE(t, bt_freelist);
 		pool_put(&vmem_btag_pool, t);
+	}
+}
 #endif	/* defined(_KERNEL) */
 /*
  * freelist[0] ... [1, 1]
  * freelist[1] ... [2, 3]
  * freelist[2] ... [4, 7]
  * freelist[3] ... [8, 15]
  *  :
  * freelist[n] ... [(1 << n), (1 << (n + 1)) - 1]
  *  :
  */
 static struct vmem_freelist *
 @@ -593,51 +622,54 @@ qc_destroy(vmem_t *vm)
+		}
 		pool_cache_destroy(qc->qc_cache);
 		prevqc = qc;
+	}
+}
 #endif
 #if defined(_KERNEL)
 static void
 vmem_bootstrap(void)
+{
 	mutex_init(&vmem_list_lock, MUTEX_DEFAULT, IPL_VM);
 	mutex_init(&vmem_refill_lock, MUTEX_DEFAULT, IPL_VM);
 	mutex_init(&vmem_btag_lock, MUTEX_DEFAULT, IPL_VM);
 	mutex_init(&vmem_btag_refill_lock, MUTEX_DEFAULT, IPL_VM);
 	while (static_bt_count-- > 0) {
 		bt_t *bt = &static_bts[static_bt_count];
 		LIST_INSERT_HEAD(&vmem_btag_freelist, bt, bt_freelist);
-		VMEM_EVCNT_INCR(bt_count);
+		VMEM_EVCNT_INCR(static_bt_count);
 		vmem_btag_freelist_count++;
+	}
 	vmem_bootstrapped = TRUE;
+}
 void
 vmem_subsystem_init(vmem_t *vm)
+{
 	kmem_va_meta_arena = vmem_init(&kmem_va_meta_arena_store, "vmem-va",
 , 0, PAGE_SIZE, vmem_alloc, vmem_free, vm,
 , VM_NOSLEEP | VM_BOOTSTRAP | VM_LARGEIMPORT,
 	    IPL_VM);
 	kmem_meta_arena = vmem_init(&kmem_meta_arena_store, "vmem-meta",
 , 0, PAGE_SIZE,
 	    uvm_km_kmem_alloc, uvm_km_kmem_free, kmem_va_meta_arena,
 , VM_NOSLEEP | VM_BOOTSTRAP, IPL_VM);
 	pool_init(&vmem_btag_pool, sizeof(bt_t), 0, 0, PR_PHINPAGE,
 		    "vmembt", &pool_allocator_vmem_meta, IPL_VM);
+}
 #endif /* defined(_KERNEL) */
 static int
 vmem_add1(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, vm_flag_t flags,
     int spanbttype)
+{
 	bt_t *btspan;
 	bt_t *btfree;
 	KASSERT((flags & (VM_SLEEP|VM_NOSLEEP)) != 0);
 	KASSERT((~flags & (VM_SLEEP|VM_NOSLEEP)) != 0);
 	KASSERT(spanbttype == BT_TYPE_SPAN ||
 @@ -685,37 +717,27 @@ vmem_destroy1(vmem_t *vm)
 			bt_t *bt;
 			while ((bt = LIST_FIRST(&vm->vm_hashlist[i])) != NULL) {
 				KASSERT(bt->bt_type == BT_TYPE_SPAN_STATIC);
 				bt_free(vm, bt);
+			}
+		}
 		if (vm->vm_hashlist != &vm->vm_hash0) {
 			xfree(vm->vm_hashlist,
 			    sizeof(struct vmem_hashlist *) * vm->vm_hashsize);
+		}
+	}
-	while (vm->vm_nfreetags > 0) {
+	bt_freetrim(vm, 0);
 		bt_t *bt = LIST_FIRST(&vm->vm_freetags);
 		LIST_REMOVE(bt, bt_freelist);
 		vm->vm_nfreetags--;
 		mutex_enter(&vmem_btag_lock);
 #if defined (_KERNEL)
 		LIST_INSERT_HEAD(&vmem_btag_freelist, bt, bt_freelist);
 		vmem_btag_freelist_count++;
 #endif /* defined(_KERNEL) */
 		mutex_exit(&vmem_btag_lock);
 	VMEM_CONDVAR_DESTROY(vm);
 	VMEM_LOCK_DESTROY(vm);
 	xfree(vm, sizeof(*vm));
+}
 static int
 vmem_import(vmem_t *vm, vmem_size_t size, vm_flag_t flags)
+{
 	vmem_addr_t addr;
 	int rc;
 	if (vm->vm_importfn == NULL) {
 @@ -1301,26 +1323,28 @@ vmem_xfree(vmem_t *vm, vmem_addr_t addr,
 		VMEM_UNLOCK(vm);
 		(*vm->vm_releasefn)(vm->vm_arg, spanaddr, spansize);
 	} else {
 		bt_insfree(vm, bt);
 		VMEM_CONDVAR_BROADCAST(vm);
 		VMEM_UNLOCK(vm);
+	}
 	while (!LIST_EMPTY(&tofree)) {
 		t = LIST_FIRST(&tofree);
 		LIST_REMOVE(t, bt_freelist);
 		bt_free(vm, t);
+	}
 	bt_freetrim(vm, BT_MAXFREE);
+}
 /*
  * vmem_add:
+ *
  * => caller must ensure appropriate spl,
  *    if the arena can be accessed from interrupt context.
  */
 int
 vmem_add(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, vm_flag_t flags)
+{