 @@ -1,14 +1,14 @@
-/*	$NetBSD: subr_vmem.c,v 1.100 2019/12/21 14:50:34 ad Exp $	*/
+/*	$NetBSD: subr_vmem.c,v 1.101 2020/04/19 21:11:42 ad Exp $	*/
 /*-
  * Copyright (c)2006,2007,2008,2009 YAMAMOTO Takashi,
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -36,27 +36,27 @@
  * - 	A pool(9) is used for vmem boundary tags
  * - 	During a pool get call the global vmem_btag_refill_lock is taken,
  *	to serialize access to the allocation reserve, but no other
  *	vmem arena locks.
  * -	During pool_put calls no vmem mutexes are locked.
  * - 	pool_drain doesn't hold the pool's mutex while releasing memory to
  * 	its backing therefore no interferance with any vmem mutexes.
  * -	The boundary tag pool is forced to put page headers into pool pages
  *  	(PR_PHINPAGE) and not off page to avoid pool recursion.
  *  	(due to sizeof(bt_t) it should be the case anyway)
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_vmem.c,v 1.100 2019/12/21 14:50:34 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_vmem.c,v 1.101 2020/04/19 21:11:42 ad Exp $");
 #if defined(_KERNEL) && defined(_KERNEL_OPT)
 #include "opt_ddb.h"
 #endif /* defined(_KERNEL) && defined(_KERNEL_OPT) */
 #include <sys/param.h>
 #include <sys/hash.h>
 #include <sys/queue.h>
 #include <sys/bitops.h>
 #if defined(_KERNEL)
 #include <sys/systm.h>
 #include <sys/kernel.h>	/* hz */
 @@ -195,26 +195,27 @@ static size_t vmem_btag_freelist_count =
 static struct pool vmem_btag_pool;
 static void
 vmem_kick_pdaemon(void)
+{
 #if defined(_KERNEL)
 	uvm_kick_pdaemon();
 #endif
+}
 /* ---- boundary tag */
 static int bt_refill(vmem_t *vm);
 static int bt_refill_locked(vmem_t *vm);
 static void *
 pool_page_alloc_vmem_meta(struct pool *pp, int flags)
+{
 	const vm_flag_t vflags = (flags & PR_WAITOK) ? VM_SLEEP: VM_NOSLEEP;
 	vmem_addr_t va;
 	int ret;
 	ret = vmem_alloc(kmem_meta_arena, pp->pr_alloc->pa_pagesz,
 	    (vflags & ~VM_FITMASK) | VM_INSTANTFIT | VM_POPULATING, &va);
 	return ret ? NULL : (void *)va;
+}
 @@ -224,33 +225,33 @@ pool_page_free_vmem_meta(struct pool *pp
+{
 	vmem_free(kmem_meta_arena, (vmem_addr_t)v, pp->pr_alloc->pa_pagesz);
+}
 /* allocator for vmem-pool metadata */
 struct pool_allocator pool_allocator_vmem_meta = {
 	.pa_alloc = pool_page_alloc_vmem_meta,
 	.pa_free = pool_page_free_vmem_meta,
 	.pa_pagesz = 0
 };
 static int
-bt_refill(vmem_t *vm)
+bt_refill_locked(vmem_t *vm)
+{
 	bt_t *bt;
-	VMEM_LOCK(vm);
+	VMEM_ASSERT_LOCKED(vm);
 	if (vm->vm_nfreetags > BT_MINRESERVE) {
 		VMEM_UNLOCK(vm);
 		return 0;
+	}
 	mutex_enter(&vmem_btag_lock);
 	while (!LIST_EMPTY(&vmem_btag_freelist) &&
 	    vm->vm_nfreetags <= BT_MINRESERVE) {
 		bt = LIST_FIRST(&vmem_btag_freelist);
 		LIST_REMOVE(bt, bt_freelist);
 		LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist);
 		vm->vm_nfreetags++;
 		vmem_btag_freelist_count--;
 		VMEM_EVCNT_INCR(static_bt_inuse);
+	}
 @@ -259,94 +260,104 @@ bt_refill(vmem_t *vm)
 	while (vm->vm_nfreetags <= BT_MINRESERVE) {
 		VMEM_UNLOCK(vm);
 		mutex_enter(&vmem_btag_refill_lock);
 		bt = pool_get(&vmem_btag_pool, PR_NOWAIT);
 		mutex_exit(&vmem_btag_refill_lock);
 		VMEM_LOCK(vm);
 		if (bt == NULL)
 			break;
 		LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist);
 		vm->vm_nfreetags++;
+	}
 	if (vm->vm_nfreetags <= BT_MINRESERVE) {
 		VMEM_UNLOCK(vm);
 		return ENOMEM;
+	}
 	VMEM_UNLOCK(vm);
 	if (kmem_meta_arena != NULL) {
 		VMEM_UNLOCK(vm);
 		(void)bt_refill(kmem_arena);
 		(void)bt_refill(kmem_va_meta_arena);
 		(void)bt_refill(kmem_meta_arena);
 		VMEM_LOCK(vm);
+	}
 	return 0;
+}
 static int
 bt_refill(vmem_t *vm)
+{
 	int rv;
 	VMEM_LOCK(vm);
 	rv = bt_refill_locked(vm);
 	VMEM_UNLOCK(vm);
 	return rv;
+}
 static bt_t *
 bt_alloc(vmem_t *vm, vm_flag_t flags)
+{
 	bt_t *bt;
 	VMEM_LOCK(vm);
 	VMEM_ASSERT_LOCKED(vm);
 	while (vm->vm_nfreetags <= BT_MINRESERVE && (flags & VM_POPULATING) == 0) {
-		VMEM_UNLOCK(vm);
+		if (bt_refill_locked(vm)) {
 		if (bt_refill(vm)) {
 			if ((flags & VM_NOSLEEP) != 0) {
 				return NULL;
+			}
 			/*
 			 * It would be nice to wait for something specific here
 			 * but there are multiple ways that a retry could
 			 * succeed and we can't wait for multiple things
 			 * simultaneously.  So we'll just sleep for an arbitrary
 			 * short period of time and retry regardless.
 			 * This should be a very rare case.
 			 */
 			vmem_kick_pdaemon();
-			kpause("btalloc", false, 1, NULL);
+			kpause("btalloc", false, 1, &vm->vm_lock);
+		}
 		VMEM_LOCK(vm);
+	}
 	bt = LIST_FIRST(&vm->vm_freetags);
 	LIST_REMOVE(bt, bt_freelist);
 	vm->vm_nfreetags--;
 	VMEM_UNLOCK(vm);
 	return bt;
+}
 static void
 bt_free(vmem_t *vm, bt_t *bt)
+{
-	VMEM_LOCK(vm);
+	VMEM_ASSERT_LOCKED(vm);
 	LIST_INSERT_HEAD(&vm->vm_freetags, bt, bt_freelist);
 	vm->vm_nfreetags++;
 	VMEM_UNLOCK(vm);
+}
 static void
 bt_freetrim(vmem_t *vm, int freelimit)
+{
 	bt_t *t;
 	LIST_HEAD(, vmem_btag) tofree;
 	VMEM_ASSERT_LOCKED(vm);
 	LIST_INIT(&tofree);
 	VMEM_LOCK(vm);
 	while (vm->vm_nfreetags > freelimit) {
 		bt_t *bt = LIST_FIRST(&vm->vm_freetags);
 		LIST_REMOVE(bt, bt_freelist);
 		vm->vm_nfreetags--;
 		if (bt >= static_bts
 		    && bt < &static_bts[STATIC_BT_COUNT]) {
 			mutex_enter(&vmem_btag_lock);
 			LIST_INSERT_HEAD(&vmem_btag_freelist, bt, bt_freelist);
 			vmem_btag_freelist_count++;
 			mutex_exit(&vmem_btag_lock);
 			VMEM_EVCNT_DECR(static_bt_inuse);
 		} else {
 			LIST_INSERT_HEAD(&tofree, bt, bt_freelist);
 @@ -413,27 +424,27 @@ bt_freehead_toalloc(vmem_t *vm, vmem_siz
 	return &vm->vm_freelist[idx];
+}
 /* ---- boundary tag hash */
 static struct vmem_hashlist *
 bt_hashhead(vmem_t *vm, vmem_addr_t addr)
+{
 	struct vmem_hashlist *list;
 	unsigned int hash;
 	hash = hash32_buf(&addr, sizeof(addr), HASH32_BUF_INIT);
-	list = &vm->vm_hashlist[hash % vm->vm_hashsize];
+	list = &vm->vm_hashlist[hash & vm->vm_hashmask];
 	return list;
+}
 static bt_t *
 bt_lookupbusy(vmem_t *vm, vmem_addr_t addr)
+{
 	struct vmem_hashlist *list;
 	bt_t *bt;
 	list = bt_hashhead(vm, addr);
 	LIST_FOREACH(bt, list, bt_hashlist) {
 		if (bt->bt_start == addr) {
 @@ -453,27 +464,29 @@ bt_rembusy(vmem_t *vm, bt_t *bt)
 	vm->vm_nbusytag--;
 	LIST_REMOVE(bt, bt_hashlist);
+}
 static void
 bt_insbusy(vmem_t *vm, bt_t *bt)
+{
 	struct vmem_hashlist *list;
 	KASSERT(bt->bt_type == BT_TYPE_BUSY);
 	list = bt_hashhead(vm, bt->bt_start);
 	LIST_INSERT_HEAD(list, bt, bt_hashlist);
-	vm->vm_nbusytag++;
+	if (++vm->vm_nbusytag > vm->vm_maxbusytag) {
 		vm->vm_maxbusytag = vm->vm_nbusytag;
+	}
 	vm->vm_inuse += bt->bt_size;
+}
 /* ---- boundary tag list */
 static void
 bt_remseg(vmem_t *vm, bt_t *bt)
+{
 	TAILQ_REMOVE(&vm->vm_seglist, bt, bt_seglist);
+}
 static void
 @@ -666,178 +679,193 @@ void
 vmem_subsystem_init(vmem_t *vm)
+{
 	kmem_va_meta_arena = vmem_init(&kmem_va_meta_arena_store, "vmem-va",
 , 0, PAGE_SIZE, vmem_alloc, vmem_free, vm,
 , VM_NOSLEEP | VM_BOOTSTRAP | VM_LARGEIMPORT,
 	    IPL_VM);
 	kmem_meta_arena = vmem_init(&kmem_meta_arena_store, "vmem-meta",
 , 0, PAGE_SIZE,
 	    uvm_km_kmem_alloc, uvm_km_kmem_free, kmem_va_meta_arena,
 , VM_NOSLEEP | VM_BOOTSTRAP, IPL_VM);
-	pool_init(&vmem_btag_pool, sizeof(bt_t), 0, 0, PR_PHINPAGE,
+	pool_init(&vmem_btag_pool, sizeof(bt_t), coherency_unit, 0,
-		    "vmembt", &pool_allocator_vmem_meta, IPL_VM);
+	    PR_PHINPAGE, "vmembt", &pool_allocator_vmem_meta, IPL_VM);
+}
 #endif /* defined(_KERNEL) */
 static int
 vmem_add1(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, vm_flag_t flags,
     int spanbttype)
+{
 	bt_t *btspan;
 	bt_t *btfree;
 	VMEM_ASSERT_LOCKED(vm);
 	KASSERT((flags & (VM_SLEEP|VM_NOSLEEP)) != 0);
 	KASSERT((~flags & (VM_SLEEP|VM_NOSLEEP)) != 0);
 	KASSERT(spanbttype == BT_TYPE_SPAN ||
 	    spanbttype == BT_TYPE_SPAN_STATIC);
 	btspan = bt_alloc(vm, flags);
 	if (btspan == NULL) {
 		return ENOMEM;
+	}
 	btfree = bt_alloc(vm, flags);
 	if (btfree == NULL) {
 		bt_free(vm, btspan);
 		return ENOMEM;
+	}
 	btspan->bt_type = spanbttype;
 	btspan->bt_start = addr;
 	btspan->bt_size = size;
 	btfree->bt_type = BT_TYPE_FREE;
 	btfree->bt_start = addr;
 	btfree->bt_size = size;
 	VMEM_LOCK(vm);
 	bt_insseg_tail(vm, btspan);
 	bt_insseg(vm, btfree, btspan);
 	bt_insfree(vm, btfree);
 	vm->vm_size += size;
 	VMEM_UNLOCK(vm);
 	return 0;
+}
 static void
 vmem_destroy1(vmem_t *vm)
+{
 #if defined(QCACHE)
 	qc_destroy(vm);
 #endif /* defined(QCACHE) */
-	if (vm->vm_hashlist != NULL) {
+	VMEM_LOCK(vm);
 		int i;
-		for (i = 0; i < vm->vm_hashsize; i++) {
+	for (int i = 0; i < vm->vm_hashsize; i++) {
-			bt_t *bt;
+		bt_t *bt;
-			while ((bt = LIST_FIRST(&vm->vm_hashlist[i])) != NULL) {
+		while ((bt = LIST_FIRST(&vm->vm_hashlist[i])) != NULL) {
-				KASSERT(bt->bt_type == BT_TYPE_SPAN_STATIC);
+			KASSERT(bt->bt_type == BT_TYPE_SPAN_STATIC);
-				bt_free(vm, bt);
+			LIST_REMOVE(bt, bt_hashlist);
 			bt_free(vm, bt);
 		if (vm->vm_hashlist != &vm->vm_hash0) {
 			xfree(vm->vm_hashlist,
 			    sizeof(struct vmem_hashlist *) * vm->vm_hashsize);
+		}
+	}
 	/* bt_freetrim() drops the lock. */
 	bt_freetrim(vm, 0);
 	if (vm->vm_hashlist != &vm->vm_hash0) {
 		xfree(vm->vm_hashlist,
 		    sizeof(struct vmem_hashlist) * vm->vm_hashsize);
+	}
 	VMEM_CONDVAR_DESTROY(vm);
 	VMEM_LOCK_DESTROY(vm);
 	xfree(vm, sizeof(*vm));
+}
 static int
 vmem_import(vmem_t *vm, vmem_size_t size, vm_flag_t flags)
+{
 	vmem_addr_t addr;
 	int rc;
 	VMEM_ASSERT_LOCKED(vm);
 	if (vm->vm_importfn == NULL) {
 		return EINVAL;
+	}
 	if (vm->vm_flags & VM_LARGEIMPORT) {
 		size *= 16;
+	}
 	VMEM_UNLOCK(vm);
 	if (vm->vm_flags & VM_XIMPORT) {
 		rc = __FPTRCAST(vmem_ximport_t *, vm->vm_importfn)(vm->vm_arg,
 		    size, &size, flags, &addr);
 	} else {
 		rc = (vm->vm_importfn)(vm->vm_arg, size, flags, &addr);
+	}
 	VMEM_LOCK(vm);
 	if (rc) {
 		return ENOMEM;
+	}
 	if (vmem_add1(vm, addr, size, flags, BT_TYPE_SPAN) != 0) {
 		VMEM_UNLOCK(vm);
 		(*vm->vm_releasefn)(vm->vm_arg, addr, size);
 		VMEM_LOCK(vm);
 		return ENOMEM;
+	}
 	return 0;
+}
 static int
 vmem_rehash(vmem_t *vm, size_t newhashsize, vm_flag_t flags)
+{
 	bt_t *bt;
 	int i;
 	struct vmem_hashlist *newhashlist;
 	struct vmem_hashlist *oldhashlist;
 	size_t oldhashsize;
 	KASSERT(newhashsize > 0);
 	/* Round hash size up to a power of 2. */
 	newhashsize = 1 << (ilog2(newhashsize) + 1);
 	newhashlist =
-	    xmalloc(sizeof(struct vmem_hashlist *) * newhashsize, flags);
+	    xmalloc(sizeof(struct vmem_hashlist) * newhashsize, flags);
 	if (newhashlist == NULL) {
 		return ENOMEM;
+	}
 	for (i = 0; i < newhashsize; i++) {
 		LIST_INIT(&newhashlist[i]);
+	}
-	if (!VMEM_TRYLOCK(vm)) {
+	VMEM_LOCK(vm);
-		xfree(newhashlist,
+	/* Decay back to a small hash slowly. */
-		    sizeof(struct vmem_hashlist *) * newhashsize);
+	if (vm->vm_maxbusytag >= 2) {
-		return EBUSY;
+		vm->vm_maxbusytag = vm->vm_maxbusytag / 2 - 1;
 		if (vm->vm_nbusytag > vm->vm_maxbusytag) {
 			vm->vm_maxbusytag = vm->vm_nbusytag;
+		}
 	} else {
 		vm->vm_maxbusytag = vm->vm_nbusytag;
+	}
 	oldhashlist = vm->vm_hashlist;
 	oldhashsize = vm->vm_hashsize;
 	vm->vm_hashlist = newhashlist;
 	vm->vm_hashsize = newhashsize;
 	vm->vm_hashmask = newhashsize - 1;
 	if (oldhashlist == NULL) {
 		VMEM_UNLOCK(vm);
 		return 0;
+	}
 	for (i = 0; i < oldhashsize; i++) {
 		while ((bt = LIST_FIRST(&oldhashlist[i])) != NULL) {
 			bt_rembusy(vm, bt); /* XXX */
 			bt_insbusy(vm, bt);
+		}
+	}
 	VMEM_UNLOCK(vm);
 	if (oldhashlist != &vm->vm_hash0) {
 		xfree(oldhashlist,
-		    sizeof(struct vmem_hashlist *) * oldhashsize);
+		    sizeof(struct vmem_hashlist) * oldhashsize);
+	}
 	return 0;
+}
 /*
  * vmem_fit: check if a bt can satisfy the given restrictions.
+ *
  * it's a caller's responsibility to ensure the region is big enough
  * before calling us.
  */
 static int
 @@ -924,38 +952,40 @@ vmem_init(vmem_t *vm, const char *name,
 	VMEM_CONDVAR_INIT(vm, "vmem");
 	VMEM_LOCK_INIT(vm, ipl);
 	vm->vm_flags = flags;
 	vm->vm_nfreetags = 0;
 	LIST_INIT(&vm->vm_freetags);
 	strlcpy(vm->vm_name, name, sizeof(vm->vm_name));
 	vm->vm_quantum_mask = quantum - 1;
 	vm->vm_quantum_shift = SIZE2ORDER(quantum);
 	KASSERT(ORDER2SIZE(vm->vm_quantum_shift) == quantum);
 	vm->vm_importfn = importfn;
 	vm->vm_releasefn = releasefn;
 	vm->vm_arg = arg;
 	vm->vm_nbusytag = 0;
 	vm->vm_maxbusytag = 0;
 	vm->vm_size = 0;
 	vm->vm_inuse = 0;
 #if defined(QCACHE)
 	qc_init(vm, qcache_max, ipl);
 #endif /* defined(QCACHE) */
 	TAILQ_INIT(&vm->vm_seglist);
 	for (i = 0; i < VMEM_MAXORDER; i++) {
 		LIST_INIT(&vm->vm_freelist[i]);
+	}
-	memset(&vm->vm_hash0, 0, sizeof(struct vmem_hashlist));
+	memset(&vm->vm_hash0, 0, sizeof(vm->vm_hash0));
 	vm->vm_hashsize = 1;
 	vm->vm_hashmask = vm->vm_hashsize - 1;
 	vm->vm_hashlist = &vm->vm_hash0;
 	if (size != 0) {
 		if (vmem_add(vm, base, size, flags) != 0) {
 			vmem_destroy1(vm);
 			return NULL;
+		}
+	}
 #if defined(_KERNEL)
 	if (flags & VM_BOOTSTRAP) {
 		bt_refill(vm);
+	}
 @@ -1097,45 +1127,47 @@ vmem_xalloc(vmem_t *vm, const vmem_size_
 	KASSERT((nocross & (nocross - 1)) == 0);
 	KASSERT((align == 0 && phase == 0) || phase < align);
 	KASSERT(nocross == 0 || nocross >= size);
 	KASSERT(minaddr <= maxaddr);
 	KASSERT(!VMEM_CROSS_P(phase, phase + size - 1, nocross));
 	if (align == 0) {
 		align = vm->vm_quantum_mask + 1;
+	}
 	/*
 	 * allocate boundary tags before acquiring the vmem lock.
 	 */
 	VMEM_LOCK(vm);
 	btnew = bt_alloc(vm, flags);
 	if (btnew == NULL) {
 		VMEM_UNLOCK(vm);
 		return ENOMEM;
+	}
 	btnew2 = bt_alloc(vm, flags); /* XXX not necessary if no restrictions */
 	if (btnew2 == NULL) {
 		bt_free(vm, btnew);
 		VMEM_UNLOCK(vm);
 		return ENOMEM;
+	}
 	/*
 	 * choose a free block from which we allocate.
 	 */
 retry_strat:
 	first = bt_freehead_toalloc(vm, size, strat);
 	end = &vm->vm_freelist[VMEM_MAXORDER];
 retry:
 	bt = NULL;
 	VMEM_LOCK(vm);
 	vmem_check(vm);
 	if (strat == VM_INSTANTFIT) {
 		/*
 		 * just choose the first block which satisfies our restrictions.
+		 *
 		 * note that we don't need to check the size of the blocks
 		 * because any blocks found on these list should be larger than
 		 * the given size.
 		 */
 		for (list = first; list < end; list++) {
 			bt = LIST_FIRST(list);
 			if (bt != NULL) {
 				rc = vmem_fit(bt, size, align, phase,
 @@ -1166,58 +1198,56 @@ retry:
 		 */
 		for (list = first; list < end; list++) {
 			LIST_FOREACH(bt, list, bt_freelist) {
 				if (bt->bt_size >= size) {
 					rc = vmem_fit(bt, size, align, phase,
 					    nocross, minaddr, maxaddr, &start);
 					if (rc == 0) {
 						goto gotit;
+					}
+				}
+			}
+		}
+	}
 	VMEM_UNLOCK(vm);
 #if 1
 	if (strat == VM_INSTANTFIT) {
 		strat = VM_BESTFIT;
 		goto retry_strat;
+	}
 #endif
 	if (align != vm->vm_quantum_mask + 1 || phase != 0 || nocross != 0) {
 		/*
 		 * XXX should try to import a region large enough to
 		 * satisfy restrictions?
 		 */
 		goto fail;
+	}
 	/* XXX eeek, minaddr & maxaddr not respected */
 	if (vmem_import(vm, size, flags) == 0) {
 		goto retry;
+	}
 	/* XXX */
 	if ((flags & VM_SLEEP) != 0) {
 		vmem_kick_pdaemon();
 		VMEM_LOCK(vm);
 		VMEM_CONDVAR_WAIT(vm);
 		VMEM_UNLOCK(vm);
 		goto retry;
+	}
 fail:
 	bt_free(vm, btnew);
 	bt_free(vm, btnew2);
 	VMEM_UNLOCK(vm);
 	return ENOMEM;
 gotit:
 	KASSERT(bt->bt_type == BT_TYPE_FREE);
 	KASSERT(bt->bt_size >= size);
 	bt_remfree(vm, bt);
 	vmem_check(vm);
 	if (bt->bt_start != start) {
 		btnew2->bt_type = BT_TYPE_FREE;
 		btnew2->bt_start = bt->bt_start;
 		btnew2->bt_size = start - bt->bt_start;
 		bt->bt_start = start;
 		bt->bt_size -= btnew2->bt_size;
 @@ -1228,43 +1258,41 @@ gotit:
+	}
 	KASSERT(bt->bt_start == start);
 	if (bt->bt_size != size && bt->bt_size - size > vm->vm_quantum_mask) {
 		/* split */
 		btnew->bt_type = BT_TYPE_BUSY;
 		btnew->bt_start = bt->bt_start;
 		btnew->bt_size = size;
 		bt->bt_start = bt->bt_start + size;
 		bt->bt_size -= size;
 		bt_insfree(vm, bt);
 		bt_insseg(vm, btnew, TAILQ_PREV(bt, vmem_seglist, bt_seglist));
 		bt_insbusy(vm, btnew);
 		vmem_check(vm);
 		VMEM_UNLOCK(vm);
 	} else {
 		bt->bt_type = BT_TYPE_BUSY;
 		bt_insbusy(vm, bt);
 		vmem_check(vm);
 		VMEM_UNLOCK(vm);
 		bt_free(vm, btnew);
 		btnew = bt;
+	}
 	if (btnew2 != NULL) {
 		bt_free(vm, btnew2);
+	}
 	KASSERT(btnew->bt_size >= size);
 	btnew->bt_type = BT_TYPE_BUSY;
 	if (addrp != NULL)
 		*addrp = btnew->bt_start;
 	VMEM_UNLOCK(vm);
 	return 0;
+}
 /*
  * vmem_free: free the resource to the arena.
  */
 void
 vmem_free(vmem_t *vm, vmem_addr_t addr, vmem_size_t size)
+{
 	KASSERT(size > 0);
 @@ -1276,108 +1304,104 @@ vmem_free(vmem_t *vm, vmem_addr_t addr,
 		pool_cache_put(qc->qc_cache, (void *)addr);
 		return;
+	}
 #endif /* defined(QCACHE) */
 	vmem_xfree(vm, addr, size);
+}
 void
 vmem_xfree(vmem_t *vm, vmem_addr_t addr, vmem_size_t size)
+{
 	bt_t *bt;
 	bt_t *t;
 	LIST_HEAD(, vmem_btag) tofree;
 	LIST_INIT(&tofree);
 	KASSERT(size > 0);
 	VMEM_LOCK(vm);
 	bt = bt_lookupbusy(vm, addr);
 	KASSERT(bt != NULL);
 	KASSERT(bt->bt_start == addr);
 	KASSERT(bt->bt_size == vmem_roundup_size(vm, size) ||
 	    bt->bt_size - vmem_roundup_size(vm, size) <= vm->vm_quantum_mask);
 	KASSERT(bt->bt_type == BT_TYPE_BUSY);
 	bt_rembusy(vm, bt);
 	bt->bt_type = BT_TYPE_FREE;
 	/* coalesce */
 	t = TAILQ_NEXT(bt, bt_seglist);
 	if (t != NULL && t->bt_type == BT_TYPE_FREE) {
 		KASSERT(BT_END(bt) < t->bt_start);	/* YYY */
 		bt_remfree(vm, t);
 		bt_remseg(vm, t);
 		bt->bt_size += t->bt_size;
-		LIST_INSERT_HEAD(&tofree, t, bt_freelist);
+		bt_free(vm, t);
+	}
 	t = TAILQ_PREV(bt, vmem_seglist, bt_seglist);
 	if (t != NULL && t->bt_type == BT_TYPE_FREE) {
 		KASSERT(BT_END(t) < bt->bt_start);	/* YYY */
 		bt_remfree(vm, t);
 		bt_remseg(vm, t);
 		bt->bt_size += t->bt_size;
 		bt->bt_start = t->bt_start;
-		LIST_INSERT_HEAD(&tofree, t, bt_freelist);
+		bt_free(vm, t);
+	}
 	t = TAILQ_PREV(bt, vmem_seglist, bt_seglist);
 	KASSERT(t != NULL);
 	KASSERT(BT_ISSPAN_P(t) || t->bt_type == BT_TYPE_BUSY);
 	if (vm->vm_releasefn != NULL && t->bt_type == BT_TYPE_SPAN &&
 	    t->bt_size == bt->bt_size) {
 		vmem_addr_t spanaddr;
 		vmem_size_t spansize;
 		KASSERT(t->bt_start == bt->bt_start);
 		spanaddr = bt->bt_start;
 		spansize = bt->bt_size;
 		bt_remseg(vm, bt);
-		LIST_INSERT_HEAD(&tofree, bt, bt_freelist);
+		bt_free(vm, bt);
 		bt_remseg(vm, t);
-		LIST_INSERT_HEAD(&tofree, t, bt_freelist);
+		bt_free(vm, t);
 		vm->vm_size -= spansize;
 		VMEM_CONDVAR_BROADCAST(vm);
-		VMEM_UNLOCK(vm);
+		/* bt_freetrim() drops the lock. */
 		bt_freetrim(vm, BT_MAXFREE);
 		(*vm->vm_releasefn)(vm->vm_arg, spanaddr, spansize);
 	} else {
 		bt_insfree(vm, bt);
 		VMEM_CONDVAR_BROADCAST(vm);
-		VMEM_UNLOCK(vm);
+		/* bt_freetrim() drops the lock. */
 		bt_freetrim(vm, BT_MAXFREE);
+	}
 	while (!LIST_EMPTY(&tofree)) {
 		t = LIST_FIRST(&tofree);
 		LIST_REMOVE(t, bt_freelist);
 		bt_free(vm, t);
 	bt_freetrim(vm, BT_MAXFREE);
+}
 /*
  * vmem_add:
+ *
  * => caller must ensure appropriate spl,
  *    if the arena can be accessed from interrupt context.
  */
 int
 vmem_add(vmem_t *vm, vmem_addr_t addr, vmem_size_t size, vm_flag_t flags)
+{
 	int rv;
-	return vmem_add1(vm, addr, size, flags, BT_TYPE_SPAN_STATIC);
+	VMEM_LOCK(vm);
 	rv = vmem_add1(vm, addr, size, flags, BT_TYPE_SPAN_STATIC);
 	VMEM_UNLOCK(vm);
 	return rv;
+}
 /*
  * vmem_size: information about arenas size
+ *
  * => return free/allocated size in arena
  */
 vmem_size_t
 vmem_size(vmem_t *vm, int typemask)
+{
 	switch (typemask) {
 	case VMEM_ALLOC:
 @@ -1400,32 +1424,28 @@ static struct workqueue *vmem_rehash_wq;
 static struct work vmem_rehash_wk;
 static void
 vmem_rehash_all(struct work *wk, void *dummy)
+{
 	vmem_t *vm;
 	KASSERT(wk == &vmem_rehash_wk);
 	mutex_enter(&vmem_list_lock);
 	LIST_FOREACH(vm, &vmem_list, vm_alllist) {
 		size_t desired;
 		size_t current;
-		if (!VMEM_TRYLOCK(vm)) {
+		desired = atomic_load_relaxed(&vm->vm_maxbusytag);
-			continue;
+		current = atomic_load_relaxed(&vm->vm_hashsize);
 		desired = vm->vm_nbusytag;
 		current = vm->vm_hashsize;
 		VMEM_UNLOCK(vm);
 		if (desired > VMEM_HASHSIZE_MAX) {
 			desired = VMEM_HASHSIZE_MAX;
 		} else if (desired < VMEM_HASHSIZE_MIN) {
 			desired = VMEM_HASHSIZE_MIN;
+		}
 		if (desired > current * 2 || desired * 2 < current) {
 			vmem_rehash(vm, desired, VM_NOSLEEP);
+		}
+	}
 	mutex_exit(&vmem_list_lock);
 	callout_schedule(&vmem_rehash_ch, vmem_rehash_interval);

 @@ -1,14 +1,14 @@
-/*	$NetBSD: vmem_impl.h,v 1.3 2013/11/22 21:04:11 christos Exp $	*/
+/*	$NetBSD: vmem_impl.h,v 1.4 2020/04/19 21:11:42 ad Exp $	*/
 /*-
  * Copyright (c)2006 YAMAMOTO Takashi,
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -85,27 +85,29 @@ typedef struct qcache qcache_t;
 /* vmem arena */
 struct vmem {
 	CONDVAR_DECL(vm_cv);
 	LOCK_DECL(vm_lock);
 	vm_flag_t vm_flags;
 	vmem_import_t *vm_importfn;
 	vmem_release_t *vm_releasefn;
 	size_t vm_nfreetags;
 	LIST_HEAD(, vmem_btag) vm_freetags;
 	void *vm_arg;
 	struct vmem_seglist vm_seglist;
 	struct vmem_freelist vm_freelist[VMEM_MAXORDER];
 	size_t vm_hashsize;
 	size_t vm_hashmask;
 	size_t vm_nbusytag;
 	size_t vm_maxbusytag;
 	struct vmem_hashlist *vm_hashlist;
 	struct vmem_hashlist vm_hash0;
 	size_t vm_quantum_mask;
 	int vm_quantum_shift;
 	size_t vm_size;
 	size_t vm_inuse;
 	char vm_name[VMEM_NAME_MAX+1];
 	LIST_ENTRY(vmem) vm_alllist;
 #if defined(QCACHE)
 	/* quantum cache */
 	size_t vm_qcache_max;
 	struct pool_allocator vm_qcache_allocator;