 @@ -1,14 +1,14 @@
-/*	$NetBSD: uvm_amap.c,v 1.112 2020/01/01 13:11:51 ad Exp $	*/
+/*	$NetBSD: uvm_amap.c,v 1.113 2020/01/01 22:01:13 ad Exp $	*/
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -25,27 +25,27 @@
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 /*
  * uvm_amap.c: amap operations
  */
 /*
  * this file contains functions that perform operations on amaps.  see
  * uvm_amap.h for a brief explanation of the role of amaps in uvm.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_amap.c,v 1.112 2020/01/01 13:11:51 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_amap.c,v 1.113 2020/01/01 22:01:13 ad Exp $");
 #include "opt_uvmhist.h"
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/pool.h>
 #include <sys/atomic.h>
 #include <uvm/uvm.h>
 #include <uvm/uvm_swap.h>
 @@ -53,49 +53,38 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_amap.c,v
  * cache for allocation of vm_map structures.  note that in order to
  * avoid an endless loop, the amap cache's allocator cannot allocate
  * memory from an amap (it currently goes through the kernel uobj, so
  * we are ok).
  */
 static struct pool_cache uvm_amap_cache;
 static kmutex_t amap_list_lock __cacheline_aligned;
 static LIST_HEAD(, vm_amap) amap_list;
 /*
  * local functions
  */
 static inline void
 amap_list_insert(struct vm_amap *amap)
 	mutex_enter(&amap_list_lock);
 	LIST_INSERT_HEAD(&amap_list, amap, am_list);
 	mutex_exit(&amap_list_lock);
 static inline void
 amap_list_remove(struct vm_amap *amap)
 	mutex_enter(&amap_list_lock);
 	LIST_REMOVE(amap, am_list);
 	mutex_exit(&amap_list_lock);
 static int
 amap_roundup_slots(int slots)
+{
 #ifdef _LP64
 	/* Align to cacheline boundary for best performance. */
 	return roundup2((slots * sizeof(struct vm_amap *)),
 	    COHERENCY_UNIT) / sizeof(struct vm_amap *);
 #else
 	/* On 32-bit, KVA shortage is a concern. */
 	return kmem_roundup_size(slots * sizeof(int)) / sizeof(int);
 #endif
+}
 #ifdef UVM_AMAP_PPREF
 /*
  * what is ppref?   ppref is an _optional_ amap feature which is used
  * to keep track of reference counts on a per-page basis.  it is enabled
  * when UVM_AMAP_PPREF is defined.
+ *
  * when enabled, an array of ints is allocated for the pprefs.  this
  * array is allocated only when a partial reference is added to the
  * map (either by unmapping part of the amap, or gaining a reference
  * to only a part of an amap).  if the allocation of the array fails
  * (KM_NOSLEEP), then we set the array pointer to PPREF_NONE to indicate
 @@ -160,66 +149,99 @@ pp_setreflen(int *ppref, int offset, int
 #endif /* UVM_AMAP_PPREF */
 /*
  * amap_alloc1: allocate an amap, but do not initialise the overlay.
+ *
  * => Note: lock is not set.
  */
 static struct vm_amap *
 amap_alloc1(int slots, int padslots, int flags)
+{
 	const bool nowait = (flags & UVM_FLAG_NOWAIT) != 0;
 	const km_flag_t kmflags = nowait ? KM_NOSLEEP : KM_SLEEP;
 	struct vm_amap *amap;
 	kmutex_t *newlock, *oldlock;
 	int totalslots;
 	size_t sz;
 	amap = pool_cache_get(&uvm_amap_cache, nowait ? PR_NOWAIT : PR_WAITOK);
 	if (amap == NULL) {
 		return NULL;
+	}
-	totalslots = amap_roundup_slots(slots + padslots);
+	KASSERT(amap->am_lock != NULL);
-	amap->am_lock = NULL;
+	KASSERT(amap->am_nused == 0);
 	/* Try to privatize the lock if currently shared. */
 	if (mutex_obj_refcnt(amap->am_lock) > 1) {
 		newlock = mutex_obj_tryalloc(MUTEX_DEFAULT, IPL_NONE);
 		if (newlock != NULL) {
 		    	oldlock = amap->am_lock;
 		    	mutex_enter(&amap_list_lock);
 		    	amap->am_lock = newlock;
 		    	mutex_exit(&amap_list_lock);
 		    	mutex_obj_free(oldlock);
+		}
+	}
 	totalslots = slots + padslots;
 	amap->am_ref = 1;
 	amap->am_flags = 0;
 #ifdef UVM_AMAP_PPREF
 	amap->am_ppref = NULL;
 #endif
 	amap->am_maxslot = totalslots;
 	amap->am_nslot = slots;
 	amap->am_nused = 0;
 	/*
-	 * Note: since allocations are likely big, we expect to reduce the
+	 * For small amaps use the storage in the amap structure.  Otherwise
-	 * memory fragmentation by allocating them in separate blocks.
+	 * go to the heap.  Note: since allocations are likely big, we
-	 */
+	 * expect to reduce the memory fragmentation by allocating them in
-	amap->am_slots = kmem_alloc(totalslots * sizeof(int), kmflags);
+	 * separate blocks.
-	if (amap->am_slots == NULL)
+	 */
-		goto fail1;
+	if (totalslots <= UVM_AMAP_TINY) {
 		amap->am_maxslot = UVM_AMAP_TINY;
-	amap->am_bckptr = kmem_alloc(totalslots * sizeof(int), kmflags);
+		amap->am_anon = AMAP_TINY_ANON(amap);
-	if (amap->am_bckptr == NULL)
+		amap->am_slots = AMAP_TINY_SLOTS(amap);
-		goto fail2;
+		amap->am_bckptr = amap->am_slots + UVM_AMAP_TINY;
 	} else if (totalslots <= UVM_AMAP_SMALL) {
-	amap->am_anon = kmem_alloc(totalslots * sizeof(struct vm_anon *),
+		amap->am_maxslot = UVM_AMAP_SMALL;
-	    kmflags);
+		amap->am_anon = AMAP_TINY_ANON(amap);
 	if (amap->am_anon == NULL)
-		goto fail3;
+		sz = UVM_AMAP_SMALL * sizeof(int) * 2;
 		sz = roundup2(sz, COHERENCY_UNIT);
 		amap->am_slots = kmem_alloc(sz, kmflags);
 		if (amap->am_slots == NULL)
 			goto fail1;
 		amap->am_bckptr = amap->am_slots + amap->am_maxslot;
 	} else {
 		amap->am_maxslot = amap_roundup_slots(totalslots);
 		sz = amap->am_maxslot * sizeof(int) * 2;
 		KASSERT((sz & (COHERENCY_UNIT - 1)) == 0);
 		amap->am_slots = kmem_alloc(sz, kmflags);
 		if (amap->am_slots == NULL)
 			goto fail1;
 		amap->am_bckptr = amap->am_slots + amap->am_maxslot;
 		amap->am_anon = kmem_alloc(amap->am_maxslot *
 		    sizeof(struct vm_anon *), kmflags);
 		if (amap->am_anon == NULL)
 			goto fail2;
+	}
 	return amap;
 fail3:
 	kmem_free(amap->am_bckptr, totalslots * sizeof(int));
 fail2:
-	kmem_free(amap->am_slots, totalslots * sizeof(int));
+	kmem_free(amap->am_slots, amap->am_maxslot * sizeof(int));
 fail1:
 	pool_cache_put(&uvm_amap_cache, amap);
 	/*
 	 * XXX hack to tell the pagedaemon how many pages we need,
 	 * since we can need more than it would normally free.
 	 */
 	if (nowait) {
 		extern u_int uvm_extrapages;
 		atomic_add_int(&uvm_extrapages,
 		    ((sizeof(int) * 2 + sizeof(struct vm_anon *)) *
 		    totalslots) >> PAGE_SHIFT);
+	}
 @@ -238,74 +260,128 @@ struct vm_amap *
 amap_alloc(vaddr_t sz, vaddr_t padsz, int waitf)
+{
 	struct vm_amap *amap;
 	int slots, padslots;
 	UVMHIST_FUNC("amap_alloc"); UVMHIST_CALLED(maphist);
 	AMAP_B2SLOT(slots, sz);
 	AMAP_B2SLOT(padslots, padsz);
 	amap = amap_alloc1(slots, padslots, waitf);
 	if (amap) {
 		memset(amap->am_anon, 0,
 		    amap->am_maxslot * sizeof(struct vm_anon *));
 		amap->am_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
 		amap_list_insert(amap);
+	}
 	UVMHIST_LOG(maphist,"<- done, amap = 0x%#jx, sz=%jd", (uintptr_t)amap,
 	    sz, 0, 0);
 	return(amap);
+}
 /*
  * amap_ctor: pool_cache constructor for new amaps
+ *
  * => carefully synchronize with amap_swap_off()
  */
 static int
 amap_ctor(void *arg, void *obj, int flags)
+{
 	struct vm_amap *amap = obj;
 	if ((flags & PR_NOWAIT) != 0) {
 		amap->am_lock = mutex_obj_tryalloc(MUTEX_DEFAULT, IPL_NONE);
 		if (amap->am_lock == NULL) {
 			return ENOMEM;
+		}
 	} else {
 		amap->am_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
+	}
 	amap->am_nused = 0;
 	amap->am_flags = 0;
 	mutex_enter(&amap_list_lock);
 	LIST_INSERT_HEAD(&amap_list, amap, am_list);
 	mutex_exit(&amap_list_lock);
 	return 0;
+}
 /*
  * amap_ctor: pool_cache destructor for amaps
+ *
  * => carefully synchronize with amap_swap_off()
  */
 static void
 amap_dtor(void *arg, void *obj)
+{
 	struct vm_amap *amap = obj;
 	KASSERT(amap->am_nused == 0);
 	mutex_enter(&amap_list_lock);
 	LIST_REMOVE(amap, am_list);
 	mutex_exit(&amap_list_lock);
 	mutex_obj_free(amap->am_lock);
+}
 /*
  * uvm_amap_init: initialize the amap system.
  */
 void
 uvm_amap_init(void)
+{
 #if defined(_LP64)
 	/*
 	 * Correct alignment helps performance.  For 32-bit platforms, KVA
 	 * availibility is a concern so leave them be.
 	 */
 	KASSERT((sizeof(struct vm_amap) & (COHERENCY_UNIT - 1)) == 0);
 #endif
 	mutex_init(&amap_list_lock, MUTEX_DEFAULT, IPL_NONE);
-	pool_cache_bootstrap(&uvm_amap_cache, sizeof(struct vm_amap), 0, 0, 0,
+	pool_cache_bootstrap(&uvm_amap_cache, sizeof(struct vm_amap),
-	    "amappl", NULL, IPL_NONE, NULL, NULL, NULL);
+	    COHERENCY_UNIT, 0, 0, "amappl", NULL, IPL_NONE, amap_ctor,
 	    amap_dtor, NULL);
+}
 /*
  * amap_free: free an amap
+ *
  * => the amap must be unlocked
  * => the amap should have a zero reference count and be empty
  */
 void
 amap_free(struct vm_amap *amap)
+{
 	int slots;
 	UVMHIST_FUNC("amap_free"); UVMHIST_CALLED(maphist);
 	KASSERT(amap->am_ref == 0 && amap->am_nused == 0);
 	KASSERT((amap->am_flags & AMAP_SWAPOFF) == 0);
 	if (amap->am_lock != NULL) {
 		KASSERT(!mutex_owned(amap->am_lock));
 		mutex_obj_free(amap->am_lock);
 	slots = amap->am_maxslot;
-	kmem_free(amap->am_slots, slots * sizeof(*amap->am_slots));
+	if (amap->am_slots != AMAP_TINY_SLOTS(amap)) {
-	kmem_free(amap->am_bckptr, slots * sizeof(*amap->am_bckptr));
+		kmem_free(amap->am_slots, roundup2(slots * sizeof(int) * 2,
-	kmem_free(amap->am_anon, slots * sizeof(*amap->am_anon));
+		    COHERENCY_UNIT));
+	}
 	if (amap->am_anon != AMAP_TINY_ANON(amap)) {
 		kmem_free(amap->am_anon, slots * sizeof(*amap->am_anon));
+	}
 #ifdef UVM_AMAP_PPREF
-	if (amap->am_ppref && amap->am_ppref != PPREF_NONE)
+	if (amap->am_ppref && amap->am_ppref != PPREF_NONE) {
-		kmem_free(amap->am_ppref, slots * sizeof(*amap->am_ppref));
+		kmem_free(amap->am_ppref, roundup2(slots * sizeof(int),
 		    COHERENCY_UNIT));
+	}
 #endif
 	pool_cache_put(&uvm_amap_cache, amap);
 	UVMHIST_LOG(maphist,"<- done, freed amap = 0x%#jx", (uintptr_t)amap,
 , 0, 0);
+}
 /*
  * amap_extend: extend the size of an amap (if needed)
+ *
  * => called from uvm_map when we want to extend an amap to cover
  *    a new mapping (rather than allocate a new one)
  * => amap should be unlocked (we will lock it)
  * => to safely extend an amap it should have a reference count of
 @@ -336,28 +412,27 @@ amap_extend(struct vm_map_entry *entry,
 	 * first, determine how many slots we need in the amap.  don't
 	 * forget that ar_pageoff could be non-zero: this means that
 	 * there are some unused slots before us in the amap.
 	 */
 	amap_lock(amap);
 	KASSERT(amap_refs(amap) == 1); /* amap can't be shared */
 	AMAP_B2SLOT(slotmapped, entry->end - entry->start); /* slots mapped */
 	AMAP_B2SLOT(slotadd, addsize);			/* slots to add */
 	if (flags & AMAP_EXTEND_FORWARDS) {
 		slotneed = slotoff + slotmapped + slotadd;
 		slotadj = 0;
 		slotarea = 0;
 	} else {
 	else {
 		slotneed = slotadd + slotmapped;
 		slotadj = slotadd - slotoff;
 		slotarea = amap->am_maxslot - slotmapped;
+	}
 	tofree = NULL;
 	/*
 	 * case 1: we already have enough slots in the map and thus
 	 * only need to bump the reference counts on the slots we are
 	 * adding.
 	 */
 	if (flags & AMAP_EXTEND_FORWARDS) {
 @@ -492,43 +567,42 @@ amap_extend(struct vm_map_entry *entry,
 	 */
 	amap_unlock(amap);
 	if (slotneed >= UVM_AMAP_LARGE) {
 		return E2BIG;
+	}
 	slotalloc = amap_roundup_slots(slotneed);
 #ifdef UVM_AMAP_PPREF
 	newppref = NULL;
 	if (amap->am_ppref && amap->am_ppref != PPREF_NONE) {
 		/* Will be handled later if fails. */
-		newppref = kmem_alloc(slotalloc * sizeof(*newppref), kmflags);
+		newppref = kmem_alloc(roundup2(slotalloc * sizeof(int),
 		    COHERENCY_UNIT), kmflags);
+	}
 #endif
-	newsl = kmem_alloc(slotalloc * sizeof(*newsl), kmflags);
+	newsl = kmem_alloc(slotalloc * sizeof(*newsl) * 2, kmflags);
-	newbck = kmem_alloc(slotalloc * sizeof(*newbck), kmflags);
+	newbck = newsl + slotalloc;
 	newover = kmem_alloc(slotalloc * sizeof(*newover), kmflags);
 	if (newsl == NULL || newbck == NULL || newover == NULL) {
 #ifdef UVM_AMAP_PPREF
 		if (newppref != NULL) {
-			kmem_free(newppref, slotalloc * sizeof(*newppref));
+			kmem_free(newppref, roundup2(slotalloc * sizeof(int),
 			    COHERENCY_UNIT));
+		}
 #endif
 		if (newsl != NULL) {
-			kmem_free(newsl, slotalloc * sizeof(*newsl));
+			kmem_free(newsl, slotalloc * sizeof(*newsl) * 2);
 		if (newbck != NULL) {
 			kmem_free(newbck, slotalloc * sizeof(*newbck));
+		}
 		if (newover != NULL) {
 			kmem_free(newover, slotalloc * sizeof(*newover));
+		}
 		return ENOMEM;
+	}
 	amap_lock(amap);
 	KASSERT(amap->am_maxslot < slotneed);
 	/*
 	 * Copy everything over to new allocated areas.
 	 */
 @@ -605,32 +679,38 @@ amap_extend(struct vm_map_entry *entry,
 	/* update master values */
 	if (flags & AMAP_EXTEND_FORWARDS)
 		amap->am_nslot = slotneed;
 	else {
 		entry->aref.ar_pageoff = slotarea - slotadd;
 		amap->am_nslot = slotalloc;
+	}
 	oldnslots = amap->am_maxslot;
 	amap->am_maxslot = slotalloc;
 	uvm_anon_freelst(amap, tofree);
-	kmem_free(oldsl, oldnslots * sizeof(*oldsl));
+	if (oldsl != AMAP_TINY_SLOTS(amap)) {
-	kmem_free(oldbck, oldnslots * sizeof(*oldbck));
+		kmem_free(oldsl, roundup2(oldnslots * sizeof(int) * 2,
-	kmem_free(oldover, oldnslots * sizeof(*oldover));
+		    COHERENCY_UNIT));
+	}
 	if (oldover != AMAP_TINY_ANON(amap)) {
 		kmem_free(oldover, oldnslots * sizeof(*oldover));
+	}
 #ifdef UVM_AMAP_PPREF
-	if (oldppref && oldppref != PPREF_NONE)
+	if (oldppref && oldppref != PPREF_NONE) {
-		kmem_free(oldppref, oldnslots * sizeof(*oldppref));
+		kmem_free(oldppref, roundup2(oldnslots * sizeof(int),
 		    COHERENCY_UNIT));
+	}
 #endif
 	UVMHIST_LOG(maphist,"<- done (case 3), amap = 0x%#jx, slotneed=%jd",
 	    (uintptr_t)amap, slotneed, 0, 0);
 	return 0;
+}
 /*
  * amap_share_protect: change protection of anons in a shared amap
+ *
  * for shared amaps, given the current data structure layout, it is
  * not possible for us to directly locate all maps referencing the
  * shared anon (to change the protection).  in order to protect data
  * in shared maps we use pmap_page_protect().  [this is useful for IPC
 @@ -699,27 +779,26 @@ amap_wipeout(struct vm_amap *amap)
 	UVMHIST_FUNC("amap_wipeout"); UVMHIST_CALLED(maphist);
 	UVMHIST_LOG(maphist,"(amap=0x%#jx)", (uintptr_t)amap, 0,0,0);
 	KASSERT(mutex_owned(amap->am_lock));
 	KASSERT(amap->am_ref == 0);
 	if (__predict_false(amap->am_flags & AMAP_SWAPOFF)) {
 		/*
 		 * Note: amap_swap_off() will call us again.
 		 */
 		amap_unlock(amap);
 		return;
+	}
 	amap_list_remove(amap);
 	for (lcv = 0 ; lcv < amap->am_nused ; lcv++) {
 		struct vm_anon *anon;
 		u_int slot;
 		slot = amap->am_slots[lcv];
 		anon = amap->am_anon[slot];
 		KASSERT(anon != NULL && anon->an_ref != 0);
 		KASSERT(anon->an_lock == amap->am_lock);
 		UVMHIST_LOG(maphist,"  processing anon 0x%#jx, ref=%jd",
 		    (uintptr_t)anon, anon->an_ref, 0, 0);
 @@ -758,26 +837,27 @@ amap_wipeout(struct vm_amap *amap)
  *     used to limit chunking (e.g. if you have a large space that you
  *     know you are going to need to allocate amaps for, there is no point
  *     in allowing that to be chunked)
  */
 void
 amap_copy(struct vm_map *map, struct vm_map_entry *entry, int flags,
     vaddr_t startva, vaddr_t endva)
+{
 	const int waitf = (flags & AMAP_COPY_NOWAIT) ? UVM_FLAG_NOWAIT : 0;
 	struct vm_amap *amap, *srcamap;
 	struct vm_anon *tofree;
 	u_int slots, lcv;
 	kmutex_t *oldlock;
 	vsize_t len;
 	UVMHIST_FUNC("amap_copy"); UVMHIST_CALLED(maphist);
 	UVMHIST_LOG(maphist, "  (map=%#j, entry=%#j, flags=%jd)",
 		    (uintptr_t)map, (uintptr_t)entry, flags, 0);
 	KASSERT(map != kernel_map);	/* we use nointr pool */
 	srcamap = entry->aref.ar_amap;
 	len = entry->end - entry->start;
 	/*
 	 * Is there an amap to copy?  If not, create one.
 @@ -842,36 +922,49 @@ amap_copy(struct vm_map *map, struct vm_
 	 */
 	if (srcamap->am_ref == 1) {
 		entry->etype &= ~UVM_ET_NEEDSCOPY;
 		UVMHIST_LOG(maphist, "<- done [ref cnt = 1, took it over]",
 , 0, 0, 0);
 		return;
+	}
 	UVMHIST_LOG(maphist,"  amap=%#j, ref=%jd, must copy it",
 	    (uintptr_t)srcamap, srcamap->am_ref, 0, 0);
 	/*
-	 * Allocate a new amap (note: not initialised, no lock set, etc).
+	 * Allocate a new amap (note: not initialised, etc).
 	 */
 	AMAP_B2SLOT(slots, len);
 	amap = amap_alloc1(slots, 0, waitf);
 	if (amap == NULL) {
 		UVMHIST_LOG(maphist, "  amap_alloc1 failed", 0,0,0,0);
 		return;
+	}
 	/*
 	 * Make the new amap share the source amap's lock, and then lock
 	 * both.  We must do this before we set am_nused != 0, otherwise
 	 * amap_swap_off() can become interested in the amap.
 	 */
 	oldlock = amap->am_lock;
 	mutex_enter(&amap_list_lock);
 	amap->am_lock = srcamap->am_lock;
 	mutex_exit(&amap_list_lock);
 	mutex_obj_hold(amap->am_lock);
 	mutex_obj_free(oldlock);
 	amap_lock(srcamap);
 	/*
 	 * Re-check the reference count with the lock held.  If it has
 	 * dropped to one - we can take over the existing map.
 	 */
 	if (srcamap->am_ref == 1) {
 		/* Just take over the existing amap. */
 		entry->etype &= ~UVM_ET_NEEDSCOPY;
 		amap_unlock(srcamap);
 		/* Destroy the new (unused) amap. */
 		amap->am_ref--;
 @@ -910,43 +1003,28 @@ amap_copy(struct vm_map *map, struct vm_
 	KASSERT(srcamap->am_ref > 0);
 	if (srcamap->am_ref == 1 && (srcamap->am_flags & AMAP_SHARED) != 0) {
 		srcamap->am_flags &= ~AMAP_SHARED;
+	}
 	tofree = NULL;
 #ifdef UVM_AMAP_PPREF
 	if (srcamap->am_ppref && srcamap->am_ppref != PPREF_NONE) {
 		amap_pp_adjref(srcamap, entry->aref.ar_pageoff,
 		    len >> PAGE_SHIFT, -1, &tofree);
+	}
 #endif
 	/*
 	 * If we referenced any anons, then share the source amap's lock.
 	 * Otherwise, we have nothing in common, so allocate a new one.
 	 */
 	KASSERT(amap->am_lock == NULL);
 	if (amap->am_nused != 0) {
 		amap->am_lock = srcamap->am_lock;
 		mutex_obj_hold(amap->am_lock);
 	uvm_anon_freelst(srcamap, tofree);
 	if (amap->am_lock == NULL) {
 		amap->am_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
 	amap_list_insert(amap);
 	/*
 	 * Install new amap.
 	 */
 	entry->aref.ar_pageoff = 0;
 	entry->aref.ar_amap = amap;
 	entry->etype &= ~UVM_ET_NEEDSCOPY;
 	UVMHIST_LOG(maphist, "<- done",0, 0, 0, 0);
+}
 /*
  * amap_cow_now: resolve all copy-on-write faults in an amap now for fork(2)
+ *
 @@ -1110,27 +1188,28 @@ amap_splitref(struct vm_aref *origref, s
 	amap_unlock(amap);
+}
 #ifdef UVM_AMAP_PPREF
 /*
  * amap_pp_establish: add a ppref array to an amap, if possible.
+ *
  * => amap should be locked by caller.
  */
 void
 amap_pp_establish(struct vm_amap *amap, vaddr_t offset)
+{
-	const size_t sz = amap->am_maxslot * sizeof(*amap->am_ppref);
+	const size_t sz = roundup2(amap->am_maxslot * sizeof(*amap->am_ppref),
 	    COHERENCY_UNIT);
 	KASSERT(mutex_owned(amap->am_lock));
 	amap->am_ppref = kmem_zalloc(sz, KM_NOSLEEP);
 	if (amap->am_ppref == NULL) {
 		/* Failure - just do not use ppref. */
 		amap->am_ppref = PPREF_NONE;
 		return;
+	}
 	pp_setreflen(amap->am_ppref, 0, 0, offset);
 	pp_setreflen(amap->am_ppref, offset, amap->am_ref,
 	    amap->am_nslot - offset);
+}
 @@ -1311,49 +1390,44 @@ amap_swap_off(int startslot, int endslot
 #if defined(DIAGNOSTIC)
 	memset(&marker_prev, 0, sizeof(marker_prev));
 	memset(&marker_next, 0, sizeof(marker_next));
 #endif /* defined(DIAGNOSTIC) */
 	mutex_enter(&amap_list_lock);
 	for (am = LIST_FIRST(&amap_list); am != NULL && !rv; am = am_next) {
 		int i;
 		LIST_INSERT_BEFORE(am, &marker_prev, am_list);
 		LIST_INSERT_AFTER(am, &marker_next, am_list);
 		/* amap_list_lock prevents the lock pointer from changing. */
 		if (!amap_lock_try(am)) {
-			mutex_exit(&amap_list_lock);
+			(void)kpause("amapswpo", false, 1, &amap_list_lock);
 			preempt();
 			mutex_enter(&amap_list_lock);
 			am_next = LIST_NEXT(&marker_prev, am_list);
 			if (am_next == &marker_next) {
 				am_next = LIST_NEXT(am_next, am_list);
 			} else {
 				KASSERT(LIST_NEXT(am_next, am_list) ==
 				    &marker_next);
+			}
 			LIST_REMOVE(&marker_prev, am_list);
 			LIST_REMOVE(&marker_next, am_list);
 			continue;
+		}
 		mutex_exit(&amap_list_lock);
-		if (am->am_nused <= 0) {
+		/* If am_nused == 0, the amap could be free - careful. */
 			amap_unlock(am);
 			goto next;
 		for (i = 0; i < am->am_nused; i++) {
 			int slot;
 			int swslot;
 			struct vm_anon *anon;
 			slot = am->am_slots[i];
 			anon = am->am_anon[slot];
 			KASSERT(anon->an_lock == am->am_lock);
 			swslot = anon->an_swslot;
 			if (swslot < startslot || endslot <= swslot) {
 				continue;
+			}
 @@ -1369,27 +1443,26 @@ amap_swap_off(int startslot, int endslot
 				am = NULL;
 				break;
+			}
 			if (rv) {
 				break;
+			}
 			i = 0;
+		}
 		if (am) {
 			amap_unlock(am);
+		}
 next:
 		mutex_enter(&amap_list_lock);
 		KASSERT(LIST_NEXT(&marker_prev, am_list) == &marker_next ||
 		    LIST_NEXT(LIST_NEXT(&marker_prev, am_list), am_list) ==
 		    &marker_next);
 		am_next = LIST_NEXT(&marker_next, am_list);
 		LIST_REMOVE(&marker_prev, am_list);
 		LIST_REMOVE(&marker_next, am_list);
+	}
 	mutex_exit(&amap_list_lock);
 	return rv;
+}

 @@ -1,14 +1,14 @@
-/*	$NetBSD: uvm_amap.h,v 1.37 2011/06/12 03:36:02 rmind Exp $	*/
+/*	$NetBSD: uvm_amap.h,v 1.38 2020/01/01 22:01:13 ad Exp $	*/
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -137,43 +137,65 @@ bool		amap_swap_off
  * part 2: amap implementation-specific info
  */
 /*
  * we currently provide an array-based amap implementation.  in this
  * implementation we provide the option of tracking split references
  * so that we don't lose track of references during partial unmaps
  * ... this is enabled with the "UVM_AMAP_PPREF" define.
  */
 #define UVM_AMAP_PPREF		/* track partial references */
 /*
  * for amaps with fewer than UVM_AMAP_TINY slots, we allocate storage
  * directly in vm_amap.  this should reduce pressure on the allocator and on
  * the CPU cache.  on _LP64, the chosen value of 3 sizes the structure at
  * 128 bytes, a multiple of the typical cache line size, which helps us to
  * avoid false sharing on MULTIPROCESSOR.
+ *
  * for amaps with fewer than UVM_AMAP_SMALL slots, anons are stored directly
  * in the vm_amap but slots and backpointers are externally allocated.
  */
 #define UVM_AMAP_TINY	3	/* # of slots in "tiny" amap */
 #ifdef _LP64
 #define UVM_AMAP_SMALL	3*2	/* # of slots if 1/2 external allocation */
 #else
 #define UVM_AMAP_SMALL	3*3	/* # of slots in 1/2 external allocation */
 #endif
 #define	AMAP_TINY_ANON(am)	((struct vm_anon **)&(am)->am_storage[0])
 #define	AMAP_TINY_SLOTS(am)	((int *)&((am)->am_storage[UVM_AMAP_TINY]))
 /*
  * here is the definition of the vm_amap structure for this implementation.
  */
 struct vm_amap {
 	kmutex_t *am_lock;	/* lock [locks all vm_amap fields] */
 	int am_ref;		/* reference count */
 	int am_flags;		/* flags */
 	int am_maxslot;		/* max # of slots allocated */
 	int am_nslot;		/* # of slots currently in map ( <= maxslot) */
 	int am_nused;		/* # of slots currently in use */
 	int *am_slots;		/* contig array of active slots */
 	int *am_bckptr;		/* back pointer array to am_slots */
 	struct vm_anon **am_anon; /* array of anonymous pages */
 #ifdef UVM_AMAP_PPREF
 	int *am_ppref;		/* per page reference count (if !NULL) */
 #endif
 	LIST_ENTRY(vm_amap) am_list;
 	uintptr_t am_storage[UVM_AMAP_SMALL];
 };
 /*
  * note that am_slots, am_bckptr, and am_anon are arrays.   this allows
  * fast lookup of pages based on their virual address at the expense of
  * some extra memory.   in the future we should be smarter about memory
  * usage and fall back to a non-array based implementation on systems
  * that are short of memory (XXXCDC).
+ *
  * the entries in the array are called slots... for example an amap that
  * covers four pages of virtual memory is said to have four slots.   here
  * is an example of the array usage for a four slot amap.   note that only
  * slots one and three have anons assigned to them.  "D/C" means that we