 @@ -1,14 +1,14 @@
-/*	$NetBSD: pmap.c,v 1.105 2021/03/12 18:10:00 thorpej Exp $	*/
+/*	$NetBSD: pmap.c,v 1.106 2021/06/27 12:26:33 martin Exp $	*/
 /*-
  * Copyright (c) 2001 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Matt Thomas <matt@3am-software.com> of Allegro Networks, Inc.
+ *
  * Support for PPC64 Bridge mode added by Sanjay Lal <sanjayl@kymasys.com>
  * of Kyma Systems LLC.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
 @@ -53,27 +53,27 @@
  * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.105 2021/03/12 18:10:00 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.106 2021/06/27 12:26:33 martin Exp $");
 #define	PMAP_NOOPNAMES
 #ifdef _KERNEL_OPT
 #include "opt_altivec.h"
 #include "opt_multiprocessor.h"
 #include "opt_pmap.h"
 #include "opt_ppcarch.h"
 #endif
 #include <sys/param.h>
 #include <sys/proc.h>
 #include <sys/pool.h>
 @@ -162,28 +162,27 @@ static u_int mem_cnt, avail_cnt;
 #define pmap_protect		PMAPNAME(protect)
 #define pmap_unwire		PMAPNAME(unwire)
 #define pmap_page_protect	PMAPNAME(page_protect)
 #define pmap_query_bit		PMAPNAME(query_bit)
 #define pmap_clear_bit		PMAPNAME(clear_bit)
 #define pmap_activate		PMAPNAME(activate)
 #define pmap_deactivate		PMAPNAME(deactivate)
 #define pmap_pinit		PMAPNAME(pinit)
 #define pmap_procwr		PMAPNAME(procwr)
 #define pmap_pool		PMAPNAME(pool)
-#define pmap_upvo_pool		PMAPNAME(upvo_pool)
+#define pmap_pvo_pool		PMAPNAME(pvo_pool)
 #define pmap_mpvo_pool		PMAPNAME(mpvo_pool)
 #define pmap_pvo_table		PMAPNAME(pvo_table)
 #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
 #define pmap_pte_print		PMAPNAME(pte_print)
 #define pmap_pteg_check		PMAPNAME(pteg_check)
 #define pmap_print_mmruregs	PMAPNAME(print_mmuregs)
 #define pmap_print_pte		PMAPNAME(print_pte)
 #define pmap_pteg_dist		PMAPNAME(pteg_dist)
 #endif
 #if defined(DEBUG) || defined(PMAPCHECK)
 #define	pmap_pvo_verify		PMAPNAME(pvo_verify)
 #define pmapcheck		PMAPNAME(check)
 #endif
 #if defined(DEBUG) || defined(PMAPDEBUG)
 @@ -323,59 +322,46 @@ struct pvo_entry {
 	((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK)))
 #define	PVO_PTEGIDX_SET(pvo,i)	\
 	((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID))
 #define	PVO_WHERE(pvo,w)	\
 	((pvo)->pvo_vaddr &= ~(PVO_WHERE_MASK << PVO_WHERE_SHFT), \
 	 (pvo)->pvo_vaddr |= ((PVO_ ## w) << PVO_WHERE_SHFT))
 TAILQ_HEAD(pvo_tqhead, pvo_entry);
 struct pvo_tqhead *pmap_pvo_table;	/* pvo entries by ptegroup index */
 static struct pvo_head pmap_pvo_kunmanaged = LIST_HEAD_INITIALIZER(pmap_pvo_kunmanaged);	/* list of unmanaged pages */
 static struct pvo_head pmap_pvo_unmanaged = LIST_HEAD_INITIALIZER(pmap_pvo_unmanaged);	/* list of unmanaged pages */
 struct pool pmap_pool;		/* pool for pmap structures */
-struct pool pmap_upvo_pool;	/* pool for pvo entries for unmanaged pages */
+struct pool pmap_pvo_pool;	/* pool for pvo entries */
 struct pool pmap_mpvo_pool;	/* pool for pvo entries for managed pages */
 /*
  * We keep a cache of unmanaged pages to be used for pvo entries for
  * unmanaged pages.
  */
 struct pvo_page {
 	SIMPLEQ_ENTRY(pvo_page) pvop_link;
 };
 SIMPLEQ_HEAD(pvop_head, pvo_page);
-static struct pvop_head pmap_upvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_upvop_head);
+static struct pvop_head pmap_pvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_pvop_head);
-static struct pvop_head pmap_mpvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_mpvop_head);
+static u_long pmap_pvop_free;
-static u_long pmap_upvop_free;
+static u_long pmap_pvop_maxfree;
 static u_long pmap_upvop_maxfree;
-static u_long pmap_mpvop_free;
+static void *pmap_pool_alloc(struct pool *, int);
-static u_long pmap_mpvop_maxfree;
+static void pmap_pool_free(struct pool *, void *);
-static void *pmap_pool_ualloc(struct pool *, int);
+static struct pool_allocator pmap_pool_allocator = {
-static void *pmap_pool_malloc(struct pool *, int);
+	.pa_alloc = pmap_pool_alloc,
 	.pa_free = pmap_pool_free,
 static void pmap_pool_ufree(struct pool *, void *);
 static void pmap_pool_mfree(struct pool *, void *);
 static struct pool_allocator pmap_pool_mallocator = {
 	.pa_alloc = pmap_pool_malloc,
 	.pa_free = pmap_pool_mfree,
 	.pa_pagesz = 0,
 };
 static struct pool_allocator pmap_pool_uallocator = {
 	.pa_alloc = pmap_pool_ualloc,
 	.pa_free = pmap_pool_ufree,
 	.pa_pagesz = 0,
 };
 #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB)
 void pmap_pte_print(volatile struct pte *);
 void pmap_pteg_check(void);
 void pmap_pteg_dist(void);
 void pmap_print_pte(pmap_t, vaddr_t);
 void pmap_print_mmuregs(void);
 #endif
 #if defined(DEBUG) || defined(PMAPCHECK)
 #ifdef PMAPCHECK
 @@ -1091,34 +1077,28 @@ pmap_real_memory(paddr_t *start, psize_t
 			return;
+		}
+	}
 	*size = 0;
+}
 /*
  * Initialize anything else for pmap handling.
  * Called during vm_init().
  */
 void
 pmap_init(void)
+{
 	pool_init(&pmap_mpvo_pool, sizeof(struct pvo_entry),
 	    sizeof(struct pvo_entry), 0, 0, "pmap_mpvopl",
 	    &pmap_pool_mallocator, IPL_NONE);
 	pool_setlowat(&pmap_mpvo_pool, 1008);
 	pmap_initialized = 1;
+}
 /*
  * How much virtual space does the kernel get?
  */
 void
 pmap_virtual_space(vaddr_t *start, vaddr_t *end)
+{
 	/*
 	 * For now, reserve one segment (minus some overhead) for kernel
 	 * virtual memory
 	 */
 	*start = VM_MIN_KERNEL_ADDRESS;
 @@ -1522,33 +1502,26 @@ pmap_pvo_reclaim(struct pmap *pm)
 		pvoh = &pmap_pvo_table[idx];
 		TAILQ_FOREACH(pvo, pvoh, pvo_olink) {
 			if (!PVO_WIRED_P(pvo)) {
 				pmap_pvo_remove(pvo, -1, NULL);
 				pmap_pvo_reclaim_nextidx = idx;
 				PMAPCOUNT(pvos_reclaimed);
 				return pvo;
+			}
+		}
+	}
 	return NULL;
+}
 static struct pool *
 pmap_pvo_pl(struct pvo_entry *pvo)
 	return PVO_MANAGED_P(pvo) ? &pmap_mpvo_pool : &pmap_upvo_pool;
 /*
  * This returns whether this is the first mapping of a page.
  */
 int
 pmap_pvo_enter(pmap_t pm, struct pool *pl, struct pvo_head *pvo_head,
 	vaddr_t va, paddr_t pa, register_t pte_lo, int flags)
+{
 	struct pvo_entry *pvo;
 	struct pvo_tqhead *pvoh;
 	register_t msr;
 	int ptegidx;
 	int i;
 	int poolflags = PR_NOWAIT;
 @@ -1593,29 +1566,27 @@ pmap_pvo_enter(pmap_t pm, struct pool *p
 			PMAPCOUNT(mappings_replaced);
 			pmap_pvo_remove(pvo, -1, NULL);
 			break;
+		}
+	}
 	/*
 	 * If we aren't overwriting an mapping, try to allocate
 	 */
 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
 	--pmap_pvo_enter_depth;
 #endif
 	pmap_interrupts_restore(msr);
-	if (pvo) {
+	if (pvo == NULL) {
 		KASSERT(pmap_pvo_pl(pvo) == pl);
 	} else {
 		pvo = pool_get(pl, poolflags);
+	}
 	KASSERT((vaddr_t)pvo < VM_MIN_KERNEL_ADDRESS);
 #ifdef DEBUG
 	/*
 	 * Exercise pmap_pvo_reclaim() a little.
 	 */
 	if (pvo && (flags & PMAP_CANFAIL) != 0 &&
 	    pmap_pvo_reclaim_debugctr++ > 0x1000 &&
 	    (pmap_pvo_reclaim_debugctr & 0xff) == 0) {
 		pool_put(pl, pvo);
 		pvo = NULL;
 @@ -1801,27 +1772,27 @@ pmap_pvo_remove(struct pvo_entry *pvo, i
 	TAILQ_REMOVE(&pmap_pvo_table[ptegidx], pvo, pvo_olink);
 	if (pvol) {
 		LIST_INSERT_HEAD(pvol, pvo, pvo_vlink);
+	}
 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK)
 	pmap_pvo_remove_depth--;
 #endif
+}
 void
 pmap_pvo_free(struct pvo_entry *pvo)
+{
-	pool_put(pmap_pvo_pl(pvo), pvo);
+	pool_put(&pmap_pvo_pool, pvo);
+}
 void
 pmap_pvo_free_list(struct pvo_head *pvol)
+{
 	struct pvo_entry *pvo, *npvo;
 	for (pvo = LIST_FIRST(pvol); pvo != NULL; pvo = npvo) {
 		npvo = LIST_NEXT(pvo, pvo_vlink);
 		LIST_REMOVE(pvo, pvo_vlink);
 		pmap_pvo_free(pvo);
+	}
+}
 @@ -1873,41 +1844,38 @@ pvo_clear_exec(struct pvo_entry *pvo)
+	}
 #endif
+}
 /*
  * Insert physical page at pa into the given pmap at virtual address va.
  */
 int
 pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
+{
 	struct mem_region *mp;
 	struct pvo_head *pvo_head;
 	struct vm_page *pg;
 	struct pool *pl;
 	register_t pte_lo;
 	int error;
 	u_int was_exec = 0;
 	PMAP_LOCK();
 	if (__predict_false(!pmap_initialized)) {
 		pvo_head = &pmap_pvo_kunmanaged;
 		pl = &pmap_upvo_pool;
 		pg = NULL;
 		was_exec = PTE_EXEC;
 	} else {
 		pvo_head = pa_to_pvoh(pa, &pg);
 		pl = &pmap_mpvo_pool;
+	}
 	DPRINTFN(ENTER,
 	    "pmap_enter(%p, %#" _PRIxva ", %#" _PRIxpa ", 0x%x, 0x%x):",
 	    pm, va, pa, prot, flags);
 	/*
 	 * If this is a managed page, and it's the first reference to the
 	 * page clear the execness of the page.  Otherwise fetch the execness.
 	 */
 	if (pg != NULL)
 		was_exec = pmap_attr_fetch(pg) & PTE_EXEC;
 @@ -1951,27 +1919,27 @@ pmap_enter(pmap_t pm, vaddr_t va, paddr_
 		pte_lo |= PTE_CHG;
 	if (flags & VM_PROT_ALL)
 		pte_lo |= PTE_REF;
 	/*
 	 * We need to know if this page can be executable
 	 */
 	flags |= (prot & VM_PROT_EXECUTE);
 	/*
 	 * Record mapping for later back-translation and pte spilling.
 	 * This will overwrite any existing mapping.
 	 */
-	error = pmap_pvo_enter(pm, pl, pvo_head, va, pa, pte_lo, flags);
+	error = pmap_pvo_enter(pm, &pmap_pvo_pool, pvo_head, va, pa, pte_lo, flags);
 	/*
 	 * Flush the real page from the instruction cache if this page is
 	 * mapped executable and cacheable and has not been flushed since
 	 * the last time it was modified.
 	 */
 	if (error == 0 &&
             (flags & VM_PROT_EXECUTE) &&
             (pte_lo & PTE_I) == 0 &&
 	    was_exec == 0) {
 		DPRINTFN(ENTER, " %s", "syncicache");
 		PMAPCOUNT(exec_synced);
 		pmap_syncicache(pa, PAGE_SIZE);
 @@ -2031,27 +1999,27 @@ pmap_kenter_pa(vaddr_t va, paddr_t pa, v
 		if (((mfpvr() >> 16) & 0xffff) == MPC603e)
 			pte_lo = PTE_M;
 #endif
+	}
 	if (prot & VM_PROT_WRITE)
 		pte_lo |= PTE_BW;
 	else
 		pte_lo |= PTE_BR;
 	/*
 	 * We don't care about REF/CHG on PVOs on the unmanaged list.
 	 */
-	error = pmap_pvo_enter(pmap_kernel(), &pmap_upvo_pool,
+	error = pmap_pvo_enter(pmap_kernel(), &pmap_pvo_pool,
 	    &pmap_pvo_kunmanaged, va, pa, pte_lo, prot|PMAP_WIRED);
 	if (error != 0)
 		panic("pmap_kenter_pa: failed to enter va %#" _PRIxva " pa %#" _PRIxpa ": %d",
 		      va, pa, error);
 	PMAP_UNLOCK();
+}
 void
 pmap_kremove(vaddr_t va, vsize_t len)
+{
 	if (va < VM_MIN_KERNEL_ADDRESS)
 @@ -2831,108 +2799,71 @@ pmap_pvo_verify(void)
 	for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) {
 		struct pvo_entry *pvo;
 		TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) {
 			if ((uintptr_t) pvo >= SEGMENT_LENGTH)
 				panic("pmap_pvo_verify: invalid pvo %p "
 				    "on list %#x", pvo, ptegidx);
 			pmap_pvo_check(pvo);
+		}
+	}
 	splx(s);
+}
 #endif /* PMAPCHECK */
 void *
-pmap_pool_ualloc(struct pool *pp, int flags)
+pmap_pool_alloc(struct pool *pp, int flags)
+{
 	struct pvo_page *pvop;
 	struct vm_page *pg;
 	if (uvm.page_init_done != true) {
 		return (void *) uvm_pageboot_alloc(PAGE_SIZE);
+	}
 	PMAP_LOCK();
-	pvop = SIMPLEQ_FIRST(&pmap_upvop_head);
+	pvop = SIMPLEQ_FIRST(&pmap_pvop_head);
 	if (pvop != NULL) {
 		pmap_upvop_free--;
 		SIMPLEQ_REMOVE_HEAD(&pmap_upvop_head, pvop_link);
 		PMAP_UNLOCK();
 		return pvop;
 	PMAP_UNLOCK();
 	return pmap_pool_malloc(pp, flags);
 void *
 pmap_pool_malloc(struct pool *pp, int flags)
 	struct pvo_page *pvop;
 	struct vm_page *pg;
 	PMAP_LOCK();
 	pvop = SIMPLEQ_FIRST(&pmap_mpvop_head);
 	if (pvop != NULL) {
-		pmap_mpvop_free--;
+		pmap_pvop_free--;
-		SIMPLEQ_REMOVE_HEAD(&pmap_mpvop_head, pvop_link);
+		SIMPLEQ_REMOVE_HEAD(&pmap_pvop_head, pvop_link);
 		PMAP_UNLOCK();
 		return pvop;
+	}
 	PMAP_UNLOCK();
  again:
 	pg = uvm_pagealloc_strat(NULL, 0, NULL, UVM_PGA_USERESERVE,
 	    UVM_PGA_STRAT_ONLY, VM_FREELIST_FIRST256);
 	if (__predict_false(pg == NULL)) {
 		if (flags & PR_WAITOK) {
 			uvm_wait("plpg");
 			goto again;
 		} else {
 			return (0);
+		}
+	}
 	KDASSERT(VM_PAGE_TO_PHYS(pg) == (uintptr_t)VM_PAGE_TO_PHYS(pg));
 	return (void *)(uintptr_t) VM_PAGE_TO_PHYS(pg);
+}
 void
-pmap_pool_ufree(struct pool *pp, void *va)
+pmap_pool_free(struct pool *pp, void *va)
 	struct pvo_page *pvop;
 #if 0
 	if (PHYS_TO_VM_PAGE((paddr_t) va) != NULL) {
 		pmap_pool_mfree(va, size, tag);
 		return;
 #endif
 	PMAP_LOCK();
 	pvop = va;
 	SIMPLEQ_INSERT_HEAD(&pmap_upvop_head, pvop, pvop_link);
 	pmap_upvop_free++;
 	if (pmap_upvop_free > pmap_upvop_maxfree)
 		pmap_upvop_maxfree = pmap_upvop_free;
 	PMAP_UNLOCK();
 void
 pmap_pool_mfree(struct pool *pp, void *va)
+{
 	struct pvo_page *pvop;
 	PMAP_LOCK();
 	pvop = va;
-	SIMPLEQ_INSERT_HEAD(&pmap_mpvop_head, pvop, pvop_link);
+	SIMPLEQ_INSERT_HEAD(&pmap_pvop_head, pvop, pvop_link);
-	pmap_mpvop_free++;
+	pmap_pvop_free++;
-	if (pmap_mpvop_free > pmap_mpvop_maxfree)
+	if (pmap_pvop_free > pmap_pvop_maxfree)
-		pmap_mpvop_maxfree = pmap_mpvop_free;
+		pmap_pvop_maxfree = pmap_pvop_free;
 	PMAP_UNLOCK();
 #if 0
 	uvm_pagefree(PHYS_TO_VM_PAGE((paddr_t) va));
 #endif
+}
 /*
  * This routine in bootstraping to steal to-be-managed memory (which will
  * then be unmanaged).  We use it to grab from the first 256MB for our
  * pmap needs and above 256MB for other stuff.
  */
 vaddr_t
 pmap_steal_memory(vsize_t vsize, vaddr_t *vstartp, vaddr_t *vendp)
 @@ -3457,34 +3388,34 @@ pmap_bootstrap1(paddr_t kernelstart, pad
 			    uvm_physseg_get_avail_start(bank);
 			printf("pmap_bootstrap: vm_physmem[%d]=%#" _PRIxpa "-%#" _PRIxpa "/%#" _PRIxpa "\n",
 			    bank,
 			    ptoa(uvm_physseg_get_avail_start(bank)),
 			    ptoa(uvm_physseg_get_avail_end(bank)),
 			    ptoa(uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank)));
+		}
 		format_bytes(pbuf, sizeof(pbuf), ptoa((u_int64_t) cnt));
 		printf("pmap_bootstrap: UVM memory = %s (%u pages)\n",
 		    pbuf, cnt);
+	}
 #endif
-	pool_init(&pmap_upvo_pool, sizeof(struct pvo_entry),
+	pool_init(&pmap_pvo_pool, sizeof(struct pvo_entry),
-	    sizeof(struct pvo_entry), 0, 0, "pmap_upvopl",
+	    sizeof(struct pvo_entry), 0, 0, "pmap_pvopl",
-	    &pmap_pool_uallocator, IPL_VM);
+	    &pmap_pool_allocator, IPL_VM);
-	pool_setlowat(&pmap_upvo_pool, 252);
+	pool_setlowat(&pmap_pvo_pool, 1008);
 	pool_init(&pmap_pool, sizeof(struct pmap),
-	    sizeof(void *), 0, 0, "pmap_pl", &pmap_pool_uallocator,
+	    sizeof(void *), 0, 0, "pmap_pl", &pmap_pool_allocator,
 	    IPL_NONE);
 #if defined(PMAP_NEED_MAPKERNEL)
+	{
 		struct pmap *pm = pmap_kernel();
 #if defined(PMAP_NEED_FULL_MAPKERNEL)
 		extern int etext[], kernel_text[];
 		vaddr_t va, va_etext = (paddr_t) etext;
 #endif
 		paddr_t pa, pa_end;
 		register_t sr;
 		struct pte pt;
 		unsigned int ptegidx;