 @@ -1,14 +1,14 @@
-/* $NetBSD: pmap.c,v 1.281 2021/05/29 23:27:22 thorpej Exp $ */
+/* $NetBSD: pmap.c,v 1.282 2021/05/30 00:34:27 thorpej Exp $ */
 /*-
  * Copyright (c) 1998, 1999, 2000, 2001, 2007, 2008, 2020
  * 	The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
  * NASA Ames Research Center, by Andrew Doran and Mindaugas Rasiukevicius,
  * and by Chris G. Demetriou.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
 @@ -125,27 +125,27 @@
  *	this module may delay invalidate or reduced protection
  *	operations until such time as they are actually
  *	necessary.  This module is given full information as
  *	to which processors are currently using which maps,
  *	and to when physical maps must be made correct.
  */
 #include "opt_lockdebug.h"
 #include "opt_sysv.h"
 #include "opt_multiprocessor.h"
 #include <sys/cdefs.h>			/* RCS ID & Copyright macro defns */
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.281 2021/05/29 23:27:22 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.282 2021/05/30 00:34:27 thorpej Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/proc.h>
 #include <sys/malloc.h>
 #include <sys/pool.h>
 #include <sys/buf.h>
 #include <sys/evcnt.h>
 #include <sys/atomic.h>
 #include <sys/cpu.h>
 #include <uvm/uvm.h>
 @@ -422,26 +422,41 @@ pmap_activation_lock(pmap_t const pmap)
 #define	PMAP_UNLOCK(pmap)	mutex_exit(pmap_pmap_lock(pmap))
 #define	PMAP_ACT_LOCK(pmap)	mutex_spin_enter(pmap_activation_lock(pmap))
 #define	PMAP_ACT_TRYLOCK(pmap)	mutex_tryenter(pmap_activation_lock(pmap))
 #define	PMAP_ACT_UNLOCK(pmap)	mutex_spin_exit(pmap_activation_lock(pmap))
 #if defined(MULTIPROCESSOR)
 #define	pmap_all_cpus()		cpus_running
 #else
 #define	pmap_all_cpus()		~0UL
 #endif /* MULTIPROCESSOR */
 /*
  * Generic routine for freeing pages on a pmap_pagelist back to
  * the system.
  */
 static void
 pmap_pagelist_free(struct pmap_pagelist * const list)
+{
 	struct vm_page *pg;
 	while ((pg = LIST_FIRST(list)) != NULL) {
 		LIST_REMOVE(pg, pageq.list);
 		uvm_pagefree(pg);
+	}
+}
 /*
  * TLB management.
+ *
  * TLB invalidations need to be performed on local and remote CPUs
  * whenever parts of the PTE that the hardware or PALcode understands
  * changes.  In order amortize the cost of these operations, we will
  * queue up to 8 addresses to invalidate in a batch.  Any more than
  * that, and we will hit the entire TLB.
+ *
  * Some things that add complexity:
+ *
  * ==> ASNs. A CPU may have valid TLB entries for other than the current
  *     address spaace.  We can only invalidate TLB entries for the current
  *     address space, so when asked to invalidate a VA for the non-current
 @@ -516,27 +531,27 @@ pmap_activation_lock(pmap_t const pmap)
 #define	TLB_CTX_INC_COUNT(ctx)	 (ctx)->t_addrdata[0]++
 #define	TLB_CTX_SET_ALLVA(ctx)	 (ctx)->t_addrdata[0] |= TLB_CTX_ALLVA
 #define	TLB_CTX_FLAGS(ctx)	((ctx)->t_addrdata[1] & PAGE_MASK)
 #define	TLB_CTX_SET_FLAG(ctx, f) (ctx)->t_addrdata[1] |= (f)
 #define	TLB_CTX_VA(ctx, i)	((ctx)->t_addrdata[(i)] & ~PAGE_MASK)
 #define	TLB_CTX_SETVA(ctx, i, va)					\
 	(ctx)->t_addrdata[(i)] = (va) | ((ctx)->t_addrdata[(i)] & PAGE_MASK)
 struct pmap_tlb_context {
 	uintptr_t	t_addrdata[TLB_CTX_MAXVA];
 	pmap_t		t_pmap;
-	LIST_HEAD(, vm_page) t_freeptq;
+	struct pmap_pagelist t_freeptq;
 };
 static struct {
 	kmutex_t	lock;
 	struct evcnt	events;
 } tlb_shootdown __cacheline_aligned;
 #define	tlb_lock	tlb_shootdown.lock
 #define	tlb_evcnt	tlb_shootdown.events
 #if defined(MULTIPROCESSOR)
 static const struct pmap_tlb_context *tlb_context __cacheline_aligned;
 static unsigned long tlb_pending __cacheline_aligned;
 #endif /* MULTIPROCESSOR */
 @@ -1072,35 +1087,30 @@ pmap_tlb_physpage_free(paddr_t const ptp
+{
 	struct vm_page * const pg = PHYS_TO_VM_PAGE(ptpa);
 	KASSERT(pg != NULL);
 #ifdef DEBUG
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	KDASSERT(md->pvh_refcnt == 0);
 #endif
 	LIST_INSERT_HEAD(&tlbctx->t_freeptq, pg, pageq.list);
+}
-static void
+static __inline void
 pmap_tlb_ptpage_drain(struct pmap_tlb_context * const tlbctx)
+{
-	struct vm_page *pg;
+	pmap_pagelist_free(&tlbctx->t_freeptq);
 	while ((pg = LIST_FIRST(&tlbctx->t_freeptq)) != NULL) {
 		LIST_REMOVE(pg, pageq.list);
 		uvm_pagefree(pg);
+}
 /*
  * Internal routines
  */
 static void	alpha_protection_init(void);
 static pt_entry_t pmap_remove_mapping(pmap_t, vaddr_t, pt_entry_t *, bool,
 				      pv_entry_t *,
 				      struct pmap_tlb_context *);
 static void	pmap_changebit(struct vm_page *, pt_entry_t, pt_entry_t,
 			       struct pmap_tlb_context *);
 /*
 @@ -1710,27 +1720,28 @@ pmap_remove_internal(pmap_t pmap, vaddr_
 			l3pte = PMAP_KERNEL_PTE(sva);
 			if (pmap_pte_v(l3pte)) {
 				pte_bits = pmap_remove_mapping(pmap, sva,
 				    l3pte, true, NULL, tlbctx);
 				pmap_tlb_shootdown(pmap, sva, pte_bits,
 				    tlbctx);
+			}
 			sva += PAGE_SIZE;
+		}
 		PMAP_MAP_TO_HEAD_UNLOCK();
 		PMAP_UNLOCK(pmap);
 		pmap_tlb_shootnow(tlbctx);
-		pmap_tlb_ptpage_drain(tlbctx);
+		/* kernel PT pages are never freed. */
 		KASSERT(LIST_EMPTY(&tlbctx->t_freeptq));
 		TLB_COUNT(reason_remove_kernel);
 		return;
+	}
 	pt_entry_t * const lev1map = pmap_lev1map(pmap);
 	KASSERT(sva < VM_MAXUSER_ADDRESS);
 	KASSERT(eva <= VM_MAXUSER_ADDRESS);
 	KASSERT(lev1map != kernel_lev1map);
 	PMAP_MAP_TO_HEAD_LOCK();
 	PMAP_LOCK(pmap);

 @@ -1,14 +1,14 @@
-/* $NetBSD: pmap.h,v 1.88 2021/05/29 23:27:22 thorpej Exp $ */
+/* $NetBSD: pmap.h,v 1.89 2021/05/30 00:34:27 thorpej Exp $ */
 /*-
  * Copyright (c) 1998, 1999, 2000, 2001, 2007 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
  * NASA Ames Research Center and by Chris G. Demetriou.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
 @@ -121,26 +121,29 @@
  * If we ever support processor numbers higher than 63, we'll have to
  * rethink the CPU mask.
+ *
  * Note pm_asn and pm_asngen are arrays allocated in pmap_create().
  * Their size is based on the PCS count from the HWRPB, and indexed
  * by processor ID (from `whami').  This is all padded to COHERENCY_UNIT
  * to avoid false sharing.
+ *
  * The kernel pmap is a special case; since the kernel uses only ASM
  * mappings and uses a reserved ASN to keep the TLB clean, we don't
  * allocate any ASN info for the kernel pmap at all.
  * arrays which hold enough for ALPHA_MAXPROCS.
  */
 LIST_HEAD(pmap_pagelist, vm_page);
 struct pmap_percpu {
 	unsigned int		pmc_asn;	/* address space number */
 	unsigned int		pmc_pad0;
 	unsigned long		pmc_asngen;	/* ASN generation number */
 	unsigned int		pmc_needisync;	/* CPU needes isync */
 	unsigned int		pmc_pad1;
 	pt_entry_t		*pmc_lev1map;	/* level 1 map */
 	unsigned long		pmc_padN[(COHERENCY_UNIT / 8) - 4];
 };
 struct pmap {	/* pmaps are aligned to COHERENCY_UNIT boundaries */
 		/* pmaps are locked by hashed mutexes */
 	unsigned long		pm_cpus;	/* [ 0] CPUs using pmap */