 @@ -1,14 +1,14 @@
-/* $NetBSD: pmap.c,v 1.283 2021/05/30 01:24:19 thorpej Exp $ */
+/* $NetBSD: pmap.c,v 1.284 2021/05/30 01:41:45 thorpej Exp $ */
 /*-
  * Copyright (c) 1998, 1999, 2000, 2001, 2007, 2008, 2020
  * 	The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
  * NASA Ames Research Center, by Andrew Doran and Mindaugas Rasiukevicius,
  * and by Chris G. Demetriou.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
 @@ -125,27 +125,27 @@
  *	this module may delay invalidate or reduced protection
  *	operations until such time as they are actually
  *	necessary.  This module is given full information as
  *	to which processors are currently using which maps,
  *	and to when physical maps must be made correct.
  */
 #include "opt_lockdebug.h"
 #include "opt_sysv.h"
 #include "opt_multiprocessor.h"
 #include <sys/cdefs.h>			/* RCS ID & Copyright macro defns */
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.283 2021/05/30 01:24:19 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.284 2021/05/30 01:41:45 thorpej Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/proc.h>
 #include <sys/malloc.h>
 #include <sys/pool.h>
 #include <sys/buf.h>
 #include <sys/evcnt.h>
 #include <sys/atomic.h>
 #include <sys/cpu.h>
 #include <uvm/uvm.h>
 @@ -1071,63 +1071,48 @@ pmap_tlb_shootnow(const struct pmap_tlb_
 void
 pmap_tlb_shootdown_ipi(struct cpu_info * const ci,
     struct trapframe * const tf __unused)
+{
 	KASSERT(tlb_context != NULL);
 	pmap_tlb_invalidate(tlb_context, ci);
 	if (atomic_and_ulong_nv(&tlb_pending, ~(1UL << ci->ci_cpuid)) == 0) {
 		atomic_store_release(&tlb_context, NULL);
+	}
+}
 #endif /* MULTIPROCESSOR */
 static void
 pmap_tlb_physpage_free(paddr_t const ptpa,
     struct pmap_tlb_context * const tlbctx)
 	struct vm_page * const pg = PHYS_TO_VM_PAGE(ptpa);
 	KASSERT(pg != NULL);
 #ifdef DEBUG
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	KDASSERT(md->pvh_refcnt == 0);
 #endif
 	LIST_INSERT_HEAD(&tlbctx->t_freeptq, pg, pageq.list);
 static __inline void
 pmap_tlb_ptpage_drain(struct pmap_tlb_context * const tlbctx)
+{
 	pmap_pagelist_free(&tlbctx->t_freeptq);
+}
 /*
  * Internal routines
  */
 static void	alpha_protection_init(void);
 static pt_entry_t pmap_remove_mapping(pmap_t, vaddr_t, pt_entry_t *, bool,
 				      pv_entry_t *,
 				      struct pmap_tlb_context *);
 static void	pmap_changebit(struct vm_page *, pt_entry_t, pt_entry_t,
 			       struct pmap_tlb_context *);
 /*
  * PT page management functions.
  */
-static int	pmap_ptpage_alloc(pt_entry_t *, int);
+static int	pmap_ptpage_alloc(pmap_t, pt_entry_t *, int);
-static void	pmap_ptpage_free(pt_entry_t *, struct pmap_tlb_context *);
+static void	pmap_ptpage_free(pmap_t, pt_entry_t *,
 				 struct pmap_tlb_context *);
 static void	pmap_l3pt_delref(pmap_t, vaddr_t, pt_entry_t *,
 		     struct pmap_tlb_context *);
 static void	pmap_l2pt_delref(pmap_t, pt_entry_t *, pt_entry_t *,
 		     struct pmap_tlb_context *);
 static void	pmap_l1pt_delref(pmap_t, pt_entry_t *);
 static void	*pmap_l1pt_alloc(struct pool *, int);
 static void	pmap_l1pt_free(struct pool *, void *);
 static struct pool_allocator pmap_l1pt_allocator = {
 	pmap_l1pt_alloc, pmap_l1pt_free, 0,
 };
 @@ -1388,26 +1373,27 @@ pmap_bootstrap(paddr_t ptaddr, u_int max
 	 * This must block any interrupt from which a TLB shootdown
 	 * could be issued, but must NOT block IPIs.
 	 */
 	mutex_init(&tlb_lock, MUTEX_SPIN, IPL_VM);
 	/*
 	 * Initialize kernel pmap.  Note that all kernel mappings
 	 * have PG_ASM set, so the ASN doesn't really matter for
 	 * the kernel pmap.  Also, since the kernel pmap always
 	 * references kernel_lev1map, it always has an invalid ASN
 	 * generation.
 	 */
 	memset(pmap_kernel(), 0, sizeof(struct pmap));
 	LIST_INIT(&pmap_kernel()->pm_ptpages);
 	atomic_store_relaxed(&pmap_kernel()->pm_count, 1);
 	/* Kernel pmap does not have per-CPU info. */
 	TAILQ_INSERT_TAIL(&pmap_all_pmaps, pmap_kernel(), pm_list);
 	/*
 	 * Set up lwp0's PCB such that the ptbr points to the right place
 	 * and has the kernel pmap's (really unused) ASN.
 	 */
 	pcb = lwp_getpcb(&lwp0);
 	pcb->pcb_hw.apcb_ptbr =
 	    ALPHA_K0SEG_TO_PHYS((vaddr_t)kernel_lev1map) >> PGSHIFT;
 	pcb->pcb_hw.apcb_asn = PMAP_ASN_KERNEL;
 @@ -1571,26 +1557,27 @@ pmap_t
 pmap_create(void)
+{
 	pmap_t pmap;
 	pt_entry_t *lev1map;
 	int i;
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_CREATE))
 		printf("pmap_create()\n");
 #endif
 	pmap = pool_cache_get(&pmap_pmap_cache, PR_WAITOK);
 	memset(pmap, 0, sizeof(*pmap));
 	LIST_INIT(&pmap->pm_ptpages);
 	atomic_store_relaxed(&pmap->pm_count, 1);
  try_again:
 	rw_enter(&pmap_growkernel_lock, RW_READER);
 	lev1map = pool_cache_get(&pmap_l1pt_cache, PR_NOWAIT);
 	if (__predict_false(lev1map == NULL)) {
 		rw_exit(&pmap_growkernel_lock);
 		(void) kpause("pmap_create", false, hz >> 2, NULL);
 		goto try_again;
+	}
 @@ -2095,51 +2082,51 @@ pmap_enter(pmap_t pmap, vaddr_t va, padd
 		KASSERT(va < VM_MAXUSER_ADDRESS);
 		KASSERT(lev1map != kernel_lev1map);
 		/*
 		 * Check to see if the level 1 PTE is valid, and
 		 * allocate a new level 2 page table page if it's not.
 		 * A reference will be added to the level 2 table when
 		 * the level 3 table is created.
 		 */
 		l1pte = pmap_l1pte(lev1map, va);
 		if (pmap_pte_v(l1pte) == 0) {
 			pmap_physpage_addref(l1pte);
-			error = pmap_ptpage_alloc(l1pte, PGU_L2PT);
+			error = pmap_ptpage_alloc(pmap, l1pte, PGU_L2PT);
 			if (error) {
 				pmap_l1pt_delref(pmap, l1pte);
 				if (flags & PMAP_CANFAIL)
 					goto out;
 				panic("pmap_enter: unable to create L2 PT "
 				    "page");
+			}
 #ifdef DEBUG
 			if (pmapdebug & PDB_PTPAGE)
 				printf("pmap_enter: new level 2 table at "
 				    "0x%lx\n", pmap_pte_pa(l1pte));
 #endif
+		}
 		/*
 		 * Check to see if the level 2 PTE is valid, and
 		 * allocate a new level 3 page table page if it's not.
 		 * A reference will be added to the level 3 table when
 		 * the mapping is validated.
 		 */
 		l2pte = pmap_l2pte(lev1map, va, l1pte);
 		if (pmap_pte_v(l2pte) == 0) {
 			pmap_physpage_addref(l2pte);
-			error = pmap_ptpage_alloc(l2pte, PGU_L3PT);
+			error = pmap_ptpage_alloc(pmap, l2pte, PGU_L3PT);
 			if (error) {
 				/* unlocks pmap */
 				pmap_enter_l2pt_delref(pmap, l1pte, l2pte);
 				if (flags & PMAP_CANFAIL) {
 					PMAP_LOCK(pmap);
 					goto out;
+				}
 				panic("pmap_enter: unable to create L3 PT "
 				    "page");
+			}
 #ifdef DEBUG
 			if (pmapdebug & PDB_PTPAGE)
 				printf("pmap_enter: new level 3 table at "
 @@ -3642,70 +3629,81 @@ pmap_l1pt_free(struct pool *pp, void *v)
 	pmap_physpage_free(ALPHA_K0SEG_TO_PHYS((vaddr_t) v));
+}
 /*
  * pmap_ptpage_alloc:
+ *
  *	Allocate a level 2 or level 3 page table page for a user
  *	pmap, and initialize the PTE that references it.
+ *
  *	Note: the pmap must already be locked.
  */
 static int
-pmap_ptpage_alloc(pt_entry_t * const pte, int const usage)
+pmap_ptpage_alloc(pmap_t pmap, pt_entry_t * const pte, int const usage)
+{
 	/*
 	 * Allocate the page table page.
 	 */
 	struct vm_page * const pg = pmap_physpage_alloc(usage);
 	if (__predict_false(pg == NULL)) {
 		return ENOMEM;
+	}
 	LIST_INSERT_HEAD(&pmap->pm_ptpages, pg, pageq.list);
 	/*
 	 * Initialize the referencing PTE.
 	 */
 	const pt_entry_t npte = ((VM_PAGE_TO_PHYS(pg) >> PGSHIFT) << PG_SHIFT) |
 	    PG_V | PG_KRE | PG_KWE | PG_WIRED;
 	atomic_store_relaxed(pte, npte);
 	return (0);
+}
 /*
  * pmap_ptpage_free:
+ *
  *	Free the level 2 or level 3 page table page referenced
  *	be the provided PTE.
+ *
  *	Note: the pmap must already be locked.
  */
 static void
-pmap_ptpage_free(pt_entry_t * const pte, struct pmap_tlb_context * const tlbctx)
+pmap_ptpage_free(pmap_t pmap, pt_entry_t * const pte,
     struct pmap_tlb_context * const tlbctx)
+{
 	/*
 	 * Extract the physical address of the page from the PTE
 	 * and clear the entry.
 	 */
 	const paddr_t ptpa = pmap_pte_pa(pte);
 	atomic_store_relaxed(pte, PG_NV);
 	struct vm_page * const pg = PHYS_TO_VM_PAGE(ptpa);
 	KASSERT(pg != NULL);
 #ifdef DEBUG
 	struct vm_page_md * const md = VM_PAGE_TO_MD(pg);
 	KDASSERT(md->pvh_refcnt == 0);
 	pmap_zero_page(ptpa);
 #endif
 	pmap_tlb_physpage_free(ptpa, tlbctx);
 	LIST_REMOVE(pg, pageq.list);
 	LIST_INSERT_HEAD(&tlbctx->t_freeptq, pg, pageq.list);
+}
 /*
  * pmap_l3pt_delref:
+ *
  *	Delete a reference on a level 3 PT page.  If the reference drops
  *	to zero, free it.
+ *
  *	Note: the pmap must already be locked.
  */
 static void
 pmap_l3pt_delref(pmap_t pmap, vaddr_t va, pt_entry_t *l3pte,
     struct pmap_tlb_context * const tlbctx)
 @@ -3725,27 +3723,27 @@ pmap_l3pt_delref(pmap_t pmap, vaddr_t va
 		/*
 		 * No more mappings; we can free the level 3 table.
 		 */
 #ifdef DEBUG
 		if (pmapdebug & PDB_PTPAGE)
 			printf("pmap_l3pt_delref: freeing level 3 table at "
 			    "0x%lx\n", pmap_pte_pa(l2pte));
 #endif
 		/*
 		 * You can pass NULL if you know the last refrence won't
 		 * be dropped.
 		 */
 		KASSERT(tlbctx != NULL);
-		pmap_ptpage_free(l2pte, tlbctx);
+		pmap_ptpage_free(pmap, l2pte, tlbctx);
 		/*
 		 * We've freed a level 3 table, so we must invalidate
 		 * any now-stale TLB entries for the corresponding VPT
 		 * VA range.  Easiest way to guarantee this is to hit
 		 * all of the user TLB entries.
 		 */
 		pmap_tlb_shootdown_all_user(pmap, PG_V, tlbctx);
 		/*
 		 * We've freed a level 3 table, so delete the reference
 		 * on the level 2 table.
 		 */
 @@ -3776,27 +3774,27 @@ pmap_l2pt_delref(pmap_t pmap, pt_entry_t
 		 * No more mappings in this segment; we can free the
 		 * level 2 table.
 		 */
 #ifdef DEBUG
 		if (pmapdebug & PDB_PTPAGE)
 			printf("pmap_l2pt_delref: freeing level 2 table at "
 			    "0x%lx\n", pmap_pte_pa(l1pte));
 #endif
 		/*
 		 * You can pass NULL if you know the last refrence won't
 		 * be dropped.
 		 */
 		KASSERT(tlbctx != NULL);
-		pmap_ptpage_free(l1pte, tlbctx);
+		pmap_ptpage_free(pmap, l1pte, tlbctx);
 		/*
 		 * We've freed a level 2 table, so we must invalidate
 		 * any now-stale TLB entries for the corresponding VPT
 		 * VA range.  Easiest way to guarantee this is to hit
 		 * all of the user TLB entries.
 		 */
 		pmap_tlb_shootdown_all_user(pmap, PG_V, tlbctx);
 		/*
 		 * We've freed a level 2 table, so delete the reference
 		 * on the level 1 table.
 		 */

 @@ -1,14 +1,14 @@
-/* $NetBSD: pmap.h,v 1.89 2021/05/30 00:34:27 thorpej Exp $ */
+/* $NetBSD: pmap.h,v 1.90 2021/05/30 01:41:45 thorpej Exp $ */
 /*-
  * Copyright (c) 1998, 1999, 2000, 2001, 2007 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
  * NASA Ames Research Center and by Chris G. Demetriou.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
 @@ -140,28 +140,28 @@ struct pmap_percpu {
 	unsigned long		pmc_asngen;	/* ASN generation number */
 	unsigned int		pmc_needisync;	/* CPU needes isync */
 	unsigned int		pmc_pad1;
 	pt_entry_t		*pmc_lev1map;	/* level 1 map */
 	unsigned long		pmc_padN[(COHERENCY_UNIT / 8) - 4];
 };
 struct pmap {	/* pmaps are aligned to COHERENCY_UNIT boundaries */
 		/* pmaps are locked by hashed mutexes */
 	unsigned long		pm_cpus;	/* [ 0] CPUs using pmap */
 	struct pmap_statistics	pm_stats;	/* [ 8] statistics */
 	unsigned int		pm_count;	/* [24] reference count */
 	unsigned int		__pm_spare0;	/* [28] spare field */
-	unsigned long		__pm_spare1;	/* [32] spare field */
+	struct pmap_pagelist	pm_ptpages;	/* [32] list of PT pages */
-	unsigned long		__pm_spare2;	/* [40] spare field */
+	unsigned long		__pm_spare1;	/* [40] spare field */
 	TAILQ_ENTRY(pmap)	pm_list;	/* [48] list of all pmaps */
 	/* -- COHERENCY_UNIT boundary -- */
 	struct pmap_percpu	pm_percpu[];	/* [64] per-CPU data */
 			/*	variable length		*/
 };
 #define	PMAP_SIZEOF(x)							\
 	(ALIGN(offsetof(struct pmap, pm_percpu[(x)])))
 #define	PMAP_ASN_KERNEL		0	/* kernel-reserved ASN */
 #define	PMAP_ASN_FIRST_USER	1	/* first user ASN */
 #define	PMAP_ASNGEN_INVALID	0	/* reserved (invalid) ASN generation */
 #define	PMAP_ASNGEN_INITIAL	1	/* first valid generatation */