 @@ -1,14 +1,14 @@
-/* $NetBSD: pmap.c,v 1.287 2021/05/30 06:41:19 thorpej Exp $ */
+/* $NetBSD: pmap.c,v 1.288 2021/05/30 13:34:21 thorpej Exp $ */
 /*-
  * Copyright (c) 1998, 1999, 2000, 2001, 2007, 2008, 2020
  * 	The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
  * NASA Ames Research Center, by Andrew Doran and Mindaugas Rasiukevicius,
  * and by Chris G. Demetriou.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
 @@ -125,27 +125,27 @@
  *	this module may delay invalidate or reduced protection
  *	operations until such time as they are actually
  *	necessary.  This module is given full information as
  *	to which processors are currently using which maps,
  *	and to when physical maps must be made correct.
  */
 #include "opt_lockdebug.h"
 #include "opt_sysv.h"
 #include "opt_multiprocessor.h"
 #include <sys/cdefs.h>			/* RCS ID & Copyright macro defns */
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.287 2021/05/30 06:41:19 thorpej Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.288 2021/05/30 13:34:21 thorpej Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/proc.h>
 #include <sys/malloc.h>
 #include <sys/pool.h>
 #include <sys/buf.h>
 #include <sys/evcnt.h>
 #include <sys/atomic.h>
 #include <sys/cpu.h>
 #include <uvm/uvm.h>
 @@ -1374,26 +1374,27 @@ pmap_bootstrap(paddr_t ptaddr, u_int max
 	 * could be issued, but must NOT block IPIs.
 	 */
 	mutex_init(&tlb_lock, MUTEX_SPIN, IPL_VM);
 	/*
 	 * Initialize kernel pmap.  Note that all kernel mappings
 	 * have PG_ASM set, so the ASN doesn't really matter for
 	 * the kernel pmap.  Also, since the kernel pmap always
 	 * references kernel_lev1map, it always has an invalid ASN
 	 * generation.
 	 */
 	memset(pmap_kernel(), 0, sizeof(struct pmap));
 	LIST_INIT(&pmap_kernel()->pm_ptpages);
 	LIST_INIT(&pmap_kernel()->pm_pvents);
 	atomic_store_relaxed(&pmap_kernel()->pm_count, 1);
 	/* Kernel pmap does not have per-CPU info. */
 	TAILQ_INSERT_TAIL(&pmap_all_pmaps, pmap_kernel(), pm_list);
 	/*
 	 * Set up lwp0's PCB such that the ptbr points to the right place
 	 * and has the kernel pmap's (really unused) ASN.
 	 */
 	pcb = lwp_getpcb(&lwp0);
 	pcb->pcb_hw.apcb_ptbr =
 	    ALPHA_K0SEG_TO_PHYS((vaddr_t)kernel_lev1map) >> PGSHIFT;
 	pcb->pcb_hw.apcb_asn = PMAP_ASN_KERNEL;
 @@ -1558,26 +1559,27 @@ pmap_create(void)
+{
 	pmap_t pmap;
 	pt_entry_t *lev1map;
 	int i;
 #ifdef DEBUG
 	if (pmapdebug & (PDB_FOLLOW|PDB_CREATE))
 		printf("pmap_create()\n");
 #endif
 	pmap = pool_cache_get(&pmap_pmap_cache, PR_WAITOK);
 	memset(pmap, 0, sizeof(*pmap));
 	LIST_INIT(&pmap->pm_ptpages);
 	LIST_INIT(&pmap->pm_pvents);
 	atomic_store_relaxed(&pmap->pm_count, 1);
  try_again:
 	rw_enter(&pmap_growkernel_lock, RW_READER);
 	lev1map = pool_cache_get(&pmap_l1pt_cache, PR_NOWAIT);
 	if (__predict_false(lev1map == NULL)) {
 		rw_exit(&pmap_growkernel_lock);
 		(void) kpause("pmap_create", false, hz >> 2, NULL);
 		goto try_again;
+	}
 @@ -3268,26 +3270,27 @@ pmap_pv_enter(pmap_t pmap, struct vm_pag
 			printf("pmap = %p, va = 0x%lx\n", pmap, va);
 			panic("pmap_pv_enter: already in pv table");
+		}
+	}
+    }
 #endif
 	/*
 	 * ...and put it in the list.
 	 */
 	uintptr_t const attrs = md->pvh_listx & PGA_ATTRS;
 	newpv->pv_next = (struct pv_entry *)(md->pvh_listx & ~PGA_ATTRS);
 	md->pvh_listx = (uintptr_t)newpv | attrs;
 	LIST_INSERT_HEAD(&pmap->pm_pvents, newpv, pv_link);
 	if (dolock) {
 		mutex_exit(lock);
+	}
 	return 0;
+}
 /*
  * pmap_pv_remove:
+ *
  *	Remove a physical->virtual entry from the pv_table.
  */
 @@ -3306,28 +3309,35 @@ pmap_pv_remove(pmap_t pmap, struct vm_pa
 		lock = NULL; /* XXX stupid gcc */
+	}
 	/*
 	 * Find the entry to remove.
 	 */
 	for (pvp = (struct pv_entry **)&md->pvh_listx, pv = VM_MDPAGE_PVS(pg);
 	     pv != NULL; pvp = &pv->pv_next, pv = *pvp)
 		if (pmap == pv->pv_pmap && va == pv->pv_va)
 			break;
 	KASSERT(pv != NULL);
 	/*
 	 * The page attributes are in the lower 2 bits of the first
 	 * PV entry pointer.  Rather than comparing the pointer address
 	 * and branching, we just always preserve what might be there
 	 * (either attribute bits or zero bits).
 	 */
 	*pvp = (pv_entry_t)((uintptr_t)pv->pv_next |
 			    (((uintptr_t)*pvp) & PGA_ATTRS));
 	LIST_REMOVE(pv, pv_link);
 	if (dolock) {
 		mutex_exit(lock);
+	}
 	if (opvp != NULL)
 		*opvp = pv;
 	else
 		pmap_pv_free(pv);
+}
 /*
  * pmap_pv_page_alloc:

 @@ -1,14 +1,14 @@
-/* $NetBSD: pmap.h,v 1.93 2021/05/30 06:41:19 thorpej Exp $ */
+/* $NetBSD: pmap.h,v 1.94 2021/05/30 13:34:21 thorpej Exp $ */
 /*-
  * Copyright (c) 1998, 1999, 2000, 2001, 2007 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
  * NASA Ames Research Center and by Chris G. Demetriou.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
 @@ -141,47 +141,48 @@ struct pmap_percpu {
 	unsigned int		pmc_needisync;	/* CPU needes isync */
 	unsigned int		pmc_pad1;
 	pt_entry_t		*pmc_lev1map;	/* level 1 map */
 	unsigned long		pmc_padN[(COHERENCY_UNIT / 8) - 4];
 };
 struct pmap {	/* pmaps are aligned to COHERENCY_UNIT boundaries */
 		/* pmaps are locked by hashed mutexes */
 	unsigned long		pm_cpus;	/* [ 0] CPUs using pmap */
 	struct pmap_statistics	pm_stats;	/* [ 8] statistics */
 	unsigned int		pm_count;	/* [24] reference count */
 	unsigned int		__pm_spare0;	/* [28] spare field */
 	struct pmap_pagelist	pm_ptpages;	/* [32] list of PT pages */
-	unsigned long		__pm_spare1;	/* [40] spare field */
+	LIST_HEAD(, pv_entry)	pm_pvents;	/* [40] list of PV entries */
 	TAILQ_ENTRY(pmap)	pm_list;	/* [48] list of all pmaps */
 	/* -- COHERENCY_UNIT boundary -- */
 	struct pmap_percpu	pm_percpu[];	/* [64] per-CPU data */
 			/*	variable length		*/
 };
 #define	PMAP_SIZEOF(x)							\
 	(ALIGN(offsetof(struct pmap, pm_percpu[(x)])))
 #define	PMAP_ASN_KERNEL		0	/* kernel-reserved ASN */
 #define	PMAP_ASN_FIRST_USER	1	/* first user ASN */
 #define	PMAP_ASNGEN_INVALID	0	/* reserved (invalid) ASN generation */
 #define	PMAP_ASNGEN_INITIAL	1	/* first valid generatation */
 /*
  * For each struct vm_page, there is a list of all currently valid virtual
  * mappings of that page.  An entry is a pv_entry_t, the list is pv_table.
  */
 typedef struct pv_entry {
-	struct pv_entry	*pv_next;	/* next pv_entry on list */
+	struct pv_entry	*pv_next;	/* next pv_entry on page list */
 	LIST_ENTRY(pv_entry) pv_link;	/* link on owning pmap's list */
 	struct pmap	*pv_pmap;	/* pmap where mapping lies */
 	vaddr_t		pv_va;		/* virtual address for mapping */
 	pt_entry_t	*pv_pte;	/* PTE that maps the VA */
 } *pv_entry_t;
 /* attrs in pvh_listx */
 #define	PGA_MODIFIED		0x01UL		/* modified */
 #define	PGA_REFERENCED		0x02UL		/* referenced */
 #define	PGA_ATTRS		(PGA_MODIFIED | PGA_REFERENCED)
 /* pvh_usage */
 #define	PGU_NORMAL		0		/* free or normal use */
 #define	PGU_PVENT		1		/* PV entries */