Fri Mar 4 08:11:48 2022 UTC ()
Rmmove an incorrect KASSERT.


(skrll)
diff -r1.51 -r1.52 src/sys/uvm/pmap/pmap_tlb.c
cvs diff -r1.51 -r1.52 src/sys/uvm/pmap/pmap_tlb.c (switch to unified diff)

--- src/sys/uvm/pmap/pmap_tlb.c 2022/01/02 16:03:30 1.51
+++ src/sys/uvm/pmap/pmap_tlb.c 2022/03/04 08:11:48 1.52
@@ -1,1093 +1,1089 @@ @@ -1,1093 +1,1089 @@
1/* $NetBSD: pmap_tlb.c,v 1.51 2022/01/02 16:03:30 christos Exp $ */ 1/* $NetBSD: pmap_tlb.c,v 1.52 2022/03/04 08:11:48 skrll Exp $ */
2 2
3/*- 3/*-
4 * Copyright (c) 2010 The NetBSD Foundation, Inc. 4 * Copyright (c) 2010 The NetBSD Foundation, Inc.
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * This code is derived from software contributed to The NetBSD Foundation 7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Matt Thomas at 3am Software Foundry. 8 * by Matt Thomas at 3am Software Foundry.
9 * 9 *
10 * Redistribution and use in source and binary forms, with or without 10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions 11 * modification, are permitted provided that the following conditions
12 * are met: 12 * are met:
13 * 1. Redistributions of source code must retain the above copyright 13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer. 14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright 15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the 16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution. 17 * documentation and/or other materials provided with the distribution.
18 * 18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE. 29 * POSSIBILITY OF SUCH DAMAGE.
30 */ 30 */
31 31
32#include <sys/cdefs.h> 32#include <sys/cdefs.h>
33 33
34__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.51 2022/01/02 16:03:30 christos Exp $"); 34__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.52 2022/03/04 08:11:48 skrll Exp $");
35 35
36/* 36/*
37 * Manages address spaces in a TLB. 37 * Manages address spaces in a TLB.
38 * 38 *
39 * Normally there is a 1:1 mapping between a TLB and a CPU. However, some 39 * Normally there is a 1:1 mapping between a TLB and a CPU. However, some
40 * implementations may share a TLB between multiple CPUs (really CPU thread 40 * implementations may share a TLB between multiple CPUs (really CPU thread
41 * contexts). This requires the TLB abstraction to be separated from the 41 * contexts). This requires the TLB abstraction to be separated from the
42 * CPU abstraction. It also requires that the TLB be locked while doing 42 * CPU abstraction. It also requires that the TLB be locked while doing
43 * TLB activities. 43 * TLB activities.
44 * 44 *
45 * For each TLB, we track the ASIDs in use in a bitmap and a list of pmaps 45 * For each TLB, we track the ASIDs in use in a bitmap and a list of pmaps
46 * that have a valid ASID. 46 * that have a valid ASID.
47 * 47 *
48 * We allocate ASIDs in increasing order until we have exhausted the supply, 48 * We allocate ASIDs in increasing order until we have exhausted the supply,
49 * then reinitialize the ASID space, and start allocating again at 1. When 49 * then reinitialize the ASID space, and start allocating again at 1. When
50 * allocating from the ASID bitmap, we skip any ASID who has a corresponding 50 * allocating from the ASID bitmap, we skip any ASID who has a corresponding
51 * bit set in the ASID bitmap. Eventually this causes the ASID bitmap to fill 51 * bit set in the ASID bitmap. Eventually this causes the ASID bitmap to fill
52 * and, when completely filled, a reinitialization of the ASID space. 52 * and, when completely filled, a reinitialization of the ASID space.
53 * 53 *
54 * To reinitialize the ASID space, the ASID bitmap is reset and then the ASIDs 54 * To reinitialize the ASID space, the ASID bitmap is reset and then the ASIDs
55 * of non-kernel TLB entries get recorded in the ASID bitmap. If the entries 55 * of non-kernel TLB entries get recorded in the ASID bitmap. If the entries
56 * in TLB consume more than half of the ASID space, all ASIDs are invalidated, 56 * in TLB consume more than half of the ASID space, all ASIDs are invalidated,
57 * the ASID bitmap is recleared, and the list of pmaps is emptied. Otherwise, 57 * the ASID bitmap is recleared, and the list of pmaps is emptied. Otherwise,
58 * (the normal case), any ASID present in the TLB (even those which are no 58 * (the normal case), any ASID present in the TLB (even those which are no
59 * longer used by a pmap) will remain active (allocated) and all other ASIDs 59 * longer used by a pmap) will remain active (allocated) and all other ASIDs
60 * will be freed. If the size of the TLB is much smaller than the ASID space, 60 * will be freed. If the size of the TLB is much smaller than the ASID space,
61 * this algorithm completely avoids TLB invalidation. 61 * this algorithm completely avoids TLB invalidation.
62 * 62 *
63 * For multiprocessors, we also have to deal TLB invalidation requests from 63 * For multiprocessors, we also have to deal TLB invalidation requests from
64 * other CPUs, some of which are dealt with the reinitialization of the ASID 64 * other CPUs, some of which are dealt with the reinitialization of the ASID
65 * space. Whereas above we keep the ASIDs of those pmaps which have active 65 * space. Whereas above we keep the ASIDs of those pmaps which have active
66 * TLB entries, this type of reinitialization preserves the ASIDs of any 66 * TLB entries, this type of reinitialization preserves the ASIDs of any
67 * "onproc" user pmap and all other ASIDs will be freed. We must do this 67 * "onproc" user pmap and all other ASIDs will be freed. We must do this
68 * since we can't change the current ASID. 68 * since we can't change the current ASID.
69 * 69 *
70 * Each pmap has two bitmaps: pm_active and pm_onproc. Each bit in pm_active 70 * Each pmap has two bitmaps: pm_active and pm_onproc. Each bit in pm_active
71 * indicates whether that pmap has an allocated ASID for a CPU. Each bit in 71 * indicates whether that pmap has an allocated ASID for a CPU. Each bit in
72 * pm_onproc indicates that the pmap's ASID is in use, i.e. a CPU has it in its 72 * pm_onproc indicates that the pmap's ASID is in use, i.e. a CPU has it in its
73 * "current ASID" field, e.g. the ASID field of the COP 0 register EntryHi for 73 * "current ASID" field, e.g. the ASID field of the COP 0 register EntryHi for
74 * MIPS, or the ASID field of TTBR0 for AA64. The bit number used in these 74 * MIPS, or the ASID field of TTBR0 for AA64. The bit number used in these
75 * bitmaps comes from the CPU's cpu_index(). Even though these bitmaps contain 75 * bitmaps comes from the CPU's cpu_index(). Even though these bitmaps contain
76 * the bits for all CPUs, the bits that correspond to the bits belonging to 76 * the bits for all CPUs, the bits that correspond to the bits belonging to
77 * the CPUs sharing a TLB can only be manipulated while holding that TLB's 77 * the CPUs sharing a TLB can only be manipulated while holding that TLB's
78 * lock. Atomic ops must be used to update them since multiple CPUs may be 78 * lock. Atomic ops must be used to update them since multiple CPUs may be
79 * changing different sets of bits at same time but these sets never overlap. 79 * changing different sets of bits at same time but these sets never overlap.
80 * 80 *
81 * When a change to the local TLB may require a change in the TLB's of other 81 * When a change to the local TLB may require a change in the TLB's of other
82 * CPUs, we try to avoid sending an IPI if at all possible. For instance, if 82 * CPUs, we try to avoid sending an IPI if at all possible. For instance, if
83 * we are updating a PTE and that PTE previously was invalid and therefore 83 * we are updating a PTE and that PTE previously was invalid and therefore
84 * couldn't support an active mapping, there's no need for an IPI since there 84 * couldn't support an active mapping, there's no need for an IPI since there
85 * can't be a TLB entry to invalidate. The other case is when we change a PTE 85 * can't be a TLB entry to invalidate. The other case is when we change a PTE
86 * to be modified we just update the local TLB. If another TLB has a stale 86 * to be modified we just update the local TLB. If another TLB has a stale
87 * entry, a TLB MOD exception will be raised and that will cause the local TLB 87 * entry, a TLB MOD exception will be raised and that will cause the local TLB
88 * to be updated. 88 * to be updated.
89 * 89 *
90 * We never need to update a non-local TLB if the pmap doesn't have a valid 90 * We never need to update a non-local TLB if the pmap doesn't have a valid
91 * ASID for that TLB. If it does have a valid ASID but isn't current "onproc" 91 * ASID for that TLB. If it does have a valid ASID but isn't current "onproc"
92 * we simply reset its ASID for that TLB and then when it goes "onproc" it 92 * we simply reset its ASID for that TLB and then when it goes "onproc" it
93 * will allocate a new ASID and any existing TLB entries will be orphaned. 93 * will allocate a new ASID and any existing TLB entries will be orphaned.
94 * Only in the case that pmap has an "onproc" ASID do we actually have to send 94 * Only in the case that pmap has an "onproc" ASID do we actually have to send
95 * an IPI. 95 * an IPI.
96 * 96 *
97 * Once we determined we must send an IPI to shootdown a TLB, we need to send 97 * Once we determined we must send an IPI to shootdown a TLB, we need to send
98 * it to one of CPUs that share that TLB. We choose the lowest numbered CPU 98 * it to one of CPUs that share that TLB. We choose the lowest numbered CPU
99 * that has one of the pmap's ASID "onproc". In reality, any CPU sharing that 99 * that has one of the pmap's ASID "onproc". In reality, any CPU sharing that
100 * TLB would do, but interrupting an active CPU seems best. 100 * TLB would do, but interrupting an active CPU seems best.
101 * 101 *
102 * A TLB might have multiple shootdowns active concurrently. The shootdown 102 * A TLB might have multiple shootdowns active concurrently. The shootdown
103 * logic compresses these into a few cases: 103 * logic compresses these into a few cases:
104 * 0) nobody needs to have its TLB entries invalidated 104 * 0) nobody needs to have its TLB entries invalidated
105 * 1) one ASID needs to have its TLB entries invalidated 105 * 1) one ASID needs to have its TLB entries invalidated
106 * 2) more than one ASID needs to have its TLB entries invalidated 106 * 2) more than one ASID needs to have its TLB entries invalidated
107 * 3) the kernel needs to have its TLB entries invalidated 107 * 3) the kernel needs to have its TLB entries invalidated
108 * 4) the kernel and one or more ASID need their TLB entries invalidated. 108 * 4) the kernel and one or more ASID need their TLB entries invalidated.
109 * 109 *
110 * And for each case we do: 110 * And for each case we do:
111 * 0) nothing, 111 * 0) nothing,
112 * 1) if that ASID is still "onproc", we invalidate the TLB entries for 112 * 1) if that ASID is still "onproc", we invalidate the TLB entries for
113 * that single ASID. If not, just reset the pmap's ASID to invalidate 113 * that single ASID. If not, just reset the pmap's ASID to invalidate
114 * and let it allocate a new ASID the next time it goes "onproc", 114 * and let it allocate a new ASID the next time it goes "onproc",
115 * 2) we reinitialize the ASID space (preserving any "onproc" ASIDs) and 115 * 2) we reinitialize the ASID space (preserving any "onproc" ASIDs) and
116 * invalidate all non-wired non-global TLB entries, 116 * invalidate all non-wired non-global TLB entries,
117 * 3) we invalidate all of the non-wired global TLB entries, 117 * 3) we invalidate all of the non-wired global TLB entries,
118 * 4) we reinitialize the ASID space (again preserving any "onproc" ASIDs) 118 * 4) we reinitialize the ASID space (again preserving any "onproc" ASIDs)
119 * invalidate all non-wired TLB entries. 119 * invalidate all non-wired TLB entries.
120 * 120 *
121 * As you can see, shootdowns are not concerned with addresses, just address 121 * As you can see, shootdowns are not concerned with addresses, just address
122 * spaces. Since the number of TLB entries is usually quite small, this avoids 122 * spaces. Since the number of TLB entries is usually quite small, this avoids
123 * a lot of overhead for not much gain. 123 * a lot of overhead for not much gain.
124 */ 124 */
125 125
126#define __PMAP_PRIVATE 126#define __PMAP_PRIVATE
127 127
128#include "opt_multiprocessor.h" 128#include "opt_multiprocessor.h"
129 129
130#include <sys/param.h> 130#include <sys/param.h>
131 131
132#include <sys/atomic.h> 132#include <sys/atomic.h>
133#include <sys/cpu.h> 133#include <sys/cpu.h>
134#include <sys/kernel.h> /* for cold */ 134#include <sys/kernel.h> /* for cold */
135#include <sys/mutex.h> 135#include <sys/mutex.h>
136#include <sys/proc.h> 136#include <sys/proc.h>
137#include <sys/systm.h> 137#include <sys/systm.h>
138 138
139#include <uvm/uvm.h> 139#include <uvm/uvm.h>
140 140
141static kmutex_t pmap_tlb0_lock __cacheline_aligned; 141static kmutex_t pmap_tlb0_lock __cacheline_aligned;
142 142
143#define IFCONSTANT(x) (__builtin_constant_p((x)) ? (x) : 0) 143#define IFCONSTANT(x) (__builtin_constant_p((x)) ? (x) : 0)
144 144
145#if KERNEL_PID > 31 145#if KERNEL_PID > 31
146#error "KERNEL_PID expected in range 0-31" 146#error "KERNEL_PID expected in range 0-31"
147#endif 147#endif
148 148
149#define TLBINFO_ASID_MARK_UNUSED(ti, asid) \ 149#define TLBINFO_ASID_MARK_UNUSED(ti, asid) \
150 __BITMAP_CLR((asid), &(ti)->ti_asid_bitmap) 150 __BITMAP_CLR((asid), &(ti)->ti_asid_bitmap)
151#define TLBINFO_ASID_MARK_USED(ti, asid) \ 151#define TLBINFO_ASID_MARK_USED(ti, asid) \
152 __BITMAP_SET((asid), &(ti)->ti_asid_bitmap) 152 __BITMAP_SET((asid), &(ti)->ti_asid_bitmap)
153#define TLBINFO_ASID_INUSE_P(ti, asid) \ 153#define TLBINFO_ASID_INUSE_P(ti, asid) \
154 __BITMAP_ISSET((asid), &(ti)->ti_asid_bitmap) 154 __BITMAP_ISSET((asid), &(ti)->ti_asid_bitmap)
155#define TLBINFO_ASID_RESET(ti) \ 155#define TLBINFO_ASID_RESET(ti) \
156 do { \ 156 do { \
157 __BITMAP_ZERO(&ti->ti_asid_bitmap); \ 157 __BITMAP_ZERO(&ti->ti_asid_bitmap); \
158 for (tlb_asid_t asid = 0; asid <= KERNEL_PID; asid++) \ 158 for (tlb_asid_t asid = 0; asid <= KERNEL_PID; asid++) \
159 TLBINFO_ASID_MARK_USED(ti, asid); \ 159 TLBINFO_ASID_MARK_USED(ti, asid); \
160 } while (0) 160 } while (0)
161#define TLBINFO_ASID_INITIAL_FREE(asid_max) \ 161#define TLBINFO_ASID_INITIAL_FREE(asid_max) \
162 (asid_max + 1 /* 0 */ - (1 + KERNEL_PID)) 162 (asid_max + 1 /* 0 */ - (1 + KERNEL_PID))
163 163
164struct pmap_tlb_info pmap_tlb0_info = { 164struct pmap_tlb_info pmap_tlb0_info = {
165 .ti_name = "tlb0", 165 .ti_name = "tlb0",
166 .ti_asid_hint = KERNEL_PID + 1, 166 .ti_asid_hint = KERNEL_PID + 1,
167#ifdef PMAP_TLB_NUM_PIDS 167#ifdef PMAP_TLB_NUM_PIDS
168 .ti_asid_max = IFCONSTANT(PMAP_TLB_NUM_PIDS - 1), 168 .ti_asid_max = IFCONSTANT(PMAP_TLB_NUM_PIDS - 1),
169 .ti_asids_free = IFCONSTANT( 169 .ti_asids_free = IFCONSTANT(
170 TLBINFO_ASID_INITIAL_FREE(PMAP_TLB_NUM_PIDS - 1)), 170 TLBINFO_ASID_INITIAL_FREE(PMAP_TLB_NUM_PIDS - 1)),
171#endif 171#endif
172 .ti_asid_bitmap._b[0] = __BITS(0, KERNEL_PID), 172 .ti_asid_bitmap._b[0] = __BITS(0, KERNEL_PID),
173#ifdef PMAP_TLB_WIRED_UPAGES 173#ifdef PMAP_TLB_WIRED_UPAGES
174 .ti_wired = PMAP_TLB_WIRED_UPAGES, 174 .ti_wired = PMAP_TLB_WIRED_UPAGES,
175#endif 175#endif
176 .ti_lock = &pmap_tlb0_lock, 176 .ti_lock = &pmap_tlb0_lock,
177 .ti_pais = LIST_HEAD_INITIALIZER(pmap_tlb0_info.ti_pais), 177 .ti_pais = LIST_HEAD_INITIALIZER(pmap_tlb0_info.ti_pais),
178#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 178#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
179 .ti_tlbinvop = TLBINV_NOBODY, 179 .ti_tlbinvop = TLBINV_NOBODY,
180#endif 180#endif
181}; 181};
182 182
183#undef IFCONSTANT 183#undef IFCONSTANT
184 184
185#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 185#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
186struct pmap_tlb_info *pmap_tlbs[PMAP_TLB_MAX] = { 186struct pmap_tlb_info *pmap_tlbs[PMAP_TLB_MAX] = {
187 [0] = &pmap_tlb0_info, 187 [0] = &pmap_tlb0_info,
188}; 188};
189u_int pmap_ntlbs = 1; 189u_int pmap_ntlbs = 1;
190#endif 190#endif
191 191
192#ifdef MULTIPROCESSOR 192#ifdef MULTIPROCESSOR
193__unused static inline bool 193__unused static inline bool
194pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti) 194pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti)
195{ 195{
196#if PMAP_TLB_MAX == 1 196#if PMAP_TLB_MAX == 1
197 return !kcpuset_iszero(pm->pm_active); 197 return !kcpuset_iszero(pm->pm_active);
198#else 198#else
199 return kcpuset_intersecting_p(pm->pm_active, ti->ti_kcpuset); 199 return kcpuset_intersecting_p(pm->pm_active, ti->ti_kcpuset);
200#endif 200#endif
201} 201}
202 202
203static inline bool 203static inline bool
204pmap_tlb_intersecting_onproc_p(pmap_t pm, struct pmap_tlb_info *ti) 204pmap_tlb_intersecting_onproc_p(pmap_t pm, struct pmap_tlb_info *ti)
205{ 205{
206#if PMAP_TLB_MAX == 1 206#if PMAP_TLB_MAX == 1
207 return !kcpuset_iszero(pm->pm_onproc); 207 return !kcpuset_iszero(pm->pm_onproc);
208#else 208#else
209 return kcpuset_intersecting_p(pm->pm_onproc, ti->ti_kcpuset); 209 return kcpuset_intersecting_p(pm->pm_onproc, ti->ti_kcpuset);
210#endif 210#endif
211} 211}
212#endif 212#endif
213 213
214static void 214static void
215pmap_tlb_pai_check(struct pmap_tlb_info *ti, bool locked_p) 215pmap_tlb_pai_check(struct pmap_tlb_info *ti, bool locked_p)
216{ 216{
217 UVMHIST_FUNC(__func__); 217 UVMHIST_FUNC(__func__);
218 UVMHIST_CALLARGS(maphist, "(ti=%#jx)", (uintptr_t)ti, 0, 0, 0); 218 UVMHIST_CALLARGS(maphist, "(ti=%#jx)", (uintptr_t)ti, 0, 0, 0);
219 219
220#ifdef DIAGNOSTIC 220#ifdef DIAGNOSTIC
221 struct pmap_asid_info *pai; 221 struct pmap_asid_info *pai;
222 if (!locked_p) 222 if (!locked_p)
223 TLBINFO_LOCK(ti); 223 TLBINFO_LOCK(ti);
224 LIST_FOREACH(pai, &ti->ti_pais, pai_link) { 224 LIST_FOREACH(pai, &ti->ti_pais, pai_link) {
225 KASSERT(pai != NULL); 225 KASSERT(pai != NULL);
226 KASSERT(PAI_PMAP(pai, ti) != pmap_kernel()); 226 KASSERT(PAI_PMAP(pai, ti) != pmap_kernel());
227 KASSERT(pai->pai_asid > KERNEL_PID); 227 KASSERT(pai->pai_asid > KERNEL_PID);
228 KASSERTMSG(pai->pai_asid <= ti->ti_asid_max, 228 KASSERTMSG(pai->pai_asid <= ti->ti_asid_max,
229 "pm %p asid %#x", PAI_PMAP(pai, ti), pai->pai_asid); 229 "pm %p asid %#x", PAI_PMAP(pai, ti), pai->pai_asid);
230 KASSERTMSG(TLBINFO_ASID_INUSE_P(ti, pai->pai_asid), 230 KASSERTMSG(TLBINFO_ASID_INUSE_P(ti, pai->pai_asid),
231 "pm %p asid %u", PAI_PMAP(pai, ti), pai->pai_asid); 231 "pm %p asid %u", PAI_PMAP(pai, ti), pai->pai_asid);
232#ifdef MULTIPROCESSOR 232#ifdef MULTIPROCESSOR
233 KASSERT(pmap_tlb_intersecting_active_p(PAI_PMAP(pai, ti), ti)); 233 KASSERT(pmap_tlb_intersecting_active_p(PAI_PMAP(pai, ti), ti));
234#endif 234#endif
235 } 235 }
236 if (!locked_p) 236 if (!locked_p)
237 TLBINFO_UNLOCK(ti); 237 TLBINFO_UNLOCK(ti);
238#endif 238#endif
239 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 239 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
240} 240}
241 241
242static void 242static void
243pmap_tlb_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai, 243pmap_tlb_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai,
244 struct pmap *pm) 244 struct pmap *pm)
245{ 245{
246 UVMHIST_FUNC(__func__); 246 UVMHIST_FUNC(__func__);
247 UVMHIST_CALLARGS(maphist, "(ti=%#jx, pai=%#jx, pm=%#jx): asid %u", 247 UVMHIST_CALLARGS(maphist, "(ti=%#jx, pai=%#jx, pm=%#jx): asid %u",
248 (uintptr_t)ti, (uintptr_t)pai, (uintptr_t)pm, pai->pai_asid); 248 (uintptr_t)ti, (uintptr_t)pai, (uintptr_t)pm, pai->pai_asid);
249 249
250 /* 250 /*
251 * We must have an ASID but it must not be onproc (on a processor). 251 * We must have an ASID but it must not be onproc (on a processor).
252 */ 252 */
253 KASSERT(pai->pai_asid > KERNEL_PID); 253 KASSERT(pai->pai_asid > KERNEL_PID);
254 KASSERT(pai->pai_asid <= ti->ti_asid_max); 254 KASSERT(pai->pai_asid <= ti->ti_asid_max);
255#if defined(MULTIPROCESSOR) 255#if defined(MULTIPROCESSOR)
256 KASSERT(pmap_tlb_intersecting_active_p(pm, ti)); 256 KASSERT(pmap_tlb_intersecting_active_p(pm, ti));
257 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); 257 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
258#endif 258#endif
259 LIST_REMOVE(pai, pai_link); 259 LIST_REMOVE(pai, pai_link);
260#ifdef DIAGNOSTIC 260#ifdef DIAGNOSTIC
261 pai->pai_link.le_prev = NULL; /* tagged as unlinked */ 261 pai->pai_link.le_prev = NULL; /* tagged as unlinked */
262#endif 262#endif
263 /* 263 /*
264 * If the platform has a cheap way to flush ASIDs then free the ASID 264 * If the platform has a cheap way to flush ASIDs then free the ASID
265 * back into the pool. On multiprocessor systems, we will flush the 265 * back into the pool. On multiprocessor systems, we will flush the
266 * ASID from the TLB when it's allocated. That way we know the flush 266 * ASID from the TLB when it's allocated. That way we know the flush
267 * was always done in the correct TLB space. On uniprocessor systems, 267 * was always done in the correct TLB space. On uniprocessor systems,
268 * just do the flush now since we know that it has been used. This has 268 * just do the flush now since we know that it has been used. This has
269 * a bit less overhead. Either way, this will mean that we will only 269 * a bit less overhead. Either way, this will mean that we will only
270 * need to flush all ASIDs if all ASIDs are in use and we need to 270 * need to flush all ASIDs if all ASIDs are in use and we need to
271 * allocate a new one. 271 * allocate a new one.
272 */ 272 */
273 if (PMAP_TLB_FLUSH_ASID_ON_RESET) { 273 if (PMAP_TLB_FLUSH_ASID_ON_RESET) {
274#ifndef MULTIPROCESSOR 274#ifndef MULTIPROCESSOR
275 UVMHIST_LOG(maphist, " ... asid %u flushed", pai->pai_asid, 0, 275 UVMHIST_LOG(maphist, " ... asid %u flushed", pai->pai_asid, 0,
276 0, 0); 276 0, 0);
277 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 277 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
278#endif 278#endif
279 if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { 279 if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
280 UVMHIST_LOG(maphist, " ... asid marked unused", 280 UVMHIST_LOG(maphist, " ... asid marked unused",
281 pai->pai_asid, 0, 0, 0); 281 pai->pai_asid, 0, 0, 0);
282 TLBINFO_ASID_MARK_UNUSED(ti, pai->pai_asid); 282 TLBINFO_ASID_MARK_UNUSED(ti, pai->pai_asid);
283 ti->ti_asids_free++; 283 ti->ti_asids_free++;
284 } 284 }
285 } 285 }
286 /* 286 /*
287 * Note that we don't mark the ASID as not in use in the TLB's ASID 287 * Note that we don't mark the ASID as not in use in the TLB's ASID
288 * bitmap (thus it can't be allocated until the ASID space is exhausted 288 * bitmap (thus it can't be allocated until the ASID space is exhausted
289 * and therefore reinitialized). We don't want to flush the TLB for 289 * and therefore reinitialized). We don't want to flush the TLB for
290 * entries belonging to this ASID so we will let natural TLB entry 290 * entries belonging to this ASID so we will let natural TLB entry
291 * replacement flush them out of the TLB. Any new entries for this 291 * replacement flush them out of the TLB. Any new entries for this
292 * pmap will need a new ASID allocated. 292 * pmap will need a new ASID allocated.
293 */ 293 */
294 pai->pai_asid = 0; 294 pai->pai_asid = 0;
295 295
296#if defined(MULTIPROCESSOR) 296#if defined(MULTIPROCESSOR)
297 /* 297 /*
298 * The bits in pm_active belonging to this TLB can only be changed 298 * The bits in pm_active belonging to this TLB can only be changed
299 * while this TLB's lock is held. 299 * while this TLB's lock is held.
300 */ 300 */
301#if PMAP_TLB_MAX == 1 301#if PMAP_TLB_MAX == 1
302 kcpuset_zero(pm->pm_active); 302 kcpuset_zero(pm->pm_active);
303#else 303#else
304 kcpuset_remove(pm->pm_active, ti->ti_kcpuset); 304 kcpuset_remove(pm->pm_active, ti->ti_kcpuset);
305#endif 305#endif
306 KASSERT(!pmap_tlb_intersecting_active_p(pm, ti)); 306 KASSERT(!pmap_tlb_intersecting_active_p(pm, ti));
307#endif /* MULTIPROCESSOR */ 307#endif /* MULTIPROCESSOR */
308 308
309 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 309 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
310} 310}
311 311
312void 312void
313pmap_tlb_info_evcnt_attach(struct pmap_tlb_info *ti) 313pmap_tlb_info_evcnt_attach(struct pmap_tlb_info *ti)
314{ 314{
315#if defined(MULTIPROCESSOR) && !defined(PMAP_TLB_NO_SYNCI_EVCNT) 315#if defined(MULTIPROCESSOR) && !defined(PMAP_TLB_NO_SYNCI_EVCNT)
316 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_desired, 316 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_desired,
317 EVCNT_TYPE_MISC, NULL, 317 EVCNT_TYPE_MISC, NULL,
318 ti->ti_name, "icache syncs desired"); 318 ti->ti_name, "icache syncs desired");
319 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_asts, 319 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_asts,
320 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired, 320 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired,
321 ti->ti_name, "icache sync asts"); 321 ti->ti_name, "icache sync asts");
322 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_all, 322 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_all,
323 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts, 323 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts,
324 ti->ti_name, "icache full syncs"); 324 ti->ti_name, "icache full syncs");
325 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_pages, 325 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_pages,
326 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts, 326 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_asts,
327 ti->ti_name, "icache pages synced"); 327 ti->ti_name, "icache pages synced");
328 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_duplicate, 328 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_duplicate,
329 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired, 329 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired,
330 ti->ti_name, "icache dup pages skipped"); 330 ti->ti_name, "icache dup pages skipped");
331 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_deferred, 331 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_synci_deferred,
332 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired, 332 EVCNT_TYPE_MISC, &ti->ti_evcnt_synci_desired,
333 ti->ti_name, "icache pages deferred"); 333 ti->ti_name, "icache pages deferred");
334#endif /* MULTIPROCESSOR && !PMAP_TLB_NO_SYNCI_EVCNT */ 334#endif /* MULTIPROCESSOR && !PMAP_TLB_NO_SYNCI_EVCNT */
335 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_asid_reinits, 335 evcnt_attach_dynamic_nozero(&ti->ti_evcnt_asid_reinits,
336 EVCNT_TYPE_MISC, NULL, 336 EVCNT_TYPE_MISC, NULL,
337 ti->ti_name, "asid pool reinit"); 337 ti->ti_name, "asid pool reinit");
338} 338}
339 339
340void 340void
341pmap_tlb_info_init(struct pmap_tlb_info *ti) 341pmap_tlb_info_init(struct pmap_tlb_info *ti)
342{ 342{
343#if defined(MULTIPROCESSOR) 343#if defined(MULTIPROCESSOR)
344#if PMAP_TLB_MAX == 1 344#if PMAP_TLB_MAX == 1
345 KASSERT(ti == &pmap_tlb0_info); 345 KASSERT(ti == &pmap_tlb0_info);
346#else 346#else
347 if (ti != &pmap_tlb0_info) { 347 if (ti != &pmap_tlb0_info) {
348 KASSERT(pmap_ntlbs < PMAP_TLB_MAX); 348 KASSERT(pmap_ntlbs < PMAP_TLB_MAX);
349 349
350 KASSERT(pmap_tlbs[pmap_ntlbs] == NULL); 350 KASSERT(pmap_tlbs[pmap_ntlbs] == NULL);
351 351
352 ti->ti_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED); 352 ti->ti_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SCHED);
353 TLBINFO_ASID_RESET(ti); 353 TLBINFO_ASID_RESET(ti);
354 ti->ti_asid_hint = KERNEL_PID + 1; 354 ti->ti_asid_hint = KERNEL_PID + 1;
355 ti->ti_asid_max = pmap_tlbs[0]->ti_asid_max; 355 ti->ti_asid_max = pmap_tlbs[0]->ti_asid_max;
356 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max); 356 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max);
357 ti->ti_tlbinvop = TLBINV_NOBODY; 357 ti->ti_tlbinvop = TLBINV_NOBODY;
358 ti->ti_victim = NULL; 358 ti->ti_victim = NULL;
359 kcpuset_create(&ti->ti_kcpuset, true); 359 kcpuset_create(&ti->ti_kcpuset, true);
360 ti->ti_index = pmap_ntlbs++; 360 ti->ti_index = pmap_ntlbs++;
361 ti->ti_wired = 0; 361 ti->ti_wired = 0;
362 pmap_tlbs[ti->ti_index] = ti; 362 pmap_tlbs[ti->ti_index] = ti;
363 snprintf(ti->ti_name, sizeof(ti->ti_name), "tlb%u", 363 snprintf(ti->ti_name, sizeof(ti->ti_name), "tlb%u",
364 ti->ti_index); 364 ti->ti_index);
365 pmap_tlb_info_evcnt_attach(ti); 365 pmap_tlb_info_evcnt_attach(ti);
366 366
367 KASSERT(ti->ti_asid_max < PMAP_TLB_BITMAP_LENGTH); 367 KASSERT(ti->ti_asid_max < PMAP_TLB_BITMAP_LENGTH);
368 return; 368 return;
369 } 369 }
370#endif 370#endif
371#endif /* MULTIPROCESSOR */ 371#endif /* MULTIPROCESSOR */
372 KASSERT(ti == &pmap_tlb0_info); 372 KASSERT(ti == &pmap_tlb0_info);
373 KASSERT(ti->ti_lock == &pmap_tlb0_lock); 373 KASSERT(ti->ti_lock == &pmap_tlb0_lock);
374 374
375 mutex_init(ti->ti_lock, MUTEX_DEFAULT, IPL_SCHED); 375 mutex_init(ti->ti_lock, MUTEX_DEFAULT, IPL_SCHED);
376#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1 376#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
377 kcpuset_create(&ti->ti_kcpuset, true); 377 kcpuset_create(&ti->ti_kcpuset, true);
378 kcpuset_set(ti->ti_kcpuset, cpu_index(curcpu())); 378 kcpuset_set(ti->ti_kcpuset, cpu_index(curcpu()));
379#endif 379#endif
380 380
381 const tlb_asid_t asid_max = pmap_md_tlb_asid_max(); 381 const tlb_asid_t asid_max = pmap_md_tlb_asid_max();
382 if (ti->ti_asid_max == 0 || asid_max < ti->ti_asid_max) { 382 if (ti->ti_asid_max == 0 || asid_max < ti->ti_asid_max) {
383 ti->ti_asid_max = asid_max; 383 ti->ti_asid_max = asid_max;
384 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max); 384 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max);
385 } 385 }
386 386
387 KASSERT(ti->ti_asid_max < PMAP_TLB_BITMAP_LENGTH); 387 KASSERT(ti->ti_asid_max < PMAP_TLB_BITMAP_LENGTH);
388} 388}
389 389
390#if defined(MULTIPROCESSOR) 390#if defined(MULTIPROCESSOR)
391void 391void
392pmap_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci) 392pmap_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci)
393{ 393{
394 KASSERT(!CPU_IS_PRIMARY(ci)); 394 KASSERT(!CPU_IS_PRIMARY(ci));
395 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 395 KASSERT(ci->ci_data.cpu_idlelwp != NULL);
396 KASSERT(cold); 396 KASSERT(cold);
397 397
398 TLBINFO_LOCK(ti); 398 TLBINFO_LOCK(ti);
399#if PMAP_TLB_MAX > 1 399#if PMAP_TLB_MAX > 1
400 kcpuset_set(ti->ti_kcpuset, cpu_index(ci)); 400 kcpuset_set(ti->ti_kcpuset, cpu_index(ci));
401 cpu_set_tlb_info(ci, ti); 401 cpu_set_tlb_info(ci, ti);
402#endif 402#endif
403 403
404 /* 404 /*
405 * Do any MD tlb info init. 405 * Do any MD tlb info init.
406 */ 406 */
407 pmap_md_tlb_info_attach(ti, ci); 407 pmap_md_tlb_info_attach(ti, ci);
408 408
409 /* 409 /*
410 * The kernel pmap uses the kcpuset_running set so it's always 410 * The kernel pmap uses the kcpuset_running set so it's always
411 * up-to-date. 411 * up-to-date.
412 */ 412 */
413 TLBINFO_UNLOCK(ti); 413 TLBINFO_UNLOCK(ti);
414} 414}
415#endif /* MULTIPROCESSOR */ 415#endif /* MULTIPROCESSOR */
416 416
417#ifdef DIAGNOSTIC 417#ifdef DIAGNOSTIC
418static size_t 418static size_t
419pmap_tlb_asid_count(struct pmap_tlb_info *ti) 419pmap_tlb_asid_count(struct pmap_tlb_info *ti)
420{ 420{
421 size_t count = 0; 421 size_t count = 0;
422 for (tlb_asid_t asid = 1; asid <= ti->ti_asid_max; asid++) { 422 for (tlb_asid_t asid = 1; asid <= ti->ti_asid_max; asid++) {
423 if (TLBINFO_ASID_INUSE_P(ti, asid)) 423 if (TLBINFO_ASID_INUSE_P(ti, asid))
424 count++; 424 count++;
425 } 425 }
426 return count; 426 return count;
427} 427}
428#endif 428#endif
429 429
430static void 430static void
431pmap_tlb_asid_reinitialize(struct pmap_tlb_info *ti, enum tlb_invalidate_op op) 431pmap_tlb_asid_reinitialize(struct pmap_tlb_info *ti, enum tlb_invalidate_op op)
432{ 432{
433 UVMHIST_FUNC(__func__); 433 UVMHIST_FUNC(__func__);
434 UVMHIST_CALLARGS(maphist, "(ti=%#jx, op=%ju)", (uintptr_t)ti, op, 0, 0); 434 UVMHIST_CALLARGS(maphist, "(ti=%#jx, op=%ju)", (uintptr_t)ti, op, 0, 0);
435 435
436 pmap_tlb_pai_check(ti, true); 436 pmap_tlb_pai_check(ti, true);
437 437
438 ti->ti_evcnt_asid_reinits.ev_count++; 438 ti->ti_evcnt_asid_reinits.ev_count++;
439 439
440 /* 440 /*
441 * First, clear the ASID bitmap (except for ASID 0 which belongs 441 * First, clear the ASID bitmap (except for ASID 0 which belongs
442 * to the kernel). 442 * to the kernel).
443 */ 443 */
444 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max); 444 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(ti->ti_asid_max);
445 ti->ti_asid_hint = KERNEL_PID + 1; 445 ti->ti_asid_hint = KERNEL_PID + 1;
446 TLBINFO_ASID_RESET(ti); 446 TLBINFO_ASID_RESET(ti);
447 447
448 switch (op) { 448 switch (op) {
449#if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 449#if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)
450 case TLBINV_ALL: 450 case TLBINV_ALL:
451 tlb_invalidate_all(); 451 tlb_invalidate_all();
452 break; 452 break;
453 case TLBINV_ALLUSER: 453 case TLBINV_ALLUSER:
454 tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max); 454 tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max);
455 break; 455 break;
456#endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */ 456#endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */
457 case TLBINV_NOBODY: { 457 case TLBINV_NOBODY: {
458 /* 458 /*
459 * If we are just reclaiming ASIDs in the TLB, let's go find 459 * If we are just reclaiming ASIDs in the TLB, let's go find
460 * what ASIDs are in use in the TLB. Since this is a 460 * what ASIDs are in use in the TLB. Since this is a
461 * semi-expensive operation, we don't want to do it too often. 461 * semi-expensive operation, we don't want to do it too often.
462 * So if more half of the ASIDs are in use, we don't have 462 * So if more half of the ASIDs are in use, we don't have
463 * enough free ASIDs so invalidate the TLB entries with ASIDs 463 * enough free ASIDs so invalidate the TLB entries with ASIDs
464 * and clear the ASID bitmap. That will force everyone to 464 * and clear the ASID bitmap. That will force everyone to
465 * allocate a new ASID. 465 * allocate a new ASID.
466 */ 466 */
467#if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN) 467#if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN)
468 pmap_tlb_asid_check(); 468 pmap_tlb_asid_check();
469 const u_int asids_found = tlb_record_asids( 469 const u_int asids_found = tlb_record_asids(
470 ti->ti_asid_bitmap._b, ti->ti_asid_max); 470 ti->ti_asid_bitmap._b, ti->ti_asid_max);
471 pmap_tlb_asid_check(); 471 pmap_tlb_asid_check();
472#ifdef DIAGNOSTIC 472#ifdef DIAGNOSTIC
473 const u_int asids_count = pmap_tlb_asid_count(ti); 473 const u_int asids_count = pmap_tlb_asid_count(ti);
474 KASSERTMSG(asids_found == asids_count, 474 KASSERTMSG(asids_found == asids_count,
475 "found %u != count %u", asids_found, asids_count); 475 "found %u != count %u", asids_found, asids_count);
476#endif 476#endif
477 if (__predict_false(asids_found >= ti->ti_asid_max / 2)) { 477 if (__predict_false(asids_found >= ti->ti_asid_max / 2)) {
478 tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max); 478 tlb_invalidate_asids(KERNEL_PID + 1, ti->ti_asid_max);
479#else /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */ 479#else /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */
480 /* 480 /*
481 * For those systems (PowerPC) that don't require 481 * For those systems (PowerPC) that don't require
482 * cross cpu TLB shootdowns, we have to invalidate the 482 * cross cpu TLB shootdowns, we have to invalidate the
483 * entire TLB because we can't record the ASIDs in use 483 * entire TLB because we can't record the ASIDs in use
484 * on the other CPUs. This is hopefully cheaper than 484 * on the other CPUs. This is hopefully cheaper than
485 * than trying to use an IPI to record all the ASIDs 485 * than trying to use an IPI to record all the ASIDs
486 * on all the CPUs (which would be a synchronization 486 * on all the CPUs (which would be a synchronization
487 * nightmare). 487 * nightmare).
488 */ 488 */
489 tlb_invalidate_all(); 489 tlb_invalidate_all();
490#endif /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */ 490#endif /* MULTIPROCESSOR && !PMAP_TLB_NEED_SHOOTDOWN */
491 TLBINFO_ASID_RESET(ti); 491 TLBINFO_ASID_RESET(ti);
492 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE( 492 ti->ti_asids_free = TLBINFO_ASID_INITIAL_FREE(
493 ti->ti_asid_max); 493 ti->ti_asid_max);
494#if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN) 494#if !defined(MULTIPROCESSOR) || defined(PMAP_TLB_NEED_SHOOTDOWN)
495 } else { 495 } else {
496 ti->ti_asids_free -= asids_found; 496 ti->ti_asids_free -= asids_found;
497 } 497 }
498#endif /* !MULTIPROCESSOR || PMAP_TLB_NEED_SHOOTDOWN */ 498#endif /* !MULTIPROCESSOR || PMAP_TLB_NEED_SHOOTDOWN */
499 KASSERTMSG(ti->ti_asids_free <= ti->ti_asid_max, "%u", 499 KASSERTMSG(ti->ti_asids_free <= ti->ti_asid_max, "%u",
500 ti->ti_asids_free); 500 ti->ti_asids_free);
501 break; 501 break;
502 } 502 }
503 default: 503 default:
504 panic("%s: unexpected op %d", __func__, op); 504 panic("%s: unexpected op %d", __func__, op);
505 } 505 }
506 506
507 /* 507 /*
508 * Now go through the active ASIDs. If the ASID is on a processor or 508 * Now go through the active ASIDs. If the ASID is on a processor or
509 * we aren't invalidating all ASIDs and the TLB has an entry owned by 509 * we aren't invalidating all ASIDs and the TLB has an entry owned by
510 * that ASID, mark it as in use. Otherwise release the ASID. 510 * that ASID, mark it as in use. Otherwise release the ASID.
511 */ 511 */
512 struct pmap_asid_info *pai, *next; 512 struct pmap_asid_info *pai, *next;
513 for (pai = LIST_FIRST(&ti->ti_pais); pai != NULL; pai = next) { 513 for (pai = LIST_FIRST(&ti->ti_pais); pai != NULL; pai = next) {
514 struct pmap * const pm = PAI_PMAP(pai, ti); 514 struct pmap * const pm = PAI_PMAP(pai, ti);
515 next = LIST_NEXT(pai, pai_link); 515 next = LIST_NEXT(pai, pai_link);
516 KASSERT(pm != pmap_kernel()); 516 KASSERT(pm != pmap_kernel());
517 KASSERT(pai->pai_asid > KERNEL_PID); 517 KASSERT(pai->pai_asid > KERNEL_PID);
518#if defined(MULTIPROCESSOR) 518#if defined(MULTIPROCESSOR)
519 if (pmap_tlb_intersecting_onproc_p(pm, ti)) { 519 if (pmap_tlb_intersecting_onproc_p(pm, ti)) {
520 if (!TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { 520 if (!TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
521 TLBINFO_ASID_MARK_USED(ti, pai->pai_asid); 521 TLBINFO_ASID_MARK_USED(ti, pai->pai_asid);
522 ti->ti_asids_free--; 522 ti->ti_asids_free--;
523 } 523 }
524 continue; 524 continue;
525 } 525 }
526#endif /* MULTIPROCESSOR */ 526#endif /* MULTIPROCESSOR */
527 if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) { 527 if (TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
528 KASSERT(op == TLBINV_NOBODY); 528 KASSERT(op == TLBINV_NOBODY);
529 } else { 529 } else {
530 pmap_tlb_pai_reset(ti, pai, pm); 530 pmap_tlb_pai_reset(ti, pai, pm);
531 } 531 }
532 } 532 }
533#ifdef DIAGNOSTIC 533#ifdef DIAGNOSTIC
534 size_t free_count __diagused = ti->ti_asid_max - pmap_tlb_asid_count(ti); 534 size_t free_count __diagused = ti->ti_asid_max - pmap_tlb_asid_count(ti);
535 KASSERTMSG(free_count == ti->ti_asids_free, 535 KASSERTMSG(free_count == ti->ti_asids_free,
536 "bitmap error: %zu != %u", free_count, ti->ti_asids_free); 536 "bitmap error: %zu != %u", free_count, ti->ti_asids_free);
537#endif 537#endif
538 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 538 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
539} 539}
540 540
541#if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 541#if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)
542#if PMAP_TLB_MAX == 1 542#if PMAP_TLB_MAX == 1
543#error shootdown not required for single TLB systems 543#error shootdown not required for single TLB systems
544#endif 544#endif
545void 545void
546pmap_tlb_shootdown_process(void) 546pmap_tlb_shootdown_process(void)
547{ 547{
548 struct cpu_info * const ci = curcpu(); 548 struct cpu_info * const ci = curcpu();
549 struct pmap_tlb_info * const ti = cpu_tlb_info(ci); 549 struct pmap_tlb_info * const ti = cpu_tlb_info(ci);
550#ifdef DIAGNOSTIC 
551 struct pmap * const pm = curlwp->l_proc->p_vmspace->vm_map.pmap; 
552#endif 
553 550
554 KASSERT(cpu_intr_p()); 551 KASSERT(cpu_intr_p());
555 KASSERTMSG(ci->ci_cpl >= IPL_SCHED, "%s: cpl (%d) < IPL_SCHED (%d)", 552 KASSERTMSG(ci->ci_cpl >= IPL_SCHED, "%s: cpl (%d) < IPL_SCHED (%d)",
556 __func__, ci->ci_cpl, IPL_SCHED); 553 __func__, ci->ci_cpl, IPL_SCHED);
557 554
558 TLBINFO_LOCK(ti); 555 TLBINFO_LOCK(ti);
559 556
560 switch (ti->ti_tlbinvop) { 557 switch (ti->ti_tlbinvop) {
561 case TLBINV_ONE: { 558 case TLBINV_ONE: {
562 /* 559 /*
563 * We only need to invalidate one user ASID. 560 * We only need to invalidate one user ASID.
564 */ 561 */
565 struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti); 562 struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti);
566 KASSERT(ti->ti_victim != pmap_kernel()); 563 KASSERT(ti->ti_victim != pmap_kernel());
567 if (pmap_tlb_intersecting_onproc_p(ti->ti_victim, ti)) { 564 if (pmap_tlb_intersecting_onproc_p(ti->ti_victim, ti)) {
568 /* 565 /*
569 * The victim is an active pmap so we will just 566 * The victim is an active pmap so we will just
570 * invalidate its TLB entries. 567 * invalidate its TLB entries.
571 */ 568 */
572 KASSERT(pai->pai_asid > KERNEL_PID); 569 KASSERT(pai->pai_asid > KERNEL_PID);
573 pmap_tlb_asid_check(); 570 pmap_tlb_asid_check();
574 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 571 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
575 pmap_tlb_asid_check(); 572 pmap_tlb_asid_check();
576 } else if (pai->pai_asid) { 573 } else if (pai->pai_asid) {
577 /* 574 /*
578 * The victim is no longer an active pmap for this TLB. 575 * The victim is no longer an active pmap for this TLB.
579 * So simply clear its ASID and when pmap_activate is 576 * So simply clear its ASID and when pmap_activate is
580 * next called for this pmap, it will allocate a new 577 * next called for this pmap, it will allocate a new
581 * ASID. 578 * ASID.
582 */ 579 */
583 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); 
584 pmap_tlb_pai_reset(ti, pai, PAI_PMAP(pai, ti)); 580 pmap_tlb_pai_reset(ti, pai, PAI_PMAP(pai, ti));
585 } 581 }
586 break; 582 break;
587 } 583 }
588 case TLBINV_ALLUSER: 584 case TLBINV_ALLUSER:
589 /* 585 /*
590 * Flush all user TLB entries. 586 * Flush all user TLB entries.
591 */ 587 */
592 pmap_tlb_asid_reinitialize(ti, TLBINV_ALLUSER); 588 pmap_tlb_asid_reinitialize(ti, TLBINV_ALLUSER);
593 break; 589 break;
594 case TLBINV_ALLKERNEL: 590 case TLBINV_ALLKERNEL:
595 /* 591 /*
596 * We need to invalidate all global TLB entries. 592 * We need to invalidate all global TLB entries.
597 */ 593 */
598 pmap_tlb_asid_check(); 594 pmap_tlb_asid_check();
599 tlb_invalidate_globals(); 595 tlb_invalidate_globals();
600 pmap_tlb_asid_check(); 596 pmap_tlb_asid_check();
601 break; 597 break;
602 case TLBINV_ALL: 598 case TLBINV_ALL:
603 /* 599 /*
604 * Flush all the TLB entries (user and kernel). 600 * Flush all the TLB entries (user and kernel).
605 */ 601 */
606 pmap_tlb_asid_reinitialize(ti, TLBINV_ALL); 602 pmap_tlb_asid_reinitialize(ti, TLBINV_ALL);
607 break; 603 break;
608 case TLBINV_NOBODY: 604 case TLBINV_NOBODY:
609 /* 605 /*
610 * Might be spurious or another SMT CPU sharing this TLB 606 * Might be spurious or another SMT CPU sharing this TLB
611 * could have already done the work. 607 * could have already done the work.
612 */ 608 */
613 break; 609 break;
614 } 610 }
615 611
616 /* 612 /*
617 * Indicate we are done with shutdown event. 613 * Indicate we are done with shutdown event.
618 */ 614 */
619 ti->ti_victim = NULL; 615 ti->ti_victim = NULL;
620 ti->ti_tlbinvop = TLBINV_NOBODY; 616 ti->ti_tlbinvop = TLBINV_NOBODY;
621 TLBINFO_UNLOCK(ti); 617 TLBINFO_UNLOCK(ti);
622} 618}
623 619
624/* 620/*
625 * This state machine could be encoded into an array of integers but since all 621 * This state machine could be encoded into an array of integers but since all
626 * the values fit in 3 bits, the 5 entry "table" fits in a 16 bit value which 622 * the values fit in 3 bits, the 5 entry "table" fits in a 16 bit value which
627 * can be loaded in a single instruction. 623 * can be loaded in a single instruction.
628 */ 624 */
629#define TLBINV_MAP(op, nobody, one, alluser, allkernel, all) \ 625#define TLBINV_MAP(op, nobody, one, alluser, allkernel, all) \
630 (((( (nobody) << 3 * TLBINV_NOBODY) \ 626 (((( (nobody) << 3 * TLBINV_NOBODY) \
631 | ( (one) << 3 * TLBINV_ONE) \ 627 | ( (one) << 3 * TLBINV_ONE) \
632 | ( (alluser) << 3 * TLBINV_ALLUSER) \ 628 | ( (alluser) << 3 * TLBINV_ALLUSER) \
633 | ((allkernel) << 3 * TLBINV_ALLKERNEL) \ 629 | ((allkernel) << 3 * TLBINV_ALLKERNEL) \
634 | ( (all) << 3 * TLBINV_ALL)) >> 3 * (op)) & 7) 630 | ( (all) << 3 * TLBINV_ALL)) >> 3 * (op)) & 7)
635 631
636#define TLBINV_USER_MAP(op) \ 632#define TLBINV_USER_MAP(op) \
637 TLBINV_MAP(op, TLBINV_ONE, TLBINV_ALLUSER, TLBINV_ALLUSER, \ 633 TLBINV_MAP(op, TLBINV_ONE, TLBINV_ALLUSER, TLBINV_ALLUSER, \
638 TLBINV_ALL, TLBINV_ALL) 634 TLBINV_ALL, TLBINV_ALL)
639 635
640#define TLBINV_KERNEL_MAP(op) \ 636#define TLBINV_KERNEL_MAP(op) \
641 TLBINV_MAP(op, TLBINV_ALLKERNEL, TLBINV_ALL, TLBINV_ALL, \ 637 TLBINV_MAP(op, TLBINV_ALLKERNEL, TLBINV_ALL, TLBINV_ALL, \
642 TLBINV_ALLKERNEL, TLBINV_ALL) 638 TLBINV_ALLKERNEL, TLBINV_ALL)
643 639
644bool 640bool
645pmap_tlb_shootdown_bystanders(pmap_t pm) 641pmap_tlb_shootdown_bystanders(pmap_t pm)
646{ 642{
647 /* 643 /*
648 * We don't need to deal with our own TLB. 644 * We don't need to deal with our own TLB.
649 */ 645 */
650 646
651 UVMHIST_FUNC(__func__); 647 UVMHIST_FUNC(__func__);
652 UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0); 648 UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0);
653 649
654 const struct cpu_info * const ci = curcpu(); 650 const struct cpu_info * const ci = curcpu();
655 kcpuset_t *pm_active = ci->ci_shootdowncpus; 651 kcpuset_t *pm_active = ci->ci_shootdowncpus;
656 kcpuset_copy(pm_active, pm->pm_active); 652 kcpuset_copy(pm_active, pm->pm_active);
657 kcpuset_remove(pm_active, cpu_tlb_info(curcpu())->ti_kcpuset); 653 kcpuset_remove(pm_active, cpu_tlb_info(curcpu())->ti_kcpuset);
658 const bool kernel_p = (pm == pmap_kernel()); 654 const bool kernel_p = (pm == pmap_kernel());
659 bool ipi_sent = false; 655 bool ipi_sent = false;
660 656
661 /* 657 /*
662 * If pm_active gets more bits set, then it's after all our changes 658 * If pm_active gets more bits set, then it's after all our changes
663 * have been made so they will already be cognizant of them. 659 * have been made so they will already be cognizant of them.
664 */ 660 */
665 661
666 for (size_t i = 0; !kcpuset_iszero(pm_active); i++) { 662 for (size_t i = 0; !kcpuset_iszero(pm_active); i++) {
667 KASSERT(i < pmap_ntlbs); 663 KASSERT(i < pmap_ntlbs);
668 struct pmap_tlb_info * const ti = pmap_tlbs[i]; 664 struct pmap_tlb_info * const ti = pmap_tlbs[i];
669 KASSERT(tlbinfo_index(ti) == i); 665 KASSERT(tlbinfo_index(ti) == i);
670 /* 666 /*
671 * Skip this TLB if there are no active mappings for it. 667 * Skip this TLB if there are no active mappings for it.
672 */ 668 */
673 if (!kcpuset_intersecting_p(pm_active, ti->ti_kcpuset)) 669 if (!kcpuset_intersecting_p(pm_active, ti->ti_kcpuset))
674 continue; 670 continue;
675 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 671 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
676 kcpuset_remove(pm_active, ti->ti_kcpuset); 672 kcpuset_remove(pm_active, ti->ti_kcpuset);
677 TLBINFO_LOCK(ti); 673 TLBINFO_LOCK(ti);
678 cpuid_t j = kcpuset_ffs_intersecting(pm->pm_onproc, 674 cpuid_t j = kcpuset_ffs_intersecting(pm->pm_onproc,
679 ti->ti_kcpuset); 675 ti->ti_kcpuset);
680 // post decrement since ffs returns bit + 1 or 0 if no bit 676 // post decrement since ffs returns bit + 1 or 0 if no bit
681 if (j-- > 0) { 677 if (j-- > 0) {
682 if (kernel_p) { 678 if (kernel_p) {
683 ti->ti_tlbinvop = 679 ti->ti_tlbinvop =
684 TLBINV_KERNEL_MAP(ti->ti_tlbinvop); 680 TLBINV_KERNEL_MAP(ti->ti_tlbinvop);
685 ti->ti_victim = NULL; 681 ti->ti_victim = NULL;
686 } else { 682 } else {
687 KASSERT(pai->pai_asid); 683 KASSERT(pai->pai_asid);
688 if (__predict_false(ti->ti_victim == pm)) { 684 if (__predict_false(ti->ti_victim == pm)) {
689 KASSERT(ti->ti_tlbinvop == TLBINV_ONE); 685 KASSERT(ti->ti_tlbinvop == TLBINV_ONE);
690 /* 686 /*
691 * We still need to invalidate this one 687 * We still need to invalidate this one
692 * ASID so there's nothing to change. 688 * ASID so there's nothing to change.
693 */ 689 */
694 } else { 690 } else {
695 ti->ti_tlbinvop = 691 ti->ti_tlbinvop =
696 TLBINV_USER_MAP(ti->ti_tlbinvop); 692 TLBINV_USER_MAP(ti->ti_tlbinvop);
697 if (ti->ti_tlbinvop == TLBINV_ONE) 693 if (ti->ti_tlbinvop == TLBINV_ONE)
698 ti->ti_victim = pm; 694 ti->ti_victim = pm;
699 else 695 else
700 ti->ti_victim = NULL; 696 ti->ti_victim = NULL;
701 } 697 }
702 } 698 }
703 TLBINFO_UNLOCK(ti); 699 TLBINFO_UNLOCK(ti);
704 /* 700 /*
705 * Now we can send out the shootdown IPIs to a CPU 701 * Now we can send out the shootdown IPIs to a CPU
706 * that shares this TLB and is currently using this 702 * that shares this TLB and is currently using this
707 * pmap. That CPU will process the IPI and do the 703 * pmap. That CPU will process the IPI and do the
708 * all the work. Any other CPUs sharing that TLB 704 * all the work. Any other CPUs sharing that TLB
709 * will take advantage of that work. pm_onproc might 705 * will take advantage of that work. pm_onproc might
710 * change now that we have released the lock but we 706 * change now that we have released the lock but we
711 * can tolerate spurious shootdowns. 707 * can tolerate spurious shootdowns.
712 */ 708 */
713 cpu_send_ipi(cpu_lookup(j), IPI_SHOOTDOWN); 709 cpu_send_ipi(cpu_lookup(j), IPI_SHOOTDOWN);
714 ipi_sent = true; 710 ipi_sent = true;
715 continue; 711 continue;
716 } 712 }
717 if (!pmap_tlb_intersecting_active_p(pm, ti)) { 713 if (!pmap_tlb_intersecting_active_p(pm, ti)) {
718 /* 714 /*
719 * If this pmap has an ASID assigned but it's not 715 * If this pmap has an ASID assigned but it's not
720 * currently running, nuke its ASID. Next time the 716 * currently running, nuke its ASID. Next time the
721 * pmap is activated, it will allocate a new ASID. 717 * pmap is activated, it will allocate a new ASID.
722 * And best of all, we avoid an IPI. 718 * And best of all, we avoid an IPI.
723 */ 719 */
724 KASSERT(!kernel_p); 720 KASSERT(!kernel_p);
725 pmap_tlb_pai_reset(ti, pai, pm); 721 pmap_tlb_pai_reset(ti, pai, pm);
726 //ti->ti_evcnt_lazy_shots.ev_count++; 722 //ti->ti_evcnt_lazy_shots.ev_count++;
727 } 723 }
728 TLBINFO_UNLOCK(ti); 724 TLBINFO_UNLOCK(ti);
729 } 725 }
730 726
731 UVMHIST_LOG(maphist, " <-- done (ipi_sent=%jd)", ipi_sent, 0, 0, 0); 727 UVMHIST_LOG(maphist, " <-- done (ipi_sent=%jd)", ipi_sent, 0, 0, 0);
732 728
733 return ipi_sent; 729 return ipi_sent;
734} 730}
735#endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */ 731#endif /* MULTIPROCESSOR && PMAP_TLB_NEED_SHOOTDOWN */
736 732
737int 733int
738pmap_tlb_update_addr(pmap_t pm, vaddr_t va, pt_entry_t pte, u_int flags) 734pmap_tlb_update_addr(pmap_t pm, vaddr_t va, pt_entry_t pte, u_int flags)
739{ 735{
740 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 736 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());
741 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 737 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
742 int rv = -1; 738 int rv = -1;
743 739
744 UVMHIST_FUNC(__func__); 740 UVMHIST_FUNC(__func__);
745 UVMHIST_CALLARGS(maphist, " (pm=%#jx va=%#jx, pte=%#jx flags=%#jx)", 741 UVMHIST_CALLARGS(maphist, " (pm=%#jx va=%#jx, pte=%#jx flags=%#jx)",
746 (uintptr_t)pm, va, pte_value(pte), flags); 742 (uintptr_t)pm, va, pte_value(pte), flags);
747 743
748 KASSERT(kpreempt_disabled()); 744 KASSERT(kpreempt_disabled());
749 745
750 KASSERTMSG(pte_valid_p(pte), "va %#"PRIxVADDR" %#"PRIxPTE, 746 KASSERTMSG(pte_valid_p(pte), "va %#"PRIxVADDR" %#"PRIxPTE,
751 va, pte_value(pte)); 747 va, pte_value(pte));
752 748
753 TLBINFO_LOCK(ti); 749 TLBINFO_LOCK(ti);
754 if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) { 750 if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) {
755 pmap_tlb_asid_check(); 751 pmap_tlb_asid_check();
756 rv = tlb_update_addr(va, pai->pai_asid, pte, 752 rv = tlb_update_addr(va, pai->pai_asid, pte,
757 (flags & PMAP_TLB_INSERT) != 0); 753 (flags & PMAP_TLB_INSERT) != 0);
758 pmap_tlb_asid_check(); 754 pmap_tlb_asid_check();
759 UVMHIST_LOG(maphist, 755 UVMHIST_LOG(maphist,
760 " %jd <-- tlb_update_addr(%#jx, %#jx, %#jx, ...)", 756 " %jd <-- tlb_update_addr(%#jx, %#jx, %#jx, ...)",
761 rv, va, pai->pai_asid, pte_value(pte)); 757 rv, va, pai->pai_asid, pte_value(pte));
762 KASSERTMSG((flags & PMAP_TLB_INSERT) == 0 || rv == 1, 758 KASSERTMSG((flags & PMAP_TLB_INSERT) == 0 || rv == 1,
763 "pmap %p (asid %u) va %#"PRIxVADDR" pte %#"PRIxPTE" rv %d", 759 "pmap %p (asid %u) va %#"PRIxVADDR" pte %#"PRIxPTE" rv %d",
764 pm, pai->pai_asid, va, pte_value(pte), rv); 760 pm, pai->pai_asid, va, pte_value(pte), rv);
765 } 761 }
766#if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 762#if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)
767 if (flags & PMAP_TLB_NEED_IPI) 763 if (flags & PMAP_TLB_NEED_IPI)
768 pm->pm_shootdown_pending = 1; 764 pm->pm_shootdown_pending = 1;
769#endif 765#endif
770 TLBINFO_UNLOCK(ti); 766 TLBINFO_UNLOCK(ti);
771 767
772 UVMHIST_LOG(maphist, " <-- done (rv=%jd)", rv, 0, 0, 0); 768 UVMHIST_LOG(maphist, " <-- done (rv=%jd)", rv, 0, 0, 0);
773 769
774 return rv; 770 return rv;
775} 771}
776 772
777void 773void
778pmap_tlb_invalidate_addr(pmap_t pm, vaddr_t va) 774pmap_tlb_invalidate_addr(pmap_t pm, vaddr_t va)
779{ 775{
780 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 776 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());
781 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 777 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
782 778
783 UVMHIST_FUNC(__func__); 779 UVMHIST_FUNC(__func__);
784 UVMHIST_CALLARGS(maphist, " (pm=%#jx va=%#jx) ti=%#jx asid=%#jx", 780 UVMHIST_CALLARGS(maphist, " (pm=%#jx va=%#jx) ti=%#jx asid=%#jx",
785 (uintptr_t)pm, va, (uintptr_t)ti, pai->pai_asid); 781 (uintptr_t)pm, va, (uintptr_t)ti, pai->pai_asid);
786 782
787 KASSERT(kpreempt_disabled()); 783 KASSERT(kpreempt_disabled());
788 784
789 TLBINFO_LOCK(ti); 785 TLBINFO_LOCK(ti);
790 if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) { 786 if (pm == pmap_kernel() || PMAP_PAI_ASIDVALID_P(pai, ti)) {
791 pmap_tlb_asid_check(); 787 pmap_tlb_asid_check();
792 UVMHIST_LOG(maphist, " invalidating %#jx asid %#jx", 788 UVMHIST_LOG(maphist, " invalidating %#jx asid %#jx",
793 va, pai->pai_asid, 0, 0); 789 va, pai->pai_asid, 0, 0);
794 tlb_invalidate_addr(va, pai->pai_asid); 790 tlb_invalidate_addr(va, pai->pai_asid);
795 pmap_tlb_asid_check(); 791 pmap_tlb_asid_check();
796 } 792 }
797#if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 793#if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN)
798 pm->pm_shootdown_pending = 1; 794 pm->pm_shootdown_pending = 1;
799#endif 795#endif
800 TLBINFO_UNLOCK(ti); 796 TLBINFO_UNLOCK(ti);
801 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 797 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
802} 798}
803 799
804static inline void 800static inline void
805pmap_tlb_asid_alloc(struct pmap_tlb_info *ti, pmap_t pm, 801pmap_tlb_asid_alloc(struct pmap_tlb_info *ti, pmap_t pm,
806 struct pmap_asid_info *pai) 802 struct pmap_asid_info *pai)
807{ 803{
808 /* 804 /*
809 * We shouldn't have an ASID assigned, and thusly must not be onproc 805 * We shouldn't have an ASID assigned, and thusly must not be onproc
810 * nor active. 806 * nor active.
811 */ 807 */
812 KASSERT(pm != pmap_kernel()); 808 KASSERT(pm != pmap_kernel());
813 KASSERT(pai->pai_asid == 0); 809 KASSERT(pai->pai_asid == 0);
814 KASSERT(pai->pai_link.le_prev == NULL); 810 KASSERT(pai->pai_link.le_prev == NULL);
815#if defined(MULTIPROCESSOR) 811#if defined(MULTIPROCESSOR)
816 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti)); 812 KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
817 KASSERT(!pmap_tlb_intersecting_active_p(pm, ti)); 813 KASSERT(!pmap_tlb_intersecting_active_p(pm, ti));
818#endif 814#endif
819 KASSERT(ti->ti_asids_free > 0); 815 KASSERT(ti->ti_asids_free > 0);
820 KASSERT(ti->ti_asid_hint > KERNEL_PID); 816 KASSERT(ti->ti_asid_hint > KERNEL_PID);
821 817
822 /* 818 /*
823 * If the last ASID allocated was the maximum ASID, then the 819 * If the last ASID allocated was the maximum ASID, then the
824 * hint will be out of range. Reset the hint to first 820 * hint will be out of range. Reset the hint to first
825 * available ASID. 821 * available ASID.
826 */ 822 */
827 if (PMAP_TLB_FLUSH_ASID_ON_RESET 823 if (PMAP_TLB_FLUSH_ASID_ON_RESET
828 && ti->ti_asid_hint > ti->ti_asid_max) { 824 && ti->ti_asid_hint > ti->ti_asid_max) {
829 ti->ti_asid_hint = KERNEL_PID + 1; 825 ti->ti_asid_hint = KERNEL_PID + 1;
830 } 826 }
831 KASSERTMSG(ti->ti_asid_hint <= ti->ti_asid_max, "hint %u", 827 KASSERTMSG(ti->ti_asid_hint <= ti->ti_asid_max, "hint %u",
832 ti->ti_asid_hint); 828 ti->ti_asid_hint);
833 829
834 /* 830 /*
835 * Let's see if the hinted ASID is free. If not search for 831 * Let's see if the hinted ASID is free. If not search for
836 * a new one. 832 * a new one.
837 */ 833 */
838 if (__predict_true(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) { 834 if (__predict_true(TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint))) {
839 const size_t nbpw = NBBY * sizeof(ti->ti_asid_bitmap._b[0]); 835 const size_t nbpw = NBBY * sizeof(ti->ti_asid_bitmap._b[0]);
840 size_t i; 836 size_t i;
841 u_long bits; 837 u_long bits;
842 for (i = 0; (bits = ~ti->ti_asid_bitmap._b[i]) == 0; i++) { 838 for (i = 0; (bits = ~ti->ti_asid_bitmap._b[i]) == 0; i++) {
843 KASSERT(i < __arraycount(ti->ti_asid_bitmap._b) - 1); 839 KASSERT(i < __arraycount(ti->ti_asid_bitmap._b) - 1);
844 } 840 }
845 /* 841 /*
846 * ffs wants to find the first bit set while we want 842 * ffs wants to find the first bit set while we want
847 * to find the first bit cleared. 843 * to find the first bit cleared.
848 */ 844 */
849 const u_int n = __builtin_ffsl(bits) - 1; 845 const u_int n = __builtin_ffsl(bits) - 1;
850 KASSERTMSG((bits << (nbpw - (n+1))) == (1ul << (nbpw-1)), 846 KASSERTMSG((bits << (nbpw - (n+1))) == (1ul << (nbpw-1)),
851 "n %u bits %#lx", n, bits); 847 "n %u bits %#lx", n, bits);
852 KASSERT(n < nbpw); 848 KASSERT(n < nbpw);
853 ti->ti_asid_hint = n + i * nbpw; 849 ti->ti_asid_hint = n + i * nbpw;
854 } 850 }
855 851
856 KASSERT(ti->ti_asid_hint > KERNEL_PID); 852 KASSERT(ti->ti_asid_hint > KERNEL_PID);
857 KASSERT(ti->ti_asid_hint <= ti->ti_asid_max); 853 KASSERT(ti->ti_asid_hint <= ti->ti_asid_max);
858 KASSERTMSG(PMAP_TLB_FLUSH_ASID_ON_RESET 854 KASSERTMSG(PMAP_TLB_FLUSH_ASID_ON_RESET
859 || TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint - 1), 855 || TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint - 1),
860 "hint %u bitmap %p", ti->ti_asid_hint, &ti->ti_asid_bitmap); 856 "hint %u bitmap %p", ti->ti_asid_hint, &ti->ti_asid_bitmap);
861 KASSERTMSG(!TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint), 857 KASSERTMSG(!TLBINFO_ASID_INUSE_P(ti, ti->ti_asid_hint),
862 "hint %u bitmap %p", ti->ti_asid_hint, &ti->ti_asid_bitmap); 858 "hint %u bitmap %p", ti->ti_asid_hint, &ti->ti_asid_bitmap);
863 859
864 /* 860 /*
865 * The hint contains our next ASID so take it and advance the hint. 861 * The hint contains our next ASID so take it and advance the hint.
866 * Mark it as used and insert the pai into the list of active asids. 862 * Mark it as used and insert the pai into the list of active asids.
867 * There is also one less asid free in this TLB. 863 * There is also one less asid free in this TLB.
868 */ 864 */
869 pai->pai_asid = ti->ti_asid_hint++; 865 pai->pai_asid = ti->ti_asid_hint++;
870#ifdef MULTIPROCESSOR 866#ifdef MULTIPROCESSOR
871 if (PMAP_TLB_FLUSH_ASID_ON_RESET) { 867 if (PMAP_TLB_FLUSH_ASID_ON_RESET) {
872 /* 868 /*
873 * Clean the new ASID from the TLB. 869 * Clean the new ASID from the TLB.
874 */ 870 */
875 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 871 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
876 } 872 }
877#endif 873#endif
878 TLBINFO_ASID_MARK_USED(ti, pai->pai_asid); 874 TLBINFO_ASID_MARK_USED(ti, pai->pai_asid);
879 LIST_INSERT_HEAD(&ti->ti_pais, pai, pai_link); 875 LIST_INSERT_HEAD(&ti->ti_pais, pai, pai_link);
880 ti->ti_asids_free--; 876 ti->ti_asids_free--;
881 877
882#if defined(MULTIPROCESSOR) 878#if defined(MULTIPROCESSOR)
883 /* 879 /*
884 * Mark that we now have an active ASID for all CPUs sharing this TLB. 880 * Mark that we now have an active ASID for all CPUs sharing this TLB.
885 * The bits in pm_active belonging to this TLB can only be changed 881 * The bits in pm_active belonging to this TLB can only be changed
886 * while this TLBs lock is held. 882 * while this TLBs lock is held.
887 */ 883 */
888#if PMAP_TLB_MAX == 1 884#if PMAP_TLB_MAX == 1
889 kcpuset_copy(pm->pm_active, kcpuset_running); 885 kcpuset_copy(pm->pm_active, kcpuset_running);
890#else 886#else
891 kcpuset_merge(pm->pm_active, ti->ti_kcpuset); 887 kcpuset_merge(pm->pm_active, ti->ti_kcpuset);
892#endif 888#endif
893#endif 889#endif
894} 890}
895 891
896/* 892/*
897 * Acquire a TLB address space tag (called ASID or TLBPID) and return it. 893 * Acquire a TLB address space tag (called ASID or TLBPID) and return it.
898 * ASID might have already been previously acquired. 894 * ASID might have already been previously acquired.
899 */ 895 */
900void 896void
901pmap_tlb_asid_acquire(pmap_t pm, struct lwp *l) 897pmap_tlb_asid_acquire(pmap_t pm, struct lwp *l)
902{ 898{
903 struct cpu_info * const ci = l->l_cpu; 899 struct cpu_info * const ci = l->l_cpu;
904 struct pmap_tlb_info * const ti = cpu_tlb_info(ci); 900 struct pmap_tlb_info * const ti = cpu_tlb_info(ci);
905 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 901 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
906 902
907 UVMHIST_FUNC(__func__); 903 UVMHIST_FUNC(__func__);
908 UVMHIST_CALLARGS(maphist, "(pm=%#jx, l=%#jx, ti=%#jx)", (uintptr_t)pm, 904 UVMHIST_CALLARGS(maphist, "(pm=%#jx, l=%#jx, ti=%#jx)", (uintptr_t)pm,
909 (uintptr_t)l, (uintptr_t)ti, 0); 905 (uintptr_t)l, (uintptr_t)ti, 0);
910 906
911 KASSERT(kpreempt_disabled()); 907 KASSERT(kpreempt_disabled());
912 908
913 /* 909 /*
914 * Kernels use a fixed ASID and thus doesn't need to acquire one. 910 * Kernels use a fixed ASID and thus doesn't need to acquire one.
915 */ 911 */
916 if (pm == pmap_kernel()) { 912 if (pm == pmap_kernel()) {
917 UVMHIST_LOG(maphist, " <-- done (kernel)", 0, 0, 0, 0); 913 UVMHIST_LOG(maphist, " <-- done (kernel)", 0, 0, 0, 0);
918 return; 914 return;
919 } 915 }
920 916
921 TLBINFO_LOCK(ti); 917 TLBINFO_LOCK(ti);
922 KASSERT(pai->pai_asid <= KERNEL_PID || pai->pai_link.le_prev != NULL); 918 KASSERT(pai->pai_asid <= KERNEL_PID || pai->pai_link.le_prev != NULL);
923 KASSERT(pai->pai_asid > KERNEL_PID || pai->pai_link.le_prev == NULL); 919 KASSERT(pai->pai_asid > KERNEL_PID || pai->pai_link.le_prev == NULL);
924 pmap_tlb_pai_check(ti, true); 920 pmap_tlb_pai_check(ti, true);
925 if (__predict_false(!PMAP_PAI_ASIDVALID_P(pai, ti))) { 921 if (__predict_false(!PMAP_PAI_ASIDVALID_P(pai, ti))) {
926 /* 922 /*
927 * If we've run out ASIDs, reinitialize the ASID space. 923 * If we've run out ASIDs, reinitialize the ASID space.
928 */ 924 */
929 if (__predict_false(tlbinfo_noasids_p(ti))) { 925 if (__predict_false(tlbinfo_noasids_p(ti))) {
930 KASSERT(l == curlwp); 926 KASSERT(l == curlwp);
931 UVMHIST_LOG(maphist, " asid reinit", 0, 0, 0, 0); 927 UVMHIST_LOG(maphist, " asid reinit", 0, 0, 0, 0);
932 pmap_tlb_asid_reinitialize(ti, TLBINV_NOBODY); 928 pmap_tlb_asid_reinitialize(ti, TLBINV_NOBODY);
933 KASSERT(!tlbinfo_noasids_p(ti)); 929 KASSERT(!tlbinfo_noasids_p(ti));
934 } 930 }
935 931
936 /* 932 /*
937 * Get an ASID. 933 * Get an ASID.
938 */ 934 */
939 pmap_tlb_asid_alloc(ti, pm, pai); 935 pmap_tlb_asid_alloc(ti, pm, pai);
940 UVMHIST_LOG(maphist, "allocated asid %#jx", pai->pai_asid, 936 UVMHIST_LOG(maphist, "allocated asid %#jx", pai->pai_asid,
941 0, 0, 0); 937 0, 0, 0);
942 } 938 }
943 pmap_tlb_pai_check(ti, true); 939 pmap_tlb_pai_check(ti, true);
944#if defined(MULTIPROCESSOR) 940#if defined(MULTIPROCESSOR)
945 KASSERT(kcpuset_isset(pm->pm_active, cpu_index(ci))); 941 KASSERT(kcpuset_isset(pm->pm_active, cpu_index(ci)));
946#endif 942#endif
947 943
948 if (l == curlwp) { 944 if (l == curlwp) {
949#if defined(MULTIPROCESSOR) 945#if defined(MULTIPROCESSOR)
950 /* 946 /*
951 * The bits in pm_onproc belonging to this TLB can only 947 * The bits in pm_onproc belonging to this TLB can only
952 * be changed while this TLBs lock is held unless atomic 948 * be changed while this TLBs lock is held unless atomic
953 * operations are used. 949 * operations are used.
954 */ 950 */
955 KASSERT(pm != pmap_kernel()); 951 KASSERT(pm != pmap_kernel());
956 kcpuset_atomic_set(pm->pm_onproc, cpu_index(ci)); 952 kcpuset_atomic_set(pm->pm_onproc, cpu_index(ci));
957#endif 953#endif
958 ci->ci_pmap_asid_cur = pai->pai_asid; 954 ci->ci_pmap_asid_cur = pai->pai_asid;
959 UVMHIST_LOG(maphist, "setting asid to %#jx", pai->pai_asid, 955 UVMHIST_LOG(maphist, "setting asid to %#jx", pai->pai_asid,
960 0, 0, 0); 956 0, 0, 0);
961 tlb_set_asid(pai->pai_asid, pm); 957 tlb_set_asid(pai->pai_asid, pm);
962 pmap_tlb_asid_check(); 958 pmap_tlb_asid_check();
963 } else { 959 } else {
964 printf("%s: l (%p) != curlwp %p\n", __func__, l, curlwp); 960 printf("%s: l (%p) != curlwp %p\n", __func__, l, curlwp);
965 } 961 }
966 TLBINFO_UNLOCK(ti); 962 TLBINFO_UNLOCK(ti);
967 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 963 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
968} 964}
969 965
970void 966void
971pmap_tlb_asid_deactivate(pmap_t pm) 967pmap_tlb_asid_deactivate(pmap_t pm)
972{ 968{
973 UVMHIST_FUNC(__func__); 969 UVMHIST_FUNC(__func__);
974 UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0); 970 UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pm, 0, 0, 0);
975 971
976 KASSERT(kpreempt_disabled()); 972 KASSERT(kpreempt_disabled());
977#if defined(MULTIPROCESSOR) 973#if defined(MULTIPROCESSOR)
978 /* 974 /*
979 * The kernel pmap is aways onproc and active and must never have 975 * The kernel pmap is aways onproc and active and must never have
980 * those bits cleared. If pmap_remove_all was called, it has already 976 * those bits cleared. If pmap_remove_all was called, it has already
981 * deactivated the pmap and thusly onproc will be 0 so there's nothing 977 * deactivated the pmap and thusly onproc will be 0 so there's nothing
982 * to do. 978 * to do.
983 */ 979 */
984 if (pm != pmap_kernel() && !kcpuset_iszero(pm->pm_onproc)) { 980 if (pm != pmap_kernel() && !kcpuset_iszero(pm->pm_onproc)) {
985 struct cpu_info * const ci = curcpu(); 981 struct cpu_info * const ci = curcpu();
986 KASSERT(!cpu_intr_p()); 982 KASSERT(!cpu_intr_p());
987 KASSERTMSG(kcpuset_isset(pm->pm_onproc, cpu_index(ci)), 983 KASSERTMSG(kcpuset_isset(pm->pm_onproc, cpu_index(ci)),
988 "%s: pmap %p onproc %p doesn't include cpu %d (%p)", 984 "%s: pmap %p onproc %p doesn't include cpu %d (%p)",
989 __func__, pm, pm->pm_onproc, cpu_index(ci), ci); 985 __func__, pm, pm->pm_onproc, cpu_index(ci), ci);
990 /* 986 /*
991 * The bits in pm_onproc that belong to this TLB can 987 * The bits in pm_onproc that belong to this TLB can
992 * be changed while this TLBs lock is not held as long 988 * be changed while this TLBs lock is not held as long
993 * as we use atomic ops. 989 * as we use atomic ops.
994 */ 990 */
995 kcpuset_atomic_clear(pm->pm_onproc, cpu_index(ci)); 991 kcpuset_atomic_clear(pm->pm_onproc, cpu_index(ci));
996 } 992 }
997#endif 993#endif
998 curcpu()->ci_pmap_asid_cur = KERNEL_PID; 994 curcpu()->ci_pmap_asid_cur = KERNEL_PID;
999 tlb_set_asid(KERNEL_PID, pmap_kernel()); 995 tlb_set_asid(KERNEL_PID, pmap_kernel());
1000 996
1001 pmap_tlb_pai_check(cpu_tlb_info(curcpu()), false); 997 pmap_tlb_pai_check(cpu_tlb_info(curcpu()), false);
1002#if defined(DEBUG) 998#if defined(DEBUG)
1003 pmap_tlb_asid_check(); 999 pmap_tlb_asid_check();
1004#endif 1000#endif
1005 UVMHIST_LOG(maphist, " <-- done (pm=%#jx)", (uintptr_t)pm, 0, 0, 0); 1001 UVMHIST_LOG(maphist, " <-- done (pm=%#jx)", (uintptr_t)pm, 0, 0, 0);
1006} 1002}
1007 1003
1008void 1004void
1009pmap_tlb_asid_release_all(struct pmap *pm) 1005pmap_tlb_asid_release_all(struct pmap *pm)
1010{ 1006{
1011 UVMHIST_FUNC(__func__); 1007 UVMHIST_FUNC(__func__);
1012 UVMHIST_CALLARGS(maphist, "(pm=%#jx)", (uintptr_t)pm, 0, 0, 0); 1008 UVMHIST_CALLARGS(maphist, "(pm=%#jx)", (uintptr_t)pm, 0, 0, 0);
1013 1009
1014 KASSERT(pm != pmap_kernel()); 1010 KASSERT(pm != pmap_kernel());
1015#if defined(MULTIPROCESSOR) 1011#if defined(MULTIPROCESSOR)
1016 //KASSERT(!kcpuset_iszero(pm->pm_onproc)); // XXX 1012 //KASSERT(!kcpuset_iszero(pm->pm_onproc)); // XXX
1017 struct cpu_info * const ci __diagused = curcpu(); 1013 struct cpu_info * const ci __diagused = curcpu();
1018 KASSERT(!kcpuset_isotherset(pm->pm_onproc, cpu_index(ci))); 1014 KASSERT(!kcpuset_isotherset(pm->pm_onproc, cpu_index(ci)));
1019#if PMAP_TLB_MAX > 1 1015#if PMAP_TLB_MAX > 1
1020 for (u_int i = 0; !kcpuset_iszero(pm->pm_active); i++) { 1016 for (u_int i = 0; !kcpuset_iszero(pm->pm_active); i++) {
1021 KASSERT(i < pmap_ntlbs); 1017 KASSERT(i < pmap_ntlbs);
1022 struct pmap_tlb_info * const ti = pmap_tlbs[i]; 1018 struct pmap_tlb_info * const ti = pmap_tlbs[i];
1023#else 1019#else
1024 struct pmap_tlb_info * const ti = &pmap_tlb0_info; 1020 struct pmap_tlb_info * const ti = &pmap_tlb0_info;
1025#endif 1021#endif
1026 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 1022 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
1027 TLBINFO_LOCK(ti); 1023 TLBINFO_LOCK(ti);
1028 if (PMAP_PAI_ASIDVALID_P(pai, ti)) { 1024 if (PMAP_PAI_ASIDVALID_P(pai, ti)) {
1029 /* 1025 /*
1030 * This pmap should not be in use by any other cpu so 1026 * This pmap should not be in use by any other cpu so
1031 * we can just reset and be happy. 1027 * we can just reset and be happy.
1032 */ 1028 */
1033 if (ti->ti_victim == pm) 1029 if (ti->ti_victim == pm)
1034 ti->ti_victim = NULL; 1030 ti->ti_victim = NULL;
1035 pmap_tlb_pai_reset(ti, pai, pm); 1031 pmap_tlb_pai_reset(ti, pai, pm);
1036 } 1032 }
1037 KASSERT(pai->pai_link.le_prev == NULL); 1033 KASSERT(pai->pai_link.le_prev == NULL);
1038 TLBINFO_UNLOCK(ti); 1034 TLBINFO_UNLOCK(ti);
1039#if PMAP_TLB_MAX > 1 1035#if PMAP_TLB_MAX > 1
1040 } 1036 }
1041#endif 1037#endif
1042#ifdef DIAGNOSTIC 1038#ifdef DIAGNOSTIC
1043 for (size_t i = 0; i < (PMAP_TLB_MAX > 1 ? pmap_ntlbs : 1); i++) { 1039 for (size_t i = 0; i < (PMAP_TLB_MAX > 1 ? pmap_ntlbs : 1); i++) {
1044 KASSERTMSG(pm->pm_pai[i].pai_asid == 0, 1040 KASSERTMSG(pm->pm_pai[i].pai_asid == 0,
1045 "pm %p i %zu asid %u", 1041 "pm %p i %zu asid %u",
1046 pm, i, pm->pm_pai[i].pai_asid); 1042 pm, i, pm->pm_pai[i].pai_asid);
1047 } 1043 }
1048#endif 1044#endif
1049#else 1045#else
1050 /* 1046 /*
1051 * Handle the case of an UP kernel which only has, at most, one TLB. 1047 * Handle the case of an UP kernel which only has, at most, one TLB.
1052 * If the pmap has an ASID allocated, free it. 1048 * If the pmap has an ASID allocated, free it.
1053 */ 1049 */
1054 struct pmap_tlb_info * const ti = &pmap_tlb0_info; 1050 struct pmap_tlb_info * const ti = &pmap_tlb0_info;
1055 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 1051 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
1056 TLBINFO_LOCK(ti); 1052 TLBINFO_LOCK(ti);
1057 if (pai->pai_asid > KERNEL_PID) { 1053 if (pai->pai_asid > KERNEL_PID) {
1058 if (curcpu()->ci_pmap_asid_cur == pai->pai_asid) { 1054 if (curcpu()->ci_pmap_asid_cur == pai->pai_asid) {
1059 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid); 1055 tlb_invalidate_asids(pai->pai_asid, pai->pai_asid);
1060 } else { 1056 } else {
1061 pmap_tlb_pai_reset(ti, pai, pm); 1057 pmap_tlb_pai_reset(ti, pai, pm);
1062 } 1058 }
1063 } 1059 }
1064 TLBINFO_UNLOCK(ti); 1060 TLBINFO_UNLOCK(ti);
1065#endif /* MULTIPROCESSOR */ 1061#endif /* MULTIPROCESSOR */
1066 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0); 1062 UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
1067} 1063}
1068 1064
1069void 1065void
1070pmap_tlb_asid_check(void) 1066pmap_tlb_asid_check(void)
1071{ 1067{
1072#ifdef DEBUG 1068#ifdef DEBUG
1073 kpreempt_disable(); 1069 kpreempt_disable();
1074 const tlb_asid_t asid __debugused = tlb_get_asid(); 1070 const tlb_asid_t asid __debugused = tlb_get_asid();
1075 KDASSERTMSG(asid == curcpu()->ci_pmap_asid_cur, 1071 KDASSERTMSG(asid == curcpu()->ci_pmap_asid_cur,
1076 "%s: asid (%#x) != current asid (%#x)", 1072 "%s: asid (%#x) != current asid (%#x)",
1077 __func__, asid, curcpu()->ci_pmap_asid_cur); 1073 __func__, asid, curcpu()->ci_pmap_asid_cur);
1078 kpreempt_enable(); 1074 kpreempt_enable();
1079#endif 1075#endif
1080} 1076}
1081 1077
1082#ifdef DEBUG 1078#ifdef DEBUG
1083void 1079void
1084pmap_tlb_check(pmap_t pm, bool (*func)(void *, vaddr_t, tlb_asid_t, pt_entry_t)) 1080pmap_tlb_check(pmap_t pm, bool (*func)(void *, vaddr_t, tlb_asid_t, pt_entry_t))
1085{ 1081{
1086 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu()); 1082 struct pmap_tlb_info * const ti = cpu_tlb_info(curcpu());
1087 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti); 1083 struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
1088 TLBINFO_LOCK(ti); 1084 TLBINFO_LOCK(ti);
1089 if (pm == pmap_kernel() || pai->pai_asid > KERNEL_PID) 1085 if (pm == pmap_kernel() || pai->pai_asid > KERNEL_PID)
1090 tlb_walk(pm, func); 1086 tlb_walk(pm, func);
1091 TLBINFO_UNLOCK(ti); 1087 TLBINFO_UNLOCK(ti);
1092} 1088}
1093#endif /* DEBUG */ 1089#endif /* DEBUG */