| @@ -1,620 +1,619 @@ | | | @@ -1,620 +1,619 @@ |
1 | /* $NetBSD: pmap.h,v 1.122 2020/05/27 19:33:40 ad Exp $ */ | | 1 | /* $NetBSD: pmap.h,v 1.123 2020/06/24 18:09:37 maxv Exp $ */ |
2 | | | 2 | |
3 | /* | | 3 | /* |
4 | * Copyright (c) 1997 Charles D. Cranor and Washington University. | | 4 | * Copyright (c) 1997 Charles D. Cranor and Washington University. |
5 | * All rights reserved. | | 5 | * All rights reserved. |
6 | * | | 6 | * |
7 | * Redistribution and use in source and binary forms, with or without | | 7 | * Redistribution and use in source and binary forms, with or without |
8 | * modification, are permitted provided that the following conditions | | 8 | * modification, are permitted provided that the following conditions |
9 | * are met: | | 9 | * are met: |
10 | * 1. Redistributions of source code must retain the above copyright | | 10 | * 1. Redistributions of source code must retain the above copyright |
11 | * notice, this list of conditions and the following disclaimer. | | 11 | * notice, this list of conditions and the following disclaimer. |
12 | * 2. Redistributions in binary form must reproduce the above copyright | | 12 | * 2. Redistributions in binary form must reproduce the above copyright |
13 | * notice, this list of conditions and the following disclaimer in the | | 13 | * notice, this list of conditions and the following disclaimer in the |
14 | * documentation and/or other materials provided with the distribution. | | 14 | * documentation and/or other materials provided with the distribution. |
15 | * | | 15 | * |
16 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR | | 16 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
17 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES | | 17 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
18 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. | | 18 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
19 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, | | 19 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
20 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | | 20 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
21 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | | 21 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
22 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | | 22 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
23 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | | 23 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
24 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF | | 24 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
25 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | | 25 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
26 | */ | | 26 | */ |
27 | | | 27 | |
28 | /* | | 28 | /* |
29 | * Copyright (c) 2001 Wasabi Systems, Inc. | | 29 | * Copyright (c) 2001 Wasabi Systems, Inc. |
30 | * All rights reserved. | | 30 | * All rights reserved. |
31 | * | | 31 | * |
32 | * Written by Frank van der Linden for Wasabi Systems, Inc. | | 32 | * Written by Frank van der Linden for Wasabi Systems, Inc. |
33 | * | | 33 | * |
34 | * Redistribution and use in source and binary forms, with or without | | 34 | * Redistribution and use in source and binary forms, with or without |
35 | * modification, are permitted provided that the following conditions | | 35 | * modification, are permitted provided that the following conditions |
36 | * are met: | | 36 | * are met: |
37 | * 1. Redistributions of source code must retain the above copyright | | 37 | * 1. Redistributions of source code must retain the above copyright |
38 | * notice, this list of conditions and the following disclaimer. | | 38 | * notice, this list of conditions and the following disclaimer. |
39 | * 2. Redistributions in binary form must reproduce the above copyright | | 39 | * 2. Redistributions in binary form must reproduce the above copyright |
40 | * notice, this list of conditions and the following disclaimer in the | | 40 | * notice, this list of conditions and the following disclaimer in the |
41 | * documentation and/or other materials provided with the distribution. | | 41 | * documentation and/or other materials provided with the distribution. |
42 | * 3. All advertising materials mentioning features or use of this software | | 42 | * 3. All advertising materials mentioning features or use of this software |
43 | * must display the following acknowledgement: | | 43 | * must display the following acknowledgement: |
44 | * This product includes software developed for the NetBSD Project by | | 44 | * This product includes software developed for the NetBSD Project by |
45 | * Wasabi Systems, Inc. | | 45 | * Wasabi Systems, Inc. |
46 | * 4. The name of Wasabi Systems, Inc. may not be used to endorse | | 46 | * 4. The name of Wasabi Systems, Inc. may not be used to endorse |
47 | * or promote products derived from this software without specific prior | | 47 | * or promote products derived from this software without specific prior |
48 | * written permission. | | 48 | * written permission. |
49 | * | | 49 | * |
50 | * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND | | 50 | * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND |
51 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED | | 51 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
52 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | | 52 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
53 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC | | 53 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC |
54 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | | 54 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
55 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | | 55 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
56 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | | 56 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
57 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | | 57 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
58 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | | 58 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
59 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | | 59 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
60 | * POSSIBILITY OF SUCH DAMAGE. | | 60 | * POSSIBILITY OF SUCH DAMAGE. |
61 | */ | | 61 | */ |
62 | | | 62 | |
63 | /* | | 63 | /* |
64 | * pmap.h: see pmap.c for the history of this pmap module. | | 64 | * pmap.h: see pmap.c for the history of this pmap module. |
65 | */ | | 65 | */ |
66 | | | 66 | |
67 | #ifndef _X86_PMAP_H_ | | 67 | #ifndef _X86_PMAP_H_ |
68 | #define _X86_PMAP_H_ | | 68 | #define _X86_PMAP_H_ |
69 | | | 69 | |
70 | /* | | 70 | /* |
71 | * pl*_pi: index in the ptp page for a pde mapping a VA. | | 71 | * pl*_pi: index in the ptp page for a pde mapping a VA. |
72 | * (pl*_i below is the index in the virtual array of all pdes per level) | | 72 | * (pl*_i below is the index in the virtual array of all pdes per level) |
73 | */ | | 73 | */ |
74 | #define pl1_pi(VA) (((VA_SIGN_POS(VA)) & L1_MASK) >> L1_SHIFT) | | 74 | #define pl1_pi(VA) (((VA_SIGN_POS(VA)) & L1_MASK) >> L1_SHIFT) |
75 | #define pl2_pi(VA) (((VA_SIGN_POS(VA)) & L2_MASK) >> L2_SHIFT) | | 75 | #define pl2_pi(VA) (((VA_SIGN_POS(VA)) & L2_MASK) >> L2_SHIFT) |
76 | #define pl3_pi(VA) (((VA_SIGN_POS(VA)) & L3_MASK) >> L3_SHIFT) | | 76 | #define pl3_pi(VA) (((VA_SIGN_POS(VA)) & L3_MASK) >> L3_SHIFT) |
77 | #define pl4_pi(VA) (((VA_SIGN_POS(VA)) & L4_MASK) >> L4_SHIFT) | | 77 | #define pl4_pi(VA) (((VA_SIGN_POS(VA)) & L4_MASK) >> L4_SHIFT) |
78 | #define pl_pi(va, lvl) \ | | 78 | #define pl_pi(va, lvl) \ |
79 | (((VA_SIGN_POS(va)) & ptp_masks[(lvl)-1]) >> ptp_shifts[(lvl)-1]) | | 79 | (((VA_SIGN_POS(va)) & ptp_masks[(lvl)-1]) >> ptp_shifts[(lvl)-1]) |
80 | | | 80 | |
81 | /* | | 81 | /* |
82 | * pl*_i: generate index into pde/pte arrays in virtual space | | 82 | * pl*_i: generate index into pde/pte arrays in virtual space |
83 | * | | 83 | * |
84 | * pl_i(va, X) == plX_i(va) <= pl_i_roundup(va, X) | | 84 | * pl_i(va, X) == plX_i(va) <= pl_i_roundup(va, X) |
85 | */ | | 85 | */ |
86 | #define pl1_i(VA) (((VA_SIGN_POS(VA)) & L1_FRAME) >> L1_SHIFT) | | 86 | #define pl1_i(VA) (((VA_SIGN_POS(VA)) & L1_FRAME) >> L1_SHIFT) |
87 | #define pl2_i(VA) (((VA_SIGN_POS(VA)) & L2_FRAME) >> L2_SHIFT) | | 87 | #define pl2_i(VA) (((VA_SIGN_POS(VA)) & L2_FRAME) >> L2_SHIFT) |
88 | #define pl3_i(VA) (((VA_SIGN_POS(VA)) & L3_FRAME) >> L3_SHIFT) | | 88 | #define pl3_i(VA) (((VA_SIGN_POS(VA)) & L3_FRAME) >> L3_SHIFT) |
89 | #define pl4_i(VA) (((VA_SIGN_POS(VA)) & L4_FRAME) >> L4_SHIFT) | | 89 | #define pl4_i(VA) (((VA_SIGN_POS(VA)) & L4_FRAME) >> L4_SHIFT) |
90 | #define pl_i(va, lvl) \ | | 90 | #define pl_i(va, lvl) \ |
91 | (((VA_SIGN_POS(va)) & ptp_frames[(lvl)-1]) >> ptp_shifts[(lvl)-1]) | | 91 | (((VA_SIGN_POS(va)) & ptp_frames[(lvl)-1]) >> ptp_shifts[(lvl)-1]) |
92 | | | 92 | |
93 | #define pl_i_roundup(va, lvl) pl_i((va)+ ~ptp_frames[(lvl)-1], (lvl)) | | 93 | #define pl_i_roundup(va, lvl) pl_i((va)+ ~ptp_frames[(lvl)-1], (lvl)) |
94 | | | 94 | |
95 | /* | | 95 | /* |
96 | * PTP macros: | | 96 | * PTP macros: |
97 | * a PTP's index is the PD index of the PDE that points to it | | 97 | * a PTP's index is the PD index of the PDE that points to it |
98 | * a PTP's offset is the byte-offset in the PTE space that this PTP is at | | 98 | * a PTP's offset is the byte-offset in the PTE space that this PTP is at |
99 | * a PTP's VA is the first VA mapped by that PTP | | 99 | * a PTP's VA is the first VA mapped by that PTP |
100 | */ | | 100 | */ |
101 | | | 101 | |
102 | #define ptp_va2o(va, lvl) (pl_i(va, (lvl)+1) * PAGE_SIZE) | | 102 | #define ptp_va2o(va, lvl) (pl_i(va, (lvl)+1) * PAGE_SIZE) |
103 | | | 103 | |
104 | /* size of a PDP: usually one page, except for PAE */ | | 104 | /* size of a PDP: usually one page, except for PAE */ |
105 | #ifdef PAE | | 105 | #ifdef PAE |
106 | #define PDP_SIZE 4 | | 106 | #define PDP_SIZE 4 |
107 | #else | | 107 | #else |
108 | #define PDP_SIZE 1 | | 108 | #define PDP_SIZE 1 |
109 | #endif | | 109 | #endif |
110 | | | 110 | |
111 | | | 111 | |
112 | #if defined(_KERNEL) | | 112 | #if defined(_KERNEL) |
113 | #include <sys/kcpuset.h> | | 113 | #include <sys/kcpuset.h> |
114 | #include <sys/rwlock.h> | | 114 | #include <sys/rwlock.h> |
115 | #include <x86/pmap_pv.h> | | 115 | #include <x86/pmap_pv.h> |
116 | #include <uvm/pmap/pmap_pvt.h> | | 116 | #include <uvm/pmap/pmap_pvt.h> |
117 | | | 117 | |
118 | #define PATENTRY(n, type) (type << ((n) * 8)) | | 118 | #define PATENTRY(n, type) (type << ((n) * 8)) |
119 | #define PAT_UC 0x0ULL | | 119 | #define PAT_UC 0x0ULL |
120 | #define PAT_WC 0x1ULL | | 120 | #define PAT_WC 0x1ULL |
121 | #define PAT_WT 0x4ULL | | 121 | #define PAT_WT 0x4ULL |
122 | #define PAT_WP 0x5ULL | | 122 | #define PAT_WP 0x5ULL |
123 | #define PAT_WB 0x6ULL | | 123 | #define PAT_WB 0x6ULL |
124 | #define PAT_UCMINUS 0x7ULL | | 124 | #define PAT_UCMINUS 0x7ULL |
125 | | | 125 | |
126 | #define BTSEG_NONE 0 | | 126 | #define BTSEG_NONE 0 |
127 | #define BTSEG_TEXT 1 | | 127 | #define BTSEG_TEXT 1 |
128 | #define BTSEG_RODATA 2 | | 128 | #define BTSEG_RODATA 2 |
129 | #define BTSEG_DATA 3 | | 129 | #define BTSEG_DATA 3 |
130 | #define BTSPACE_NSEGS 64 | | 130 | #define BTSPACE_NSEGS 64 |
131 | | | 131 | |
132 | struct bootspace { | | 132 | struct bootspace { |
133 | struct { | | 133 | struct { |
134 | vaddr_t va; | | 134 | vaddr_t va; |
135 | paddr_t pa; | | 135 | paddr_t pa; |
136 | size_t sz; | | 136 | size_t sz; |
137 | } head; | | 137 | } head; |
138 | | | 138 | |
139 | /* Kernel segments. */ | | 139 | /* Kernel segments. */ |
140 | struct { | | 140 | struct { |
141 | int type; | | 141 | int type; |
142 | vaddr_t va; | | 142 | vaddr_t va; |
143 | paddr_t pa; | | 143 | paddr_t pa; |
144 | size_t sz; | | 144 | size_t sz; |
145 | } segs[BTSPACE_NSEGS]; | | 145 | } segs[BTSPACE_NSEGS]; |
146 | | | 146 | |
147 | /* | | 147 | /* |
148 | * The area used by the early kernel bootstrap. It contains the kernel | | 148 | * The area used by the early kernel bootstrap. It contains the kernel |
149 | * symbols, the preloaded modules, the bootstrap tables, and the ISA I/O | | 149 | * symbols, the preloaded modules, the bootstrap tables, and the ISA I/O |
150 | * mem. | | 150 | * mem. |
151 | */ | | 151 | */ |
152 | struct { | | 152 | struct { |
153 | vaddr_t va; | | 153 | vaddr_t va; |
154 | paddr_t pa; | | 154 | paddr_t pa; |
155 | size_t sz; | | 155 | size_t sz; |
156 | } boot; | | 156 | } boot; |
157 | | | 157 | |
158 | /* A magic VA usable by the bootstrap code. */ | | 158 | /* A magic VA usable by the bootstrap code. */ |
159 | vaddr_t spareva; | | 159 | vaddr_t spareva; |
160 | | | 160 | |
161 | /* Virtual address of the page directory. */ | | 161 | /* Virtual address of the page directory. */ |
162 | vaddr_t pdir; | | 162 | vaddr_t pdir; |
163 | | | 163 | |
164 | /* Area dedicated to kernel modules (amd64 only). */ | | 164 | /* Area dedicated to kernel modules (amd64 only). */ |
165 | vaddr_t smodule; | | 165 | vaddr_t smodule; |
166 | vaddr_t emodule; | | 166 | vaddr_t emodule; |
167 | }; | | 167 | }; |
168 | | | 168 | |
169 | #define SLAREA_USER 0 | | 169 | #define SLAREA_USER 0 |
170 | #define SLAREA_PTE 1 | | 170 | #define SLAREA_PTE 1 |
171 | #define SLAREA_MAIN 2 | | 171 | #define SLAREA_MAIN 2 |
172 | #define SLAREA_PCPU 3 | | 172 | #define SLAREA_PCPU 3 |
173 | #define SLAREA_DMAP 4 | | 173 | #define SLAREA_DMAP 4 |
174 | #define SLAREA_HYPV 5 | | 174 | #define SLAREA_HYPV 5 |
175 | #define SLAREA_ASAN 6 | | 175 | #define SLAREA_ASAN 6 |
176 | #define SLAREA_MSAN 7 | | 176 | #define SLAREA_MSAN 7 |
177 | #define SLAREA_KERN 8 | | 177 | #define SLAREA_KERN 8 |
178 | #define SLSPACE_NAREAS 9 | | 178 | #define SLSPACE_NAREAS 9 |
179 | | | 179 | |
180 | struct slotspace { | | 180 | struct slotspace { |
181 | struct { | | 181 | struct { |
182 | size_t sslot; /* start slot */ | | 182 | size_t sslot; /* start slot */ |
183 | size_t nslot; /* # of slots */ | | 183 | size_t nslot; /* # of slots */ |
184 | bool active; /* area is active */ | | 184 | bool active; /* area is active */ |
185 | } area[SLSPACE_NAREAS]; | | 185 | } area[SLSPACE_NAREAS]; |
186 | }; | | 186 | }; |
187 | | | 187 | |
188 | extern struct slotspace slotspace; | | 188 | extern struct slotspace slotspace; |
189 | | | 189 | |
190 | #ifndef MAXGDTSIZ | | 190 | #ifndef MAXGDTSIZ |
191 | #define MAXGDTSIZ 65536 /* XXX */ | | 191 | #define MAXGDTSIZ 65536 /* XXX */ |
192 | #endif | | 192 | #endif |
193 | | | 193 | |
194 | #ifndef MAX_USERLDT_SIZE | | 194 | #ifndef MAX_USERLDT_SIZE |
195 | #define MAX_USERLDT_SIZE PAGE_SIZE /* XXX */ | | 195 | #define MAX_USERLDT_SIZE PAGE_SIZE /* XXX */ |
196 | #endif | | 196 | #endif |
197 | | | 197 | |
198 | struct pcpu_entry { | | 198 | struct pcpu_entry { |
199 | uint8_t gdt[MAXGDTSIZ]; | | 199 | uint8_t gdt[MAXGDTSIZ]; |
200 | uint8_t ldt[MAX_USERLDT_SIZE]; | | 200 | uint8_t ldt[MAX_USERLDT_SIZE]; |
201 | uint8_t tss[PAGE_SIZE]; | | 201 | uint8_t tss[PAGE_SIZE]; |
202 | uint8_t ist0[PAGE_SIZE]; | | 202 | uint8_t ist0[PAGE_SIZE]; |
203 | uint8_t ist1[PAGE_SIZE]; | | 203 | uint8_t ist1[PAGE_SIZE]; |
204 | uint8_t ist2[PAGE_SIZE]; | | 204 | uint8_t ist2[PAGE_SIZE]; |
205 | uint8_t ist3[PAGE_SIZE]; | | 205 | uint8_t ist3[PAGE_SIZE]; |
206 | uint8_t rsp0[2 * PAGE_SIZE]; | | 206 | uint8_t rsp0[2 * PAGE_SIZE]; |
207 | } __packed; | | 207 | } __packed; |
208 | | | 208 | |
209 | struct pcpu_area { | | 209 | struct pcpu_area { |
210 | #ifdef SVS | | 210 | #ifdef SVS |
211 | uint8_t utls[PAGE_SIZE]; | | 211 | uint8_t utls[PAGE_SIZE]; |
212 | #endif | | 212 | #endif |
213 | uint8_t idt[PAGE_SIZE]; | | 213 | uint8_t idt[PAGE_SIZE]; |
214 | uint8_t ldt[PAGE_SIZE]; | | 214 | uint8_t ldt[PAGE_SIZE]; |
215 | struct pcpu_entry ent[MAXCPUS]; | | 215 | struct pcpu_entry ent[MAXCPUS]; |
216 | } __packed; | | 216 | } __packed; |
217 | | | 217 | |
218 | extern struct pcpu_area *pcpuarea; | | 218 | extern struct pcpu_area *pcpuarea; |
219 | | | 219 | |
220 | #define PMAP_PCID_KERN 0 | | 220 | #define PMAP_PCID_KERN 0 |
221 | #define PMAP_PCID_USER 1 | | 221 | #define PMAP_PCID_USER 1 |
222 | | | 222 | |
223 | /* | | 223 | /* |
224 | * pmap data structures: see pmap.c for details of locking. | | 224 | * pmap data structures: see pmap.c for details of locking. |
225 | */ | | 225 | */ |
226 | | | 226 | |
227 | /* | | 227 | /* |
228 | * we maintain a list of all non-kernel pmaps | | 228 | * we maintain a list of all non-kernel pmaps |
229 | */ | | 229 | */ |
230 | | | 230 | |
231 | LIST_HEAD(pmap_head, pmap); /* struct pmap_head: head of a pmap list */ | | 231 | LIST_HEAD(pmap_head, pmap); /* struct pmap_head: head of a pmap list */ |
232 | | | 232 | |
233 | /* | | 233 | /* |
234 | * linked list of all non-kernel pmaps | | 234 | * linked list of all non-kernel pmaps |
235 | */ | | 235 | */ |
236 | extern struct pmap_head pmaps; | | 236 | extern struct pmap_head pmaps; |
237 | extern kmutex_t pmaps_lock; /* protects pmaps */ | | 237 | extern kmutex_t pmaps_lock; /* protects pmaps */ |
238 | | | 238 | |
239 | /* | | 239 | /* |
240 | * pool_cache(9) that pmaps are allocated from | | 240 | * pool_cache(9) that pmaps are allocated from |
241 | */ | | 241 | */ |
242 | extern struct pool_cache pmap_cache; | | 242 | extern struct pool_cache pmap_cache; |
243 | | | 243 | |
244 | /* | | 244 | /* |
245 | * the pmap structure | | 245 | * the pmap structure |
246 | * | | 246 | * |
247 | * note that the pm_obj contains the lock pointer, the reference count, | | 247 | * note that the pm_obj contains the lock pointer, the reference count, |
248 | * page list, and number of PTPs within the pmap. | | 248 | * page list, and number of PTPs within the pmap. |
249 | * | | 249 | * |
250 | * pm_lock is the same as the lock for vm object 0. Changes to | | 250 | * pm_lock is the same as the lock for vm object 0. Changes to |
251 | * the other objects may only be made if that lock has been taken | | 251 | * the other objects may only be made if that lock has been taken |
252 | * (the other object locks are only used when uvm_pagealloc is called) | | 252 | * (the other object locks are only used when uvm_pagealloc is called) |
253 | */ | | 253 | */ |
254 | | | 254 | |
255 | struct pv_page; | | 255 | struct pv_page; |
256 | | | 256 | |
257 | struct pmap { | | 257 | struct pmap { |
258 | struct uvm_object pm_obj[PTP_LEVELS-1];/* objects for lvl >= 1) */ | | 258 | struct uvm_object pm_obj[PTP_LEVELS-1];/* objects for lvl >= 1) */ |
259 | LIST_ENTRY(pmap) pm_list; /* list of all pmaps */ | | 259 | LIST_ENTRY(pmap) pm_list; /* list of all pmaps */ |
260 | pd_entry_t *pm_pdir; /* VA of PD */ | | 260 | pd_entry_t *pm_pdir; /* VA of PD */ |
261 | paddr_t pm_pdirpa[PDP_SIZE]; /* PA of PDs (read-only after create) */ | | 261 | paddr_t pm_pdirpa[PDP_SIZE]; /* PA of PDs (read-only after create) */ |
262 | struct vm_page *pm_ptphint[PTP_LEVELS-1]; | | 262 | struct vm_page *pm_ptphint[PTP_LEVELS-1]; |
263 | /* pointer to a PTP in our pmap */ | | 263 | /* pointer to a PTP in our pmap */ |
264 | struct pmap_statistics pm_stats; /* pmap stats */ | | 264 | struct pmap_statistics pm_stats; /* pmap stats */ |
265 | struct pv_entry *pm_pve; /* spare pv_entry */ | | 265 | struct pv_entry *pm_pve; /* spare pv_entry */ |
266 | LIST_HEAD(, pv_page) pm_pvp_part; | | 266 | LIST_HEAD(, pv_page) pm_pvp_part; |
267 | LIST_HEAD(, pv_page) pm_pvp_empty; | | 267 | LIST_HEAD(, pv_page) pm_pvp_empty; |
268 | LIST_HEAD(, pv_page) pm_pvp_full; | | 268 | LIST_HEAD(, pv_page) pm_pvp_full; |
269 | | | 269 | |
270 | #if !defined(__x86_64__) | | 270 | #if !defined(__x86_64__) |
271 | vaddr_t pm_hiexec; /* highest executable mapping */ | | 271 | vaddr_t pm_hiexec; /* highest executable mapping */ |
272 | #endif /* !defined(__x86_64__) */ | | 272 | #endif /* !defined(__x86_64__) */ |
273 | | | 273 | |
274 | union descriptor *pm_ldt; /* user-set LDT */ | | 274 | union descriptor *pm_ldt; /* user-set LDT */ |
275 | size_t pm_ldt_len; /* XXX unused, remove */ | | 275 | size_t pm_ldt_len; /* XXX unused, remove */ |
276 | int pm_ldt_sel; /* LDT selector */ | | 276 | int pm_ldt_sel; /* LDT selector */ |
277 | | | 277 | |
278 | kcpuset_t *pm_cpus; /* mask of CPUs using pmap */ | | 278 | kcpuset_t *pm_cpus; /* mask of CPUs using pmap */ |
279 | kcpuset_t *pm_kernel_cpus; /* mask of CPUs using kernel part | | 279 | kcpuset_t *pm_kernel_cpus; /* mask of CPUs using kernel part |
280 | of pmap */ | | 280 | of pmap */ |
281 | kcpuset_t *pm_xen_ptp_cpus; /* mask of CPUs which have this pmap's | | 281 | kcpuset_t *pm_xen_ptp_cpus; /* mask of CPUs which have this pmap's |
282 | ptp mapped */ | | 282 | ptp mapped */ |
283 | uint64_t pm_ncsw; /* for assertions */ | | 283 | uint64_t pm_ncsw; /* for assertions */ |
284 | LIST_HEAD(,vm_page) pm_gc_ptp; /* PTPs queued for free */ | | 284 | LIST_HEAD(,vm_page) pm_gc_ptp; /* PTPs queued for free */ |
285 | | | 285 | |
286 | /* Used by NVMM and Xen */ | | 286 | /* Used by NVMM and Xen */ |
287 | int (*pm_enter)(struct pmap *, vaddr_t, paddr_t, vm_prot_t, u_int); | | 287 | int (*pm_enter)(struct pmap *, vaddr_t, paddr_t, vm_prot_t, u_int); |
288 | bool (*pm_extract)(struct pmap *, vaddr_t, paddr_t *); | | 288 | bool (*pm_extract)(struct pmap *, vaddr_t, paddr_t *); |
289 | void (*pm_remove)(struct pmap *, vaddr_t, vaddr_t); | | 289 | void (*pm_remove)(struct pmap *, vaddr_t, vaddr_t); |
290 | int (*pm_sync_pv)(struct vm_page *, vaddr_t, paddr_t, int, uint8_t *, | | 290 | int (*pm_sync_pv)(struct vm_page *, vaddr_t, paddr_t, int, uint8_t *, |
291 | pt_entry_t *); | | 291 | pt_entry_t *); |
292 | void (*pm_pp_remove_ent)(struct pmap *, struct vm_page *, pt_entry_t, | | 292 | void (*pm_pp_remove_ent)(struct pmap *, struct vm_page *, pt_entry_t, |
293 | vaddr_t); | | 293 | vaddr_t); |
294 | void (*pm_write_protect)(struct pmap *, vaddr_t, vaddr_t, vm_prot_t); | | 294 | void (*pm_write_protect)(struct pmap *, vaddr_t, vaddr_t, vm_prot_t); |
295 | void (*pm_unwire)(struct pmap *, vaddr_t); | | 295 | void (*pm_unwire)(struct pmap *, vaddr_t); |
296 | | | 296 | |
297 | void (*pm_tlb_flush)(struct pmap *); | | 297 | void (*pm_tlb_flush)(struct pmap *); |
298 | void *pm_data; | | 298 | void *pm_data; |
299 | | | 299 | |
300 | kmutex_t pm_lock /* locks for pm_objs */ | | 300 | kmutex_t pm_lock /* locks for pm_objs */ |
301 | __aligned(64); /* give lock own cache line */ | | 301 | __aligned(64); /* give lock own cache line */ |
302 | krwlock_t pm_dummy_lock; /* ugly hack for abusing uvm_object */ | | 302 | krwlock_t pm_dummy_lock; /* ugly hack for abusing uvm_object */ |
303 | }; | | 303 | }; |
304 | | | 304 | |
305 | /* macro to access pm_pdirpa slots */ | | 305 | /* macro to access pm_pdirpa slots */ |
306 | #ifdef PAE | | 306 | #ifdef PAE |
307 | #define pmap_pdirpa(pmap, index) \ | | 307 | #define pmap_pdirpa(pmap, index) \ |
308 | ((pmap)->pm_pdirpa[l2tol3(index)] + l2tol2(index) * sizeof(pd_entry_t)) | | 308 | ((pmap)->pm_pdirpa[l2tol3(index)] + l2tol2(index) * sizeof(pd_entry_t)) |
309 | #else | | 309 | #else |
310 | #define pmap_pdirpa(pmap, index) \ | | 310 | #define pmap_pdirpa(pmap, index) \ |
311 | ((pmap)->pm_pdirpa[0] + (index) * sizeof(pd_entry_t)) | | 311 | ((pmap)->pm_pdirpa[0] + (index) * sizeof(pd_entry_t)) |
312 | #endif | | 312 | #endif |
313 | | | 313 | |
314 | /* | | 314 | /* |
315 | * MD flags that we use for pmap_enter and pmap_kenter_pa: | | 315 | * MD flags that we use for pmap_enter and pmap_kenter_pa: |
316 | */ | | 316 | */ |
317 | | | 317 | |
318 | /* | | 318 | /* |
319 | * global kernel variables | | 319 | * global kernel variables |
320 | */ | | 320 | */ |
321 | | | 321 | |
322 | /* | | 322 | /* |
323 | * PDPpaddr is the physical address of the kernel's PDP. | | 323 | * PDPpaddr is the physical address of the kernel's PDP. |
324 | * - i386 non-PAE and amd64: PDPpaddr corresponds directly to the %cr3 | | 324 | * - i386 non-PAE and amd64: PDPpaddr corresponds directly to the %cr3 |
325 | * value associated to the kernel process, proc0. | | 325 | * value associated to the kernel process, proc0. |
326 | * - i386 PAE: it still represents the PA of the kernel's PDP (L2). Due to | | 326 | * - i386 PAE: it still represents the PA of the kernel's PDP (L2). Due to |
327 | * the L3 PD, it cannot be considered as the equivalent of a %cr3 any more. | | 327 | * the L3 PD, it cannot be considered as the equivalent of a %cr3 any more. |
328 | * - Xen: it corresponds to the PFN of the kernel's PDP. | | 328 | * - Xen: it corresponds to the PFN of the kernel's PDP. |
329 | */ | | 329 | */ |
330 | extern u_long PDPpaddr; | | 330 | extern u_long PDPpaddr; |
331 | | | 331 | |
332 | extern pd_entry_t pmap_pg_g; /* do we support PTE_G? */ | | 332 | extern pd_entry_t pmap_pg_g; /* do we support PTE_G? */ |
333 | extern pd_entry_t pmap_pg_nx; /* do we support PTE_NX? */ | | 333 | extern pd_entry_t pmap_pg_nx; /* do we support PTE_NX? */ |
334 | extern int pmap_largepages; | | 334 | extern int pmap_largepages; |
335 | extern long nkptp[PTP_LEVELS]; | | 335 | extern long nkptp[PTP_LEVELS]; |
336 | | | 336 | |
337 | /* | | 337 | /* |
338 | * macros | | 338 | * macros |
339 | */ | | 339 | */ |
340 | | | 340 | |
341 | #define pmap_resident_count(pmap) ((pmap)->pm_stats.resident_count) | | 341 | #define pmap_resident_count(pmap) ((pmap)->pm_stats.resident_count) |
342 | #define pmap_wired_count(pmap) ((pmap)->pm_stats.wired_count) | | 342 | #define pmap_wired_count(pmap) ((pmap)->pm_stats.wired_count) |
343 | | | 343 | |
344 | #define pmap_clear_modify(pg) pmap_clear_attrs(pg, PP_ATTRS_D) | | 344 | #define pmap_clear_modify(pg) pmap_clear_attrs(pg, PP_ATTRS_D) |
345 | #define pmap_clear_reference(pg) pmap_clear_attrs(pg, PP_ATTRS_A) | | 345 | #define pmap_clear_reference(pg) pmap_clear_attrs(pg, PP_ATTRS_A) |
346 | #define pmap_copy(DP,SP,D,L,S) __USE(L) | | 346 | #define pmap_copy(DP,SP,D,L,S) __USE(L) |
347 | #define pmap_is_modified(pg) pmap_test_attrs(pg, PP_ATTRS_D) | | 347 | #define pmap_is_modified(pg) pmap_test_attrs(pg, PP_ATTRS_D) |
348 | #define pmap_is_referenced(pg) pmap_test_attrs(pg, PP_ATTRS_A) | | 348 | #define pmap_is_referenced(pg) pmap_test_attrs(pg, PP_ATTRS_A) |
349 | #define pmap_move(DP,SP,D,L,S) | | 349 | #define pmap_move(DP,SP,D,L,S) |
350 | #define pmap_phys_address(ppn) (x86_ptob(ppn) & ~X86_MMAP_FLAG_MASK) | | 350 | #define pmap_phys_address(ppn) (x86_ptob(ppn) & ~X86_MMAP_FLAG_MASK) |
351 | #define pmap_mmap_flags(ppn) x86_mmap_flags(ppn) | | 351 | #define pmap_mmap_flags(ppn) x86_mmap_flags(ppn) |
352 | #define pmap_valid_entry(E) ((E) & PTE_P) /* is PDE or PTE valid? */ | | 352 | #define pmap_valid_entry(E) ((E) & PTE_P) /* is PDE or PTE valid? */ |
353 | | | 353 | |
354 | #if defined(__x86_64__) || defined(PAE) | | 354 | #if defined(__x86_64__) || defined(PAE) |
355 | #define X86_MMAP_FLAG_SHIFT (64 - PGSHIFT) | | 355 | #define X86_MMAP_FLAG_SHIFT (64 - PGSHIFT) |
356 | #else | | 356 | #else |
357 | #define X86_MMAP_FLAG_SHIFT (32 - PGSHIFT) | | 357 | #define X86_MMAP_FLAG_SHIFT (32 - PGSHIFT) |
358 | #endif | | 358 | #endif |
359 | | | 359 | |
360 | #define X86_MMAP_FLAG_MASK 0xf | | 360 | #define X86_MMAP_FLAG_MASK 0xf |
361 | #define X86_MMAP_FLAG_PREFETCH 0x1 | | 361 | #define X86_MMAP_FLAG_PREFETCH 0x1 |
362 | | | 362 | |
363 | /* | | 363 | /* |
364 | * prototypes | | 364 | * prototypes |
365 | */ | | 365 | */ |
366 | | | 366 | |
367 | void pmap_activate(struct lwp *); | | 367 | void pmap_activate(struct lwp *); |
368 | void pmap_bootstrap(vaddr_t); | | 368 | void pmap_bootstrap(vaddr_t); |
369 | bool pmap_clear_attrs(struct vm_page *, unsigned); | | 369 | bool pmap_clear_attrs(struct vm_page *, unsigned); |
370 | bool pmap_pv_clear_attrs(paddr_t, unsigned); | | 370 | bool pmap_pv_clear_attrs(paddr_t, unsigned); |
371 | void pmap_deactivate(struct lwp *); | | 371 | void pmap_deactivate(struct lwp *); |
372 | void pmap_page_remove(struct vm_page *); | | 372 | void pmap_page_remove(struct vm_page *); |
373 | void pmap_pv_remove(paddr_t); | | 373 | void pmap_pv_remove(paddr_t); |
374 | void pmap_remove(struct pmap *, vaddr_t, vaddr_t); | | 374 | void pmap_remove(struct pmap *, vaddr_t, vaddr_t); |
375 | bool pmap_test_attrs(struct vm_page *, unsigned); | | 375 | bool pmap_test_attrs(struct vm_page *, unsigned); |
376 | void pmap_write_protect(struct pmap *, vaddr_t, vaddr_t, vm_prot_t); | | 376 | void pmap_write_protect(struct pmap *, vaddr_t, vaddr_t, vm_prot_t); |
377 | void pmap_load(void); | | 377 | void pmap_load(void); |
378 | paddr_t pmap_init_tmp_pgtbl(paddr_t); | | 378 | paddr_t pmap_init_tmp_pgtbl(paddr_t); |
379 | bool pmap_remove_all(struct pmap *); | | 379 | bool pmap_remove_all(struct pmap *); |
380 | void pmap_ldt_cleanup(struct lwp *); | | 380 | void pmap_ldt_cleanup(struct lwp *); |
381 | void pmap_ldt_sync(struct pmap *); | | 381 | void pmap_ldt_sync(struct pmap *); |
382 | void pmap_kremove_local(vaddr_t, vsize_t); | | 382 | void pmap_kremove_local(vaddr_t, vsize_t); |
383 | | | 383 | |
384 | #define __HAVE_PMAP_PV_TRACK 1 | | 384 | #define __HAVE_PMAP_PV_TRACK 1 |
385 | void pmap_pv_init(void); | | 385 | void pmap_pv_init(void); |
386 | void pmap_pv_track(paddr_t, psize_t); | | 386 | void pmap_pv_track(paddr_t, psize_t); |
387 | void pmap_pv_untrack(paddr_t, psize_t); | | 387 | void pmap_pv_untrack(paddr_t, psize_t); |
388 | | | 388 | |
389 | void pmap_map_ptes(struct pmap *, struct pmap **, pd_entry_t **, | | 389 | void pmap_map_ptes(struct pmap *, struct pmap **, pd_entry_t **, |
390 | pd_entry_t * const **); | | 390 | pd_entry_t * const **); |
391 | void pmap_unmap_ptes(struct pmap *, struct pmap *); | | 391 | void pmap_unmap_ptes(struct pmap *, struct pmap *); |
392 | | | 392 | |
393 | bool pmap_pdes_valid(vaddr_t, pd_entry_t * const *, pd_entry_t *, | | 393 | bool pmap_pdes_valid(vaddr_t, pd_entry_t * const *, pd_entry_t *, |
394 | int *lastlvl); | | 394 | int *lastlvl); |
395 | | | 395 | |
396 | u_int x86_mmap_flags(paddr_t); | | 396 | u_int x86_mmap_flags(paddr_t); |
397 | | | 397 | |
398 | bool pmap_is_curpmap(struct pmap *); | | 398 | bool pmap_is_curpmap(struct pmap *); |
399 | | | 399 | |
400 | void pmap_ept_transform(struct pmap *); | | 400 | void pmap_ept_transform(struct pmap *); |
401 | | | 401 | |
402 | #ifndef __HAVE_DIRECT_MAP | | 402 | #ifndef __HAVE_DIRECT_MAP |
403 | void pmap_vpage_cpu_init(struct cpu_info *); | | 403 | void pmap_vpage_cpu_init(struct cpu_info *); |
404 | #endif | | 404 | #endif |
405 | vaddr_t slotspace_rand(int, size_t, size_t, size_t, vaddr_t); | | 405 | vaddr_t slotspace_rand(int, size_t, size_t, size_t, vaddr_t); |
406 | | | 406 | |
407 | vaddr_t reserve_dumppages(vaddr_t); /* XXX: not a pmap fn */ | | 407 | vaddr_t reserve_dumppages(vaddr_t); /* XXX: not a pmap fn */ |
408 | | | 408 | |
409 | typedef enum tlbwhy { | | 409 | typedef enum tlbwhy { |
410 | TLBSHOOT_REMOVE_ALL, | | 410 | TLBSHOOT_REMOVE_ALL, |
411 | TLBSHOOT_KENTER, | | 411 | TLBSHOOT_KENTER, |
412 | TLBSHOOT_KREMOVE, | | 412 | TLBSHOOT_KREMOVE, |
413 | TLBSHOOT_FREE_PTP, | | 413 | TLBSHOOT_FREE_PTP, |
414 | TLBSHOOT_REMOVE_PTE, | | 414 | TLBSHOOT_REMOVE_PTE, |
415 | TLBSHOOT_SYNC_PV, | | 415 | TLBSHOOT_SYNC_PV, |
416 | TLBSHOOT_WRITE_PROTECT, | | 416 | TLBSHOOT_WRITE_PROTECT, |
417 | TLBSHOOT_ENTER, | | 417 | TLBSHOOT_ENTER, |
418 | TLBSHOOT_NVMM, | | 418 | TLBSHOOT_NVMM, |
419 | TLBSHOOT_BUS_DMA, | | 419 | TLBSHOOT_BUS_DMA, |
420 | TLBSHOOT_BUS_SPACE, | | 420 | TLBSHOOT_BUS_SPACE, |
421 | TLBSHOOT__MAX, | | 421 | TLBSHOOT__MAX, |
422 | } tlbwhy_t; | | 422 | } tlbwhy_t; |
423 | | | 423 | |
424 | void pmap_tlb_init(void); | | 424 | void pmap_tlb_init(void); |
425 | void pmap_tlb_cpu_init(struct cpu_info *); | | 425 | void pmap_tlb_cpu_init(struct cpu_info *); |
426 | void pmap_tlb_shootdown(pmap_t, vaddr_t, pt_entry_t, tlbwhy_t); | | 426 | void pmap_tlb_shootdown(pmap_t, vaddr_t, pt_entry_t, tlbwhy_t); |
427 | void pmap_tlb_shootnow(void); | | 427 | void pmap_tlb_shootnow(void); |
428 | void pmap_tlb_intr(void); | | 428 | void pmap_tlb_intr(void); |
429 | | | 429 | |
430 | #define PMAP_GROWKERNEL /* turn on pmap_growkernel interface */ | | 430 | #define PMAP_GROWKERNEL /* turn on pmap_growkernel interface */ |
431 | #define PMAP_FORK /* turn on pmap_fork interface */ | | 431 | #define PMAP_FORK /* turn on pmap_fork interface */ |
432 | | | 432 | |
433 | /* | | 433 | /* |
434 | * inline functions | | 434 | * inline functions |
435 | */ | | 435 | */ |
436 | | | 436 | |
437 | /* | | 437 | /* |
438 | * pmap_update_pg: flush one page from the TLB (or flush the whole thing | | 438 | * pmap_update_pg: flush one page from the TLB (or flush the whole thing |
439 | * if hardware doesn't support one-page flushing) | | 439 | * if hardware doesn't support one-page flushing) |
440 | */ | | 440 | */ |
441 | | | 441 | |
442 | __inline static void __unused | | 442 | __inline static void __unused |
443 | pmap_update_pg(vaddr_t va) | | 443 | pmap_update_pg(vaddr_t va) |
444 | { | | 444 | { |
445 | invlpg(va); | | 445 | invlpg(va); |
446 | } | | 446 | } |
447 | | | 447 | |
448 | /* | | 448 | /* |
449 | * pmap_page_protect: change the protection of all recorded mappings | | 449 | * pmap_page_protect: change the protection of all recorded mappings |
450 | * of a managed page | | 450 | * of a managed page |
451 | * | | 451 | * |
452 | * => this function is a frontend for pmap_page_remove/pmap_clear_attrs | | 452 | * => this function is a frontend for pmap_page_remove/pmap_clear_attrs |
453 | * => we only have to worry about making the page more protected. | | 453 | * => we only have to worry about making the page more protected. |
454 | * unprotecting a page is done on-demand at fault time. | | 454 | * unprotecting a page is done on-demand at fault time. |
455 | */ | | 455 | */ |
456 | | | 456 | |
457 | __inline static void __unused | | 457 | __inline static void __unused |
458 | pmap_page_protect(struct vm_page *pg, vm_prot_t prot) | | 458 | pmap_page_protect(struct vm_page *pg, vm_prot_t prot) |
459 | { | | 459 | { |
460 | if ((prot & VM_PROT_WRITE) == 0) { | | 460 | if ((prot & VM_PROT_WRITE) == 0) { |
461 | if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) { | | 461 | if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) { |
462 | (void)pmap_clear_attrs(pg, PP_ATTRS_W); | | 462 | (void)pmap_clear_attrs(pg, PP_ATTRS_W); |
463 | } else { | | 463 | } else { |
464 | pmap_page_remove(pg); | | 464 | pmap_page_remove(pg); |
465 | } | | 465 | } |
466 | } | | 466 | } |
467 | } | | 467 | } |
468 | | | 468 | |
469 | /* | | 469 | /* |
470 | * pmap_pv_protect: change the protection of all recorded mappings | | 470 | * pmap_pv_protect: change the protection of all recorded mappings |
471 | * of an unmanaged page | | 471 | * of an unmanaged page |
472 | */ | | 472 | */ |
473 | | | 473 | |
474 | __inline static void __unused | | 474 | __inline static void __unused |
475 | pmap_pv_protect(paddr_t pa, vm_prot_t prot) | | 475 | pmap_pv_protect(paddr_t pa, vm_prot_t prot) |
476 | { | | 476 | { |
477 | if ((prot & VM_PROT_WRITE) == 0) { | | 477 | if ((prot & VM_PROT_WRITE) == 0) { |
478 | if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) { | | 478 | if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) { |
479 | (void)pmap_pv_clear_attrs(pa, PP_ATTRS_W); | | 479 | (void)pmap_pv_clear_attrs(pa, PP_ATTRS_W); |
480 | } else { | | 480 | } else { |
481 | pmap_pv_remove(pa); | | 481 | pmap_pv_remove(pa); |
482 | } | | 482 | } |
483 | } | | 483 | } |
484 | } | | 484 | } |
485 | | | 485 | |
486 | /* | | 486 | /* |
487 | * pmap_protect: change the protection of pages in a pmap | | 487 | * pmap_protect: change the protection of pages in a pmap |
488 | * | | 488 | * |
489 | * => this function is a frontend for pmap_remove/pmap_write_protect | | 489 | * => this function is a frontend for pmap_remove/pmap_write_protect |
490 | * => we only have to worry about making the page more protected. | | 490 | * => we only have to worry about making the page more protected. |
491 | * unprotecting a page is done on-demand at fault time. | | 491 | * unprotecting a page is done on-demand at fault time. |
492 | */ | | 492 | */ |
493 | | | 493 | |
494 | __inline static void __unused | | 494 | __inline static void __unused |
495 | pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) | | 495 | pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) |
496 | { | | 496 | { |
497 | if ((prot & VM_PROT_WRITE) == 0) { | | 497 | if ((prot & VM_PROT_WRITE) == 0) { |
498 | if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) { | | 498 | if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) { |
499 | pmap_write_protect(pmap, sva, eva, prot); | | 499 | pmap_write_protect(pmap, sva, eva, prot); |
500 | } else { | | 500 | } else { |
501 | pmap_remove(pmap, sva, eva); | | 501 | pmap_remove(pmap, sva, eva); |
502 | } | | 502 | } |
503 | } | | 503 | } |
504 | } | | 504 | } |
505 | | | 505 | |
506 | /* | | 506 | /* |
507 | * various address inlines | | 507 | * various address inlines |
508 | * | | 508 | * |
509 | * vtopte: return a pointer to the PTE mapping a VA, works only for | | 509 | * vtopte: return a pointer to the PTE mapping a VA, works only for |
510 | * user and PT addresses | | 510 | * user and PT addresses |
511 | * | | 511 | * |
512 | * kvtopte: return a pointer to the PTE mapping a kernel VA | | 512 | * kvtopte: return a pointer to the PTE mapping a kernel VA |
513 | */ | | 513 | */ |
514 | | | 514 | |
515 | #include <lib/libkern/libkern.h> | | 515 | #include <lib/libkern/libkern.h> |
516 | | | 516 | |
517 | static __inline pt_entry_t * __unused | | 517 | static __inline pt_entry_t * __unused |
518 | vtopte(vaddr_t va) | | 518 | vtopte(vaddr_t va) |
519 | { | | 519 | { |
520 | | | 520 | |
521 | KASSERT(va < VM_MIN_KERNEL_ADDRESS); | | 521 | KASSERT(va < VM_MIN_KERNEL_ADDRESS); |
522 | | | 522 | |
523 | return (PTE_BASE + pl1_i(va)); | | 523 | return (PTE_BASE + pl1_i(va)); |
524 | } | | 524 | } |
525 | | | 525 | |
526 | static __inline pt_entry_t * __unused | | 526 | static __inline pt_entry_t * __unused |
527 | kvtopte(vaddr_t va) | | 527 | kvtopte(vaddr_t va) |
528 | { | | 528 | { |
529 | pd_entry_t *pde; | | 529 | pd_entry_t *pde; |
530 | | | 530 | |
531 | KASSERT(va >= VM_MIN_KERNEL_ADDRESS); | | 531 | KASSERT(va >= VM_MIN_KERNEL_ADDRESS); |
532 | | | 532 | |
533 | pde = L2_BASE + pl2_i(va); | | 533 | pde = L2_BASE + pl2_i(va); |
534 | if (*pde & PTE_PS) | | 534 | if (*pde & PTE_PS) |
535 | return ((pt_entry_t *)pde); | | 535 | return ((pt_entry_t *)pde); |
536 | | | 536 | |
537 | return (PTE_BASE + pl1_i(va)); | | 537 | return (PTE_BASE + pl1_i(va)); |
538 | } | | 538 | } |
539 | | | 539 | |
540 | paddr_t vtophys(vaddr_t); | | 540 | paddr_t vtophys(vaddr_t); |
541 | vaddr_t pmap_map(vaddr_t, paddr_t, paddr_t, vm_prot_t); | | 541 | vaddr_t pmap_map(vaddr_t, paddr_t, paddr_t, vm_prot_t); |
542 | void pmap_cpu_init_late(struct cpu_info *); | | 542 | void pmap_cpu_init_late(struct cpu_info *); |
543 | | | 543 | |
544 | #ifdef XENPV | | 544 | #ifdef XENPV |
545 | #include <sys/bitops.h> | | 545 | #include <sys/bitops.h> |
546 | | | 546 | |
547 | #define XPTE_MASK L1_FRAME | | 547 | #define XPTE_MASK L1_FRAME |
548 | /* Selects the index of a PTE in (A)PTE_BASE */ | | 548 | /* Selects the index of a PTE in (A)PTE_BASE */ |
549 | #define XPTE_SHIFT (L1_SHIFT - ilog2(sizeof(pt_entry_t))) | | 549 | #define XPTE_SHIFT (L1_SHIFT - ilog2(sizeof(pt_entry_t))) |
550 | | | 550 | |
551 | /* PTE access inline fuctions */ | | 551 | /* PTE access inline fuctions */ |
552 | | | 552 | |
553 | /* | | 553 | /* |
554 | * Get the machine address of the pointed pte | | 554 | * Get the machine address of the pointed pte |
555 | * We use hardware MMU to get value so works only for levels 1-3 | | 555 | * We use hardware MMU to get value so works only for levels 1-3 |
556 | */ | | 556 | */ |
557 | | | 557 | |
558 | static __inline paddr_t | | 558 | static __inline paddr_t |
559 | xpmap_ptetomach(pt_entry_t *pte) | | 559 | xpmap_ptetomach(pt_entry_t *pte) |
560 | { | | 560 | { |
561 | pt_entry_t *up_pte; | | 561 | pt_entry_t *up_pte; |
562 | vaddr_t va = (vaddr_t) pte; | | 562 | vaddr_t va = (vaddr_t) pte; |
563 | | | 563 | |
564 | va = ((va & XPTE_MASK) >> XPTE_SHIFT) | (vaddr_t) PTE_BASE; | | 564 | va = ((va & XPTE_MASK) >> XPTE_SHIFT) | (vaddr_t) PTE_BASE; |
565 | up_pte = (pt_entry_t *) va; | | 565 | up_pte = (pt_entry_t *) va; |
566 | | | 566 | |
567 | return (paddr_t) (((*up_pte) & PTE_FRAME) + (((vaddr_t) pte) & (~PTE_FRAME & ~VA_SIGN_MASK))); | | 567 | return (paddr_t) (((*up_pte) & PTE_FRAME) + (((vaddr_t) pte) & (~PTE_FRAME & ~VA_SIGN_MASK))); |
568 | } | | 568 | } |
569 | | | 569 | |
570 | /* Xen helpers to change bits of a pte */ | | 570 | /* Xen helpers to change bits of a pte */ |
571 | #define XPMAP_UPDATE_DIRECT 1 /* Update direct map entry flags too */ | | 571 | #define XPMAP_UPDATE_DIRECT 1 /* Update direct map entry flags too */ |
572 | | | 572 | |
573 | paddr_t vtomach(vaddr_t); | | 573 | paddr_t vtomach(vaddr_t); |
574 | #define vtomfn(va) (vtomach(va) >> PAGE_SHIFT) | | 574 | #define vtomfn(va) (vtomach(va) >> PAGE_SHIFT) |
575 | #endif /* XENPV */ | | 575 | #endif /* XENPV */ |
576 | | | 576 | |
577 | /* pmap functions with machine addresses */ | | 577 | /* pmap functions with machine addresses */ |
578 | void pmap_kenter_ma(vaddr_t, paddr_t, vm_prot_t, u_int); | | 578 | void pmap_kenter_ma(vaddr_t, paddr_t, vm_prot_t, u_int); |
579 | int pmap_enter_ma(struct pmap *, vaddr_t, paddr_t, paddr_t, | | 579 | int pmap_enter_ma(struct pmap *, vaddr_t, paddr_t, paddr_t, |
580 | vm_prot_t, u_int, int); | | 580 | vm_prot_t, u_int, int); |
581 | bool pmap_extract_ma(pmap_t, vaddr_t, paddr_t *); | | 581 | bool pmap_extract_ma(pmap_t, vaddr_t, paddr_t *); |
582 | | | 582 | |
583 | paddr_t pmap_get_physpage(void); | | 583 | paddr_t pmap_get_physpage(void); |
584 | | | 584 | |
585 | /* | | 585 | /* |
586 | * Hooks for the pool allocator. | | 586 | * Hooks for the pool allocator. |
587 | */ | | 587 | */ |
588 | #define POOL_VTOPHYS(va) vtophys((vaddr_t) (va)) | | 588 | #define POOL_VTOPHYS(va) vtophys((vaddr_t) (va)) |
589 | | | 589 | |
590 | #ifdef __HAVE_PCPU_AREA | | 590 | #ifdef __HAVE_PCPU_AREA |
591 | extern struct pcpu_area *pcpuarea; | | 591 | extern struct pcpu_area *pcpuarea; |
592 | #define PDIR_SLOT_PCPU 510 | | 592 | #define PDIR_SLOT_PCPU 510 |
593 | #define PMAP_PCPU_BASE (VA_SIGN_NEG((PDIR_SLOT_PCPU * NBPD_L4))) | | 593 | #define PMAP_PCPU_BASE (VA_SIGN_NEG((PDIR_SLOT_PCPU * NBPD_L4))) |
594 | #endif | | 594 | #endif |
595 | | | 595 | |
596 | #ifdef __HAVE_DIRECT_MAP | | 596 | #ifdef __HAVE_DIRECT_MAP |
597 | | | 597 | |
598 | extern vaddr_t pmap_direct_base; | | 598 | extern vaddr_t pmap_direct_base; |
599 | extern vaddr_t pmap_direct_end; | | 599 | extern vaddr_t pmap_direct_end; |
600 | | | 600 | |
601 | #define PMAP_DIRECT_BASE pmap_direct_base | | 601 | #define PMAP_DIRECT_BASE pmap_direct_base |
602 | #define PMAP_DIRECT_END pmap_direct_end | | 602 | #define PMAP_DIRECT_END pmap_direct_end |
603 | | | 603 | |
604 | #define PMAP_DIRECT_MAP(pa) ((vaddr_t)PMAP_DIRECT_BASE + (pa)) | | 604 | #define PMAP_DIRECT_MAP(pa) ((vaddr_t)PMAP_DIRECT_BASE + (pa)) |
605 | #define PMAP_DIRECT_UNMAP(va) ((paddr_t)(va) - PMAP_DIRECT_BASE) | | 605 | #define PMAP_DIRECT_UNMAP(va) ((paddr_t)(va) - PMAP_DIRECT_BASE) |
606 | | | 606 | |
607 | /* | | 607 | /* |
608 | * Alternate mapping hooks for pool pages. | | 608 | * Alternate mapping hooks for pool pages. |
609 | */ | | 609 | */ |
610 | #define PMAP_MAP_POOLPAGE(pa) PMAP_DIRECT_MAP((pa)) | | 610 | #define PMAP_MAP_POOLPAGE(pa) PMAP_DIRECT_MAP((pa)) |
611 | #define PMAP_UNMAP_POOLPAGE(va) PMAP_DIRECT_UNMAP((va)) | | 611 | #define PMAP_UNMAP_POOLPAGE(va) PMAP_DIRECT_UNMAP((va)) |
612 | | | 612 | |
613 | #endif /* __HAVE_DIRECT_MAP */ | | 613 | #endif /* __HAVE_DIRECT_MAP */ |
614 | | | 614 | |
615 | void x86_stos(void *, long, long); | | | |
616 | void x86_movs(void *, void *, long); | | 615 | void x86_movs(void *, void *, long); |
617 | | | 616 | |
618 | #endif /* _KERNEL */ | | 617 | #endif /* _KERNEL */ |
619 | | | 618 | |
620 | #endif /* _X86_PMAP_H_ */ | | 619 | #endif /* _X86_PMAP_H_ */ |