| @@ -1,453 +1,451 @@ | | | @@ -1,453 +1,451 @@ |
1 | /* $NetBSD: pmap.h,v 1.35 2011/02/11 23:08:38 jmcneill Exp $ */ | | 1 | /* $NetBSD: pmap.h,v 1.36 2011/04/25 22:24:00 yamt Exp $ */ |
2 | | | 2 | |
3 | /* | | 3 | /* |
4 | * Copyright (c) 1997 Charles D. Cranor and Washington University. | | 4 | * Copyright (c) 1997 Charles D. Cranor and Washington University. |
5 | * All rights reserved. | | 5 | * All rights reserved. |
6 | * | | 6 | * |
7 | * Redistribution and use in source and binary forms, with or without | | 7 | * Redistribution and use in source and binary forms, with or without |
8 | * modification, are permitted provided that the following conditions | | 8 | * modification, are permitted provided that the following conditions |
9 | * are met: | | 9 | * are met: |
10 | * 1. Redistributions of source code must retain the above copyright | | 10 | * 1. Redistributions of source code must retain the above copyright |
11 | * notice, this list of conditions and the following disclaimer. | | 11 | * notice, this list of conditions and the following disclaimer. |
12 | * 2. Redistributions in binary form must reproduce the above copyright | | 12 | * 2. Redistributions in binary form must reproduce the above copyright |
13 | * notice, this list of conditions and the following disclaimer in the | | 13 | * notice, this list of conditions and the following disclaimer in the |
14 | * documentation and/or other materials provided with the distribution. | | 14 | * documentation and/or other materials provided with the distribution. |
15 | * | | 15 | * |
16 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR | | 16 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
17 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES | | 17 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
18 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. | | 18 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
19 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, | | 19 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
20 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | | 20 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
21 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | | 21 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
22 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | | 22 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
23 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | | 23 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
24 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF | | 24 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
25 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | | 25 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
26 | */ | | 26 | */ |
27 | | | 27 | |
28 | /* | | 28 | /* |
29 | * Copyright (c) 2001 Wasabi Systems, Inc. | | 29 | * Copyright (c) 2001 Wasabi Systems, Inc. |
30 | * All rights reserved. | | 30 | * All rights reserved. |
31 | * | | 31 | * |
32 | * Written by Frank van der Linden for Wasabi Systems, Inc. | | 32 | * Written by Frank van der Linden for Wasabi Systems, Inc. |
33 | * | | 33 | * |
34 | * Redistribution and use in source and binary forms, with or without | | 34 | * Redistribution and use in source and binary forms, with or without |
35 | * modification, are permitted provided that the following conditions | | 35 | * modification, are permitted provided that the following conditions |
36 | * are met: | | 36 | * are met: |
37 | * 1. Redistributions of source code must retain the above copyright | | 37 | * 1. Redistributions of source code must retain the above copyright |
38 | * notice, this list of conditions and the following disclaimer. | | 38 | * notice, this list of conditions and the following disclaimer. |
39 | * 2. Redistributions in binary form must reproduce the above copyright | | 39 | * 2. Redistributions in binary form must reproduce the above copyright |
40 | * notice, this list of conditions and the following disclaimer in the | | 40 | * notice, this list of conditions and the following disclaimer in the |
41 | * documentation and/or other materials provided with the distribution. | | 41 | * documentation and/or other materials provided with the distribution. |
42 | * 3. All advertising materials mentioning features or use of this software | | 42 | * 3. All advertising materials mentioning features or use of this software |
43 | * must display the following acknowledgement: | | 43 | * must display the following acknowledgement: |
44 | * This product includes software developed for the NetBSD Project by | | 44 | * This product includes software developed for the NetBSD Project by |
45 | * Wasabi Systems, Inc. | | 45 | * Wasabi Systems, Inc. |
46 | * 4. The name of Wasabi Systems, Inc. may not be used to endorse | | 46 | * 4. The name of Wasabi Systems, Inc. may not be used to endorse |
47 | * or promote products derived from this software without specific prior | | 47 | * or promote products derived from this software without specific prior |
48 | * written permission. | | 48 | * written permission. |
49 | * | | 49 | * |
50 | * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND | | 50 | * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND |
51 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED | | 51 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
52 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | | 52 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
53 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC | | 53 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC |
54 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | | 54 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
55 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | | 55 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
56 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | | 56 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
57 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | | 57 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
58 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | | 58 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
59 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | | 59 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
60 | * POSSIBILITY OF SUCH DAMAGE. | | 60 | * POSSIBILITY OF SUCH DAMAGE. |
61 | */ | | 61 | */ |
62 | | | 62 | |
63 | /* | | 63 | /* |
64 | * pmap.h: see pmap.c for the history of this pmap module. | | 64 | * pmap.h: see pmap.c for the history of this pmap module. |
65 | */ | | 65 | */ |
66 | | | 66 | |
67 | #ifndef _X86_PMAP_H_ | | 67 | #ifndef _X86_PMAP_H_ |
68 | #define _X86_PMAP_H_ | | 68 | #define _X86_PMAP_H_ |
69 | | | 69 | |
70 | #define ptei(VA) (((VA_SIGN_POS(VA)) & L1_MASK) >> L1_SHIFT) | | | |
71 | | | | |
72 | /* | | 70 | /* |
73 | * pl*_pi: index in the ptp page for a pde mapping a VA. | | 71 | * pl*_pi: index in the ptp page for a pde mapping a VA. |
74 | * (pl*_i below is the index in the virtual array of all pdes per level) | | 72 | * (pl*_i below is the index in the virtual array of all pdes per level) |
75 | */ | | 73 | */ |
76 | #define pl1_pi(VA) (((VA_SIGN_POS(VA)) & L1_MASK) >> L1_SHIFT) | | 74 | #define pl1_pi(VA) (((VA_SIGN_POS(VA)) & L1_MASK) >> L1_SHIFT) |
77 | #define pl2_pi(VA) (((VA_SIGN_POS(VA)) & L2_MASK) >> L2_SHIFT) | | 75 | #define pl2_pi(VA) (((VA_SIGN_POS(VA)) & L2_MASK) >> L2_SHIFT) |
78 | #define pl3_pi(VA) (((VA_SIGN_POS(VA)) & L3_MASK) >> L3_SHIFT) | | 76 | #define pl3_pi(VA) (((VA_SIGN_POS(VA)) & L3_MASK) >> L3_SHIFT) |
79 | #define pl4_pi(VA) (((VA_SIGN_POS(VA)) & L4_MASK) >> L4_SHIFT) | | 77 | #define pl4_pi(VA) (((VA_SIGN_POS(VA)) & L4_MASK) >> L4_SHIFT) |
80 | | | 78 | |
81 | /* | | 79 | /* |
82 | * pl*_i: generate index into pde/pte arrays in virtual space | | 80 | * pl*_i: generate index into pde/pte arrays in virtual space |
83 | */ | | 81 | */ |
84 | #define pl1_i(VA) (((VA_SIGN_POS(VA)) & L1_FRAME) >> L1_SHIFT) | | 82 | #define pl1_i(VA) (((VA_SIGN_POS(VA)) & L1_FRAME) >> L1_SHIFT) |
85 | #define pl2_i(VA) (((VA_SIGN_POS(VA)) & L2_FRAME) >> L2_SHIFT) | | 83 | #define pl2_i(VA) (((VA_SIGN_POS(VA)) & L2_FRAME) >> L2_SHIFT) |
86 | #define pl3_i(VA) (((VA_SIGN_POS(VA)) & L3_FRAME) >> L3_SHIFT) | | 84 | #define pl3_i(VA) (((VA_SIGN_POS(VA)) & L3_FRAME) >> L3_SHIFT) |
87 | #define pl4_i(VA) (((VA_SIGN_POS(VA)) & L4_FRAME) >> L4_SHIFT) | | 85 | #define pl4_i(VA) (((VA_SIGN_POS(VA)) & L4_FRAME) >> L4_SHIFT) |
88 | #define pl_i(va, lvl) \ | | 86 | #define pl_i(va, lvl) \ |
89 | (((VA_SIGN_POS(va)) & ptp_masks[(lvl)-1]) >> ptp_shifts[(lvl)-1]) | | 87 | (((VA_SIGN_POS(va)) & ptp_masks[(lvl)-1]) >> ptp_shifts[(lvl)-1]) |
90 | | | 88 | |
91 | #define pl_i_roundup(va, lvl) pl_i((va)+ ~ptp_masks[(lvl)-1], (lvl)) | | 89 | #define pl_i_roundup(va, lvl) pl_i((va)+ ~ptp_masks[(lvl)-1], (lvl)) |
92 | | | 90 | |
93 | /* | | 91 | /* |
94 | * PTP macros: | | 92 | * PTP macros: |
95 | * a PTP's index is the PD index of the PDE that points to it | | 93 | * a PTP's index is the PD index of the PDE that points to it |
96 | * a PTP's offset is the byte-offset in the PTE space that this PTP is at | | 94 | * a PTP's offset is the byte-offset in the PTE space that this PTP is at |
97 | * a PTP's VA is the first VA mapped by that PTP | | 95 | * a PTP's VA is the first VA mapped by that PTP |
98 | */ | | 96 | */ |
99 | | | 97 | |
100 | #define ptp_va2o(va, lvl) (pl_i(va, (lvl)+1) * PAGE_SIZE) | | 98 | #define ptp_va2o(va, lvl) (pl_i(va, (lvl)+1) * PAGE_SIZE) |
101 | | | 99 | |
102 | /* size of a PDP: usually one page, except for PAE */ | | 100 | /* size of a PDP: usually one page, except for PAE */ |
103 | #ifdef PAE | | 101 | #ifdef PAE |
104 | #define PDP_SIZE 4 | | 102 | #define PDP_SIZE 4 |
105 | #else | | 103 | #else |
106 | #define PDP_SIZE 1 | | 104 | #define PDP_SIZE 1 |
107 | #endif | | 105 | #endif |
108 | | | 106 | |
109 | | | 107 | |
110 | #if defined(_KERNEL) | | 108 | #if defined(_KERNEL) |
111 | /* | | 109 | /* |
112 | * pmap data structures: see pmap.c for details of locking. | | 110 | * pmap data structures: see pmap.c for details of locking. |
113 | */ | | 111 | */ |
114 | | | 112 | |
115 | /* | | 113 | /* |
116 | * we maintain a list of all non-kernel pmaps | | 114 | * we maintain a list of all non-kernel pmaps |
117 | */ | | 115 | */ |
118 | | | 116 | |
119 | LIST_HEAD(pmap_head, pmap); /* struct pmap_head: head of a pmap list */ | | 117 | LIST_HEAD(pmap_head, pmap); /* struct pmap_head: head of a pmap list */ |
120 | | | 118 | |
121 | /* | | 119 | /* |
122 | * the pmap structure | | 120 | * the pmap structure |
123 | * | | 121 | * |
124 | * note that the pm_obj contains the simple_lock, the reference count, | | 122 | * note that the pm_obj contains the simple_lock, the reference count, |
125 | * page list, and number of PTPs within the pmap. | | 123 | * page list, and number of PTPs within the pmap. |
126 | * | | 124 | * |
127 | * pm_lock is the same as the spinlock for vm object 0. Changes to | | 125 | * pm_lock is the same as the spinlock for vm object 0. Changes to |
128 | * the other objects may only be made if that lock has been taken | | 126 | * the other objects may only be made if that lock has been taken |
129 | * (the other object locks are only used when uvm_pagealloc is called) | | 127 | * (the other object locks are only used when uvm_pagealloc is called) |
130 | * | | 128 | * |
131 | * XXX If we ever support processor numbers higher than 31, we'll have | | 129 | * XXX If we ever support processor numbers higher than 31, we'll have |
132 | * XXX to rethink the CPU mask. | | 130 | * XXX to rethink the CPU mask. |
133 | */ | | 131 | */ |
134 | | | 132 | |
135 | struct pmap { | | 133 | struct pmap { |
136 | struct uvm_object pm_obj[PTP_LEVELS-1]; /* objects for lvl >= 1) */ | | 134 | struct uvm_object pm_obj[PTP_LEVELS-1]; /* objects for lvl >= 1) */ |
137 | #define pm_lock pm_obj[0].vmobjlock | | 135 | #define pm_lock pm_obj[0].vmobjlock |
138 | LIST_ENTRY(pmap) pm_list; /* list (lck by pm_list lock) */ | | 136 | LIST_ENTRY(pmap) pm_list; /* list (lck by pm_list lock) */ |
139 | pd_entry_t *pm_pdir; /* VA of PD (lck by object lock) */ | | 137 | pd_entry_t *pm_pdir; /* VA of PD (lck by object lock) */ |
140 | paddr_t pm_pdirpa[PDP_SIZE]; /* PA of PDs (read-only after create) */ | | 138 | paddr_t pm_pdirpa[PDP_SIZE]; /* PA of PDs (read-only after create) */ |
141 | struct vm_page *pm_ptphint[PTP_LEVELS-1]; | | 139 | struct vm_page *pm_ptphint[PTP_LEVELS-1]; |
142 | /* pointer to a PTP in our pmap */ | | 140 | /* pointer to a PTP in our pmap */ |
143 | struct pmap_statistics pm_stats; /* pmap stats (lck by object lock) */ | | 141 | struct pmap_statistics pm_stats; /* pmap stats (lck by object lock) */ |
144 | | | 142 | |
145 | #if !defined(__x86_64__) | | 143 | #if !defined(__x86_64__) |
146 | vaddr_t pm_hiexec; /* highest executable mapping */ | | 144 | vaddr_t pm_hiexec; /* highest executable mapping */ |
147 | #endif /* !defined(__x86_64__) */ | | 145 | #endif /* !defined(__x86_64__) */ |
148 | int pm_flags; /* see below */ | | 146 | int pm_flags; /* see below */ |
149 | | | 147 | |
150 | union descriptor *pm_ldt; /* user-set LDT */ | | 148 | union descriptor *pm_ldt; /* user-set LDT */ |
151 | size_t pm_ldt_len; /* size of LDT in bytes */ | | 149 | size_t pm_ldt_len; /* size of LDT in bytes */ |
152 | int pm_ldt_sel; /* LDT selector */ | | 150 | int pm_ldt_sel; /* LDT selector */ |
153 | uint32_t pm_cpus; /* mask of CPUs using pmap */ | | 151 | uint32_t pm_cpus; /* mask of CPUs using pmap */ |
154 | uint32_t pm_kernel_cpus; /* mask of CPUs using kernel part | | 152 | uint32_t pm_kernel_cpus; /* mask of CPUs using kernel part |
155 | of pmap */ | | 153 | of pmap */ |
156 | }; | | 154 | }; |
157 | | | 155 | |
158 | /* macro to access pm_pdirpa slots */ | | 156 | /* macro to access pm_pdirpa slots */ |
159 | #ifdef PAE | | 157 | #ifdef PAE |
160 | #define pmap_pdirpa(pmap, index) \ | | 158 | #define pmap_pdirpa(pmap, index) \ |
161 | ((pmap)->pm_pdirpa[l2tol3(index)] + l2tol2(index) * sizeof(pd_entry_t)) | | 159 | ((pmap)->pm_pdirpa[l2tol3(index)] + l2tol2(index) * sizeof(pd_entry_t)) |
162 | #else | | 160 | #else |
163 | #define pmap_pdirpa(pmap, index) \ | | 161 | #define pmap_pdirpa(pmap, index) \ |
164 | ((pmap)->pm_pdirpa[0] + (index) * sizeof(pd_entry_t)) | | 162 | ((pmap)->pm_pdirpa[0] + (index) * sizeof(pd_entry_t)) |
165 | #endif | | 163 | #endif |
166 | | | 164 | |
167 | /* | | 165 | /* |
168 | * MD flags that we use for pmap_enter and pmap_kenter_pa: | | 166 | * MD flags that we use for pmap_enter and pmap_kenter_pa: |
169 | */ | | 167 | */ |
170 | | | 168 | |
171 | /* | | 169 | /* |
172 | * global kernel variables | | 170 | * global kernel variables |
173 | */ | | 171 | */ |
174 | | | 172 | |
175 | /* | | 173 | /* |
176 | * PDPpaddr is the physical address of the kernel's PDP. | | 174 | * PDPpaddr is the physical address of the kernel's PDP. |
177 | * - i386 non-PAE and amd64: PDPpaddr corresponds directly to the %cr3 | | 175 | * - i386 non-PAE and amd64: PDPpaddr corresponds directly to the %cr3 |
178 | * value associated to the kernel process, proc0. | | 176 | * value associated to the kernel process, proc0. |
179 | * - i386 PAE: it still represents the PA of the kernel's PDP (L2). Due to | | 177 | * - i386 PAE: it still represents the PA of the kernel's PDP (L2). Due to |
180 | * the L3 PD, it cannot be considered as the equivalent of a %cr3 any more. | | 178 | * the L3 PD, it cannot be considered as the equivalent of a %cr3 any more. |
181 | * - Xen: it corresponds to the PFN of the kernel's PDP. | | 179 | * - Xen: it corresponds to the PFN of the kernel's PDP. |
182 | */ | | 180 | */ |
183 | extern u_long PDPpaddr; | | 181 | extern u_long PDPpaddr; |
184 | | | 182 | |
185 | extern int pmap_pg_g; /* do we support PG_G? */ | | 183 | extern int pmap_pg_g; /* do we support PG_G? */ |
186 | extern long nkptp[PTP_LEVELS]; | | 184 | extern long nkptp[PTP_LEVELS]; |
187 | | | 185 | |
188 | /* | | 186 | /* |
189 | * macros | | 187 | * macros |
190 | */ | | 188 | */ |
191 | | | 189 | |
192 | #define pmap_resident_count(pmap) ((pmap)->pm_stats.resident_count) | | 190 | #define pmap_resident_count(pmap) ((pmap)->pm_stats.resident_count) |
193 | #define pmap_wired_count(pmap) ((pmap)->pm_stats.wired_count) | | 191 | #define pmap_wired_count(pmap) ((pmap)->pm_stats.wired_count) |
194 | | | 192 | |
195 | #define pmap_clear_modify(pg) pmap_clear_attrs(pg, PG_M) | | 193 | #define pmap_clear_modify(pg) pmap_clear_attrs(pg, PG_M) |
196 | #define pmap_clear_reference(pg) pmap_clear_attrs(pg, PG_U) | | 194 | #define pmap_clear_reference(pg) pmap_clear_attrs(pg, PG_U) |
197 | #define pmap_copy(DP,SP,D,L,S) | | 195 | #define pmap_copy(DP,SP,D,L,S) |
198 | #define pmap_is_modified(pg) pmap_test_attrs(pg, PG_M) | | 196 | #define pmap_is_modified(pg) pmap_test_attrs(pg, PG_M) |
199 | #define pmap_is_referenced(pg) pmap_test_attrs(pg, PG_U) | | 197 | #define pmap_is_referenced(pg) pmap_test_attrs(pg, PG_U) |
200 | #define pmap_move(DP,SP,D,L,S) | | 198 | #define pmap_move(DP,SP,D,L,S) |
201 | #define pmap_phys_address(ppn) (x86_ptob(ppn) & ~X86_MMAP_FLAG_MASK) | | 199 | #define pmap_phys_address(ppn) (x86_ptob(ppn) & ~X86_MMAP_FLAG_MASK) |
202 | #define pmap_mmap_flags(ppn) x86_mmap_flags(ppn) | | 200 | #define pmap_mmap_flags(ppn) x86_mmap_flags(ppn) |
203 | #define pmap_valid_entry(E) ((E) & PG_V) /* is PDE or PTE valid? */ | | 201 | #define pmap_valid_entry(E) ((E) & PG_V) /* is PDE or PTE valid? */ |
204 | | | 202 | |
205 | #if defined(__x86_64__) || defined(PAE) | | 203 | #if defined(__x86_64__) || defined(PAE) |
206 | #define X86_MMAP_FLAG_SHIFT (64 - PGSHIFT) | | 204 | #define X86_MMAP_FLAG_SHIFT (64 - PGSHIFT) |
207 | #else | | 205 | #else |
208 | #define X86_MMAP_FLAG_SHIFT (32 - PGSHIFT) | | 206 | #define X86_MMAP_FLAG_SHIFT (32 - PGSHIFT) |
209 | #endif | | 207 | #endif |
210 | | | 208 | |
211 | #define X86_MMAP_FLAG_MASK 0xf | | 209 | #define X86_MMAP_FLAG_MASK 0xf |
212 | #define X86_MMAP_FLAG_PREFETCH 0x1 | | 210 | #define X86_MMAP_FLAG_PREFETCH 0x1 |
213 | | | 211 | |
214 | /* | | 212 | /* |
215 | * prototypes | | 213 | * prototypes |
216 | */ | | 214 | */ |
217 | | | 215 | |
218 | void pmap_activate(struct lwp *); | | 216 | void pmap_activate(struct lwp *); |
219 | void pmap_bootstrap(vaddr_t); | | 217 | void pmap_bootstrap(vaddr_t); |
220 | bool pmap_clear_attrs(struct vm_page *, unsigned); | | 218 | bool pmap_clear_attrs(struct vm_page *, unsigned); |
221 | void pmap_deactivate(struct lwp *); | | 219 | void pmap_deactivate(struct lwp *); |
222 | void pmap_page_remove (struct vm_page *); | | 220 | void pmap_page_remove (struct vm_page *); |
223 | void pmap_remove(struct pmap *, vaddr_t, vaddr_t); | | 221 | void pmap_remove(struct pmap *, vaddr_t, vaddr_t); |
224 | bool pmap_test_attrs(struct vm_page *, unsigned); | | 222 | bool pmap_test_attrs(struct vm_page *, unsigned); |
225 | void pmap_write_protect(struct pmap *, vaddr_t, vaddr_t, vm_prot_t); | | 223 | void pmap_write_protect(struct pmap *, vaddr_t, vaddr_t, vm_prot_t); |
226 | void pmap_load(void); | | 224 | void pmap_load(void); |
227 | paddr_t pmap_init_tmp_pgtbl(paddr_t); | | 225 | paddr_t pmap_init_tmp_pgtbl(paddr_t); |
228 | void pmap_remove_all(struct pmap *); | | 226 | void pmap_remove_all(struct pmap *); |
229 | void pmap_ldt_sync(struct pmap *); | | 227 | void pmap_ldt_sync(struct pmap *); |
230 | | | 228 | |
231 | void pmap_emap_enter(vaddr_t, paddr_t, vm_prot_t); | | 229 | void pmap_emap_enter(vaddr_t, paddr_t, vm_prot_t); |
232 | void pmap_emap_remove(vaddr_t, vsize_t); | | 230 | void pmap_emap_remove(vaddr_t, vsize_t); |
233 | void pmap_emap_sync(bool); | | 231 | void pmap_emap_sync(bool); |
234 | | | 232 | |
235 | void pmap_map_ptes(struct pmap *, struct pmap **, pd_entry_t **, | | 233 | void pmap_map_ptes(struct pmap *, struct pmap **, pd_entry_t **, |
236 | pd_entry_t * const **); | | 234 | pd_entry_t * const **); |
237 | void pmap_unmap_ptes(struct pmap *, struct pmap *); | | 235 | void pmap_unmap_ptes(struct pmap *, struct pmap *); |
238 | | | 236 | |
239 | int pmap_pdes_invalid(vaddr_t, pd_entry_t * const *, pd_entry_t *); | | 237 | int pmap_pdes_invalid(vaddr_t, pd_entry_t * const *, pd_entry_t *); |
240 | | | 238 | |
241 | u_int x86_mmap_flags(paddr_t); | | 239 | u_int x86_mmap_flags(paddr_t); |
242 | | | 240 | |
243 | vaddr_t reserve_dumppages(vaddr_t); /* XXX: not a pmap fn */ | | 241 | vaddr_t reserve_dumppages(vaddr_t); /* XXX: not a pmap fn */ |
244 | | | 242 | |
245 | void pmap_tlb_shootdown(pmap_t, vaddr_t, vaddr_t, pt_entry_t); | | 243 | void pmap_tlb_shootdown(pmap_t, vaddr_t, vaddr_t, pt_entry_t); |
246 | void pmap_tlb_shootwait(void); | | 244 | void pmap_tlb_shootwait(void); |
247 | | | 245 | |
248 | #define __HAVE_PMAP_EMAP | | 246 | #define __HAVE_PMAP_EMAP |
249 | | | 247 | |
250 | #define PMAP_GROWKERNEL /* turn on pmap_growkernel interface */ | | 248 | #define PMAP_GROWKERNEL /* turn on pmap_growkernel interface */ |
251 | #define PMAP_FORK /* turn on pmap_fork interface */ | | 249 | #define PMAP_FORK /* turn on pmap_fork interface */ |
252 | | | 250 | |
253 | /* | | 251 | /* |
254 | * Do idle page zero'ing uncached to avoid polluting the cache. | | 252 | * Do idle page zero'ing uncached to avoid polluting the cache. |
255 | */ | | 253 | */ |
256 | bool pmap_pageidlezero(paddr_t); | | 254 | bool pmap_pageidlezero(paddr_t); |
257 | #define PMAP_PAGEIDLEZERO(pa) pmap_pageidlezero((pa)) | | 255 | #define PMAP_PAGEIDLEZERO(pa) pmap_pageidlezero((pa)) |
258 | | | 256 | |
259 | /* | | 257 | /* |
260 | * inline functions | | 258 | * inline functions |
261 | */ | | 259 | */ |
262 | | | 260 | |
263 | __inline static bool __unused | | 261 | __inline static bool __unused |
264 | pmap_pdes_valid(vaddr_t va, pd_entry_t * const *pdes, pd_entry_t *lastpde) | | 262 | pmap_pdes_valid(vaddr_t va, pd_entry_t * const *pdes, pd_entry_t *lastpde) |
265 | { | | 263 | { |
266 | return pmap_pdes_invalid(va, pdes, lastpde) == 0; | | 264 | return pmap_pdes_invalid(va, pdes, lastpde) == 0; |
267 | } | | 265 | } |
268 | | | 266 | |
269 | /* | | 267 | /* |
270 | * pmap_update_pg: flush one page from the TLB (or flush the whole thing | | 268 | * pmap_update_pg: flush one page from the TLB (or flush the whole thing |
271 | * if hardware doesn't support one-page flushing) | | 269 | * if hardware doesn't support one-page flushing) |
272 | */ | | 270 | */ |
273 | | | 271 | |
274 | __inline static void __unused | | 272 | __inline static void __unused |
275 | pmap_update_pg(vaddr_t va) | | 273 | pmap_update_pg(vaddr_t va) |
276 | { | | 274 | { |
277 | invlpg(va); | | 275 | invlpg(va); |
278 | } | | 276 | } |
279 | | | 277 | |
280 | /* | | 278 | /* |
281 | * pmap_update_2pg: flush two pages from the TLB | | 279 | * pmap_update_2pg: flush two pages from the TLB |
282 | */ | | 280 | */ |
283 | | | 281 | |
284 | __inline static void __unused | | 282 | __inline static void __unused |
285 | pmap_update_2pg(vaddr_t va, vaddr_t vb) | | 283 | pmap_update_2pg(vaddr_t va, vaddr_t vb) |
286 | { | | 284 | { |
287 | invlpg(va); | | 285 | invlpg(va); |
288 | invlpg(vb); | | 286 | invlpg(vb); |
289 | } | | 287 | } |
290 | | | 288 | |
291 | /* | | 289 | /* |
292 | * pmap_page_protect: change the protection of all recorded mappings | | 290 | * pmap_page_protect: change the protection of all recorded mappings |
293 | * of a managed page | | 291 | * of a managed page |
294 | * | | 292 | * |
295 | * => this function is a frontend for pmap_page_remove/pmap_clear_attrs | | 293 | * => this function is a frontend for pmap_page_remove/pmap_clear_attrs |
296 | * => we only have to worry about making the page more protected. | | 294 | * => we only have to worry about making the page more protected. |
297 | * unprotecting a page is done on-demand at fault time. | | 295 | * unprotecting a page is done on-demand at fault time. |
298 | */ | | 296 | */ |
299 | | | 297 | |
300 | __inline static void __unused | | 298 | __inline static void __unused |
301 | pmap_page_protect(struct vm_page *pg, vm_prot_t prot) | | 299 | pmap_page_protect(struct vm_page *pg, vm_prot_t prot) |
302 | { | | 300 | { |
303 | if ((prot & VM_PROT_WRITE) == 0) { | | 301 | if ((prot & VM_PROT_WRITE) == 0) { |
304 | if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) { | | 302 | if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) { |
305 | (void) pmap_clear_attrs(pg, PG_RW); | | 303 | (void) pmap_clear_attrs(pg, PG_RW); |
306 | } else { | | 304 | } else { |
307 | pmap_page_remove(pg); | | 305 | pmap_page_remove(pg); |
308 | } | | 306 | } |
309 | } | | 307 | } |
310 | } | | 308 | } |
311 | | | 309 | |
312 | /* | | 310 | /* |
313 | * pmap_protect: change the protection of pages in a pmap | | 311 | * pmap_protect: change the protection of pages in a pmap |
314 | * | | 312 | * |
315 | * => this function is a frontend for pmap_remove/pmap_write_protect | | 313 | * => this function is a frontend for pmap_remove/pmap_write_protect |
316 | * => we only have to worry about making the page more protected. | | 314 | * => we only have to worry about making the page more protected. |
317 | * unprotecting a page is done on-demand at fault time. | | 315 | * unprotecting a page is done on-demand at fault time. |
318 | */ | | 316 | */ |
319 | | | 317 | |
320 | __inline static void __unused | | 318 | __inline static void __unused |
321 | pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) | | 319 | pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) |
322 | { | | 320 | { |
323 | if ((prot & VM_PROT_WRITE) == 0) { | | 321 | if ((prot & VM_PROT_WRITE) == 0) { |
324 | if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) { | | 322 | if (prot & (VM_PROT_READ|VM_PROT_EXECUTE)) { |
325 | pmap_write_protect(pmap, sva, eva, prot); | | 323 | pmap_write_protect(pmap, sva, eva, prot); |
326 | } else { | | 324 | } else { |
327 | pmap_remove(pmap, sva, eva); | | 325 | pmap_remove(pmap, sva, eva); |
328 | } | | 326 | } |
329 | } | | 327 | } |
330 | } | | 328 | } |
331 | | | 329 | |
332 | /* | | 330 | /* |
333 | * various address inlines | | 331 | * various address inlines |
334 | * | | 332 | * |
335 | * vtopte: return a pointer to the PTE mapping a VA, works only for | | 333 | * vtopte: return a pointer to the PTE mapping a VA, works only for |
336 | * user and PT addresses | | 334 | * user and PT addresses |
337 | * | | 335 | * |
338 | * kvtopte: return a pointer to the PTE mapping a kernel VA | | 336 | * kvtopte: return a pointer to the PTE mapping a kernel VA |
339 | */ | | 337 | */ |
340 | | | 338 | |
341 | #include <lib/libkern/libkern.h> | | 339 | #include <lib/libkern/libkern.h> |
342 | | | 340 | |
343 | static __inline pt_entry_t * __unused | | 341 | static __inline pt_entry_t * __unused |
344 | vtopte(vaddr_t va) | | 342 | vtopte(vaddr_t va) |
345 | { | | 343 | { |
346 | | | 344 | |
347 | KASSERT(va < VM_MIN_KERNEL_ADDRESS); | | 345 | KASSERT(va < VM_MIN_KERNEL_ADDRESS); |
348 | | | 346 | |
349 | return (PTE_BASE + pl1_i(va)); | | 347 | return (PTE_BASE + pl1_i(va)); |
350 | } | | 348 | } |
351 | | | 349 | |
352 | static __inline pt_entry_t * __unused | | 350 | static __inline pt_entry_t * __unused |
353 | kvtopte(vaddr_t va) | | 351 | kvtopte(vaddr_t va) |
354 | { | | 352 | { |
355 | pd_entry_t *pde; | | 353 | pd_entry_t *pde; |
356 | | | 354 | |
357 | KASSERT(va >= VM_MIN_KERNEL_ADDRESS); | | 355 | KASSERT(va >= VM_MIN_KERNEL_ADDRESS); |
358 | | | 356 | |
359 | pde = L2_BASE + pl2_i(va); | | 357 | pde = L2_BASE + pl2_i(va); |
360 | if (*pde & PG_PS) | | 358 | if (*pde & PG_PS) |
361 | return ((pt_entry_t *)pde); | | 359 | return ((pt_entry_t *)pde); |
362 | | | 360 | |
363 | return (PTE_BASE + pl1_i(va)); | | 361 | return (PTE_BASE + pl1_i(va)); |
364 | } | | 362 | } |
365 | | | 363 | |
366 | paddr_t vtophys(vaddr_t); | | 364 | paddr_t vtophys(vaddr_t); |
367 | vaddr_t pmap_map(vaddr_t, paddr_t, paddr_t, vm_prot_t); | | 365 | vaddr_t pmap_map(vaddr_t, paddr_t, paddr_t, vm_prot_t); |
368 | void pmap_cpu_init_early(struct cpu_info *); | | 366 | void pmap_cpu_init_early(struct cpu_info *); |
369 | void pmap_cpu_init_late(struct cpu_info *); | | 367 | void pmap_cpu_init_late(struct cpu_info *); |
370 | bool sse2_idlezero_page(void *); | | 368 | bool sse2_idlezero_page(void *); |
371 | | | 369 | |
372 | | | 370 | |
373 | #ifdef XEN | | 371 | #ifdef XEN |
374 | | | 372 | |
375 | #define XPTE_MASK L1_FRAME | | 373 | #define XPTE_MASK L1_FRAME |
376 | /* XPTE_SHIFT = L1_SHIFT - log2(sizeof(pt_entry_t)) */ | | 374 | /* XPTE_SHIFT = L1_SHIFT - log2(sizeof(pt_entry_t)) */ |
377 | #if defined(__x86_64__) || defined(PAE) | | 375 | #if defined(__x86_64__) || defined(PAE) |
378 | #define XPTE_SHIFT 9 | | 376 | #define XPTE_SHIFT 9 |
379 | #else | | 377 | #else |
380 | #define XPTE_SHIFT 10 | | 378 | #define XPTE_SHIFT 10 |
381 | #endif | | 379 | #endif |
382 | | | 380 | |
383 | /* PTE access inline fuctions */ | | 381 | /* PTE access inline fuctions */ |
384 | | | 382 | |
385 | /* | | 383 | /* |
386 | * Get the machine address of the pointed pte | | 384 | * Get the machine address of the pointed pte |
387 | * We use hardware MMU to get value so works only for levels 1-3 | | 385 | * We use hardware MMU to get value so works only for levels 1-3 |
388 | */ | | 386 | */ |
389 | | | 387 | |
390 | static __inline paddr_t | | 388 | static __inline paddr_t |
391 | xpmap_ptetomach(pt_entry_t *pte) | | 389 | xpmap_ptetomach(pt_entry_t *pte) |
392 | { | | 390 | { |
393 | pt_entry_t *up_pte; | | 391 | pt_entry_t *up_pte; |
394 | vaddr_t va = (vaddr_t) pte; | | 392 | vaddr_t va = (vaddr_t) pte; |
395 | | | 393 | |
396 | va = ((va & XPTE_MASK) >> XPTE_SHIFT) | (vaddr_t) PTE_BASE; | | 394 | va = ((va & XPTE_MASK) >> XPTE_SHIFT) | (vaddr_t) PTE_BASE; |
397 | up_pte = (pt_entry_t *) va; | | 395 | up_pte = (pt_entry_t *) va; |
398 | | | 396 | |
399 | return (paddr_t) (((*up_pte) & PG_FRAME) + (((vaddr_t) pte) & (~PG_FRAME & ~VA_SIGN_MASK))); | | 397 | return (paddr_t) (((*up_pte) & PG_FRAME) + (((vaddr_t) pte) & (~PG_FRAME & ~VA_SIGN_MASK))); |
400 | } | | 398 | } |
401 | | | 399 | |
402 | /* | | 400 | /* |
403 | * xpmap_update() | | 401 | * xpmap_update() |
404 | * Update an active pt entry with Xen | | 402 | * Update an active pt entry with Xen |
405 | * Equivalent to *pte = npte | | 403 | * Equivalent to *pte = npte |
406 | */ | | 404 | */ |
407 | | | 405 | |
408 | static __inline void | | 406 | static __inline void |
409 | xpmap_update (pt_entry_t *pte, pt_entry_t npte) | | 407 | xpmap_update (pt_entry_t *pte, pt_entry_t npte) |
410 | { | | 408 | { |
411 | int s = splvm(); | | 409 | int s = splvm(); |
412 | | | 410 | |
413 | xpq_queue_pte_update(xpmap_ptetomach(pte), npte); | | 411 | xpq_queue_pte_update(xpmap_ptetomach(pte), npte); |
414 | xpq_flush_queue(); | | 412 | xpq_flush_queue(); |
415 | splx(s); | | 413 | splx(s); |
416 | } | | 414 | } |
417 | | | 415 | |
418 | | | 416 | |
419 | /* Xen helpers to change bits of a pte */ | | 417 | /* Xen helpers to change bits of a pte */ |
420 | #define XPMAP_UPDATE_DIRECT 1 /* Update direct map entry flags too */ | | 418 | #define XPMAP_UPDATE_DIRECT 1 /* Update direct map entry flags too */ |
421 | | | 419 | |
422 | paddr_t vtomach(vaddr_t); | | 420 | paddr_t vtomach(vaddr_t); |
423 | #define vtomfn(va) (vtomach(va) >> PAGE_SHIFT) | | 421 | #define vtomfn(va) (vtomach(va) >> PAGE_SHIFT) |
424 | | | 422 | |
425 | #endif /* XEN */ | | 423 | #endif /* XEN */ |
426 | | | 424 | |
427 | /* pmap functions with machine addresses */ | | 425 | /* pmap functions with machine addresses */ |
428 | void pmap_kenter_ma(vaddr_t, paddr_t, vm_prot_t, u_int); | | 426 | void pmap_kenter_ma(vaddr_t, paddr_t, vm_prot_t, u_int); |
429 | int pmap_enter_ma(struct pmap *, vaddr_t, paddr_t, paddr_t, | | 427 | int pmap_enter_ma(struct pmap *, vaddr_t, paddr_t, paddr_t, |
430 | vm_prot_t, u_int, int); | | 428 | vm_prot_t, u_int, int); |
431 | bool pmap_extract_ma(pmap_t, vaddr_t, paddr_t *); | | 429 | bool pmap_extract_ma(pmap_t, vaddr_t, paddr_t *); |
432 | | | 430 | |
433 | /* | | 431 | /* |
434 | * Hooks for the pool allocator. | | 432 | * Hooks for the pool allocator. |
435 | */ | | 433 | */ |
436 | #define POOL_VTOPHYS(va) vtophys((vaddr_t) (va)) | | 434 | #define POOL_VTOPHYS(va) vtophys((vaddr_t) (va)) |
437 | | | 435 | |
438 | /* | | 436 | /* |
439 | * TLB shootdown mailbox. | | 437 | * TLB shootdown mailbox. |
440 | */ | | 438 | */ |
441 | | | 439 | |
442 | struct pmap_mbox { | | 440 | struct pmap_mbox { |
443 | volatile void *mb_pointer; | | 441 | volatile void *mb_pointer; |
444 | volatile uintptr_t mb_addr1; | | 442 | volatile uintptr_t mb_addr1; |
445 | volatile uintptr_t mb_addr2; | | 443 | volatile uintptr_t mb_addr2; |
446 | volatile uintptr_t mb_head; | | 444 | volatile uintptr_t mb_head; |
447 | volatile uintptr_t mb_tail; | | 445 | volatile uintptr_t mb_tail; |
448 | volatile uintptr_t mb_global; | | 446 | volatile uintptr_t mb_global; |
449 | }; | | 447 | }; |
450 | | | 448 | |
451 | #endif /* _KERNEL */ | | 449 | #endif /* _KERNEL */ |
452 | | | 450 | |
453 | #endif /* _X86_PMAP_H_ */ | | 451 | #endif /* _X86_PMAP_H_ */ |