| @@ -1,1068 +1,1068 @@ | | | @@ -1,1068 +1,1068 @@ |
1 | /* $NetBSD: uvm_map.c,v 1.379 2020/04/18 03:27:13 thorpej Exp $ */ | | 1 | /* $NetBSD: uvm_map.c,v 1.380 2020/04/18 17:22:26 riastradh Exp $ */ |
2 | | | 2 | |
3 | /* | | 3 | /* |
4 | * Copyright (c) 1997 Charles D. Cranor and Washington University. | | 4 | * Copyright (c) 1997 Charles D. Cranor and Washington University. |
5 | * Copyright (c) 1991, 1993, The Regents of the University of California. | | 5 | * Copyright (c) 1991, 1993, The Regents of the University of California. |
6 | * | | 6 | * |
7 | * All rights reserved. | | 7 | * All rights reserved. |
8 | * | | 8 | * |
9 | * This code is derived from software contributed to Berkeley by | | 9 | * This code is derived from software contributed to Berkeley by |
10 | * The Mach Operating System project at Carnegie-Mellon University. | | 10 | * The Mach Operating System project at Carnegie-Mellon University. |
11 | * | | 11 | * |
12 | * Redistribution and use in source and binary forms, with or without | | 12 | * Redistribution and use in source and binary forms, with or without |
13 | * modification, are permitted provided that the following conditions | | 13 | * modification, are permitted provided that the following conditions |
14 | * are met: | | 14 | * are met: |
15 | * 1. Redistributions of source code must retain the above copyright | | 15 | * 1. Redistributions of source code must retain the above copyright |
16 | * notice, this list of conditions and the following disclaimer. | | 16 | * notice, this list of conditions and the following disclaimer. |
17 | * 2. Redistributions in binary form must reproduce the above copyright | | 17 | * 2. Redistributions in binary form must reproduce the above copyright |
18 | * notice, this list of conditions and the following disclaimer in the | | 18 | * notice, this list of conditions and the following disclaimer in the |
19 | * documentation and/or other materials provided with the distribution. | | 19 | * documentation and/or other materials provided with the distribution. |
20 | * 3. Neither the name of the University nor the names of its contributors | | 20 | * 3. Neither the name of the University nor the names of its contributors |
21 | * may be used to endorse or promote products derived from this software | | 21 | * may be used to endorse or promote products derived from this software |
22 | * without specific prior written permission. | | 22 | * without specific prior written permission. |
23 | * | | 23 | * |
24 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND | | 24 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND |
25 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | | 25 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
26 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | | 26 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
27 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE | | 27 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE |
28 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | | 28 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
29 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | | 29 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
30 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | | 30 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
31 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | | 31 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
32 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | | 32 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
33 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | | 33 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
34 | * SUCH DAMAGE. | | 34 | * SUCH DAMAGE. |
35 | * | | 35 | * |
36 | * @(#)vm_map.c 8.3 (Berkeley) 1/12/94 | | 36 | * @(#)vm_map.c 8.3 (Berkeley) 1/12/94 |
37 | * from: Id: uvm_map.c,v 1.1.2.27 1998/02/07 01:16:54 chs Exp | | 37 | * from: Id: uvm_map.c,v 1.1.2.27 1998/02/07 01:16:54 chs Exp |
38 | * | | 38 | * |
39 | * | | 39 | * |
40 | * Copyright (c) 1987, 1990 Carnegie-Mellon University. | | 40 | * Copyright (c) 1987, 1990 Carnegie-Mellon University. |
41 | * All rights reserved. | | 41 | * All rights reserved. |
42 | * | | 42 | * |
43 | * Permission to use, copy, modify and distribute this software and | | 43 | * Permission to use, copy, modify and distribute this software and |
44 | * its documentation is hereby granted, provided that both the copyright | | 44 | * its documentation is hereby granted, provided that both the copyright |
45 | * notice and this permission notice appear in all copies of the | | 45 | * notice and this permission notice appear in all copies of the |
46 | * software, derivative works or modified versions, and any portions | | 46 | * software, derivative works or modified versions, and any portions |
47 | * thereof, and that both notices appear in supporting documentation. | | 47 | * thereof, and that both notices appear in supporting documentation. |
48 | * | | 48 | * |
49 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" | | 49 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" |
50 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND | | 50 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND |
51 | * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. | | 51 | * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. |
52 | * | | 52 | * |
53 | * Carnegie Mellon requests users of this software to return to | | 53 | * Carnegie Mellon requests users of this software to return to |
54 | * | | 54 | * |
55 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU | | 55 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU |
56 | * School of Computer Science | | 56 | * School of Computer Science |
57 | * Carnegie Mellon University | | 57 | * Carnegie Mellon University |
58 | * Pittsburgh PA 15213-3890 | | 58 | * Pittsburgh PA 15213-3890 |
59 | * | | 59 | * |
60 | * any improvements or extensions that they make and grant Carnegie the | | 60 | * any improvements or extensions that they make and grant Carnegie the |
61 | * rights to redistribute these changes. | | 61 | * rights to redistribute these changes. |
62 | */ | | 62 | */ |
63 | | | 63 | |
64 | /* | | 64 | /* |
65 | * uvm_map.c: uvm map operations | | 65 | * uvm_map.c: uvm map operations |
66 | */ | | 66 | */ |
67 | | | 67 | |
68 | #include <sys/cdefs.h> | | 68 | #include <sys/cdefs.h> |
69 | __KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.379 2020/04/18 03:27:13 thorpej Exp $"); | | 69 | __KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.380 2020/04/18 17:22:26 riastradh Exp $"); |
70 | | | 70 | |
71 | #include "opt_ddb.h" | | 71 | #include "opt_ddb.h" |
72 | #include "opt_pax.h" | | 72 | #include "opt_pax.h" |
73 | #include "opt_uvmhist.h" | | 73 | #include "opt_uvmhist.h" |
74 | #include "opt_uvm.h" | | 74 | #include "opt_uvm.h" |
75 | #include "opt_sysv.h" | | 75 | #include "opt_sysv.h" |
76 | | | 76 | |
77 | #include <sys/param.h> | | 77 | #include <sys/param.h> |
78 | #include <sys/systm.h> | | 78 | #include <sys/systm.h> |
79 | #include <sys/mman.h> | | 79 | #include <sys/mman.h> |
80 | #include <sys/proc.h> | | 80 | #include <sys/proc.h> |
81 | #include <sys/pool.h> | | 81 | #include <sys/pool.h> |
82 | #include <sys/kernel.h> | | 82 | #include <sys/kernel.h> |
83 | #include <sys/mount.h> | | 83 | #include <sys/mount.h> |
84 | #include <sys/pax.h> | | 84 | #include <sys/pax.h> |
85 | #include <sys/vnode.h> | | 85 | #include <sys/vnode.h> |
86 | #include <sys/filedesc.h> | | 86 | #include <sys/filedesc.h> |
87 | #include <sys/lockdebug.h> | | 87 | #include <sys/lockdebug.h> |
88 | #include <sys/atomic.h> | | 88 | #include <sys/atomic.h> |
89 | #include <sys/sysctl.h> | | 89 | #include <sys/sysctl.h> |
90 | #ifndef __USER_VA0_IS_SAFE | | 90 | #ifndef __USER_VA0_IS_SAFE |
91 | #include <sys/kauth.h> | | 91 | #include <sys/kauth.h> |
92 | #include "opt_user_va0_disable_default.h" | | 92 | #include "opt_user_va0_disable_default.h" |
93 | #endif | | 93 | #endif |
94 | | | 94 | |
95 | #include <sys/shm.h> | | 95 | #include <sys/shm.h> |
96 | | | 96 | |
97 | #include <uvm/uvm.h> | | 97 | #include <uvm/uvm.h> |
98 | #include <uvm/uvm_readahead.h> | | 98 | #include <uvm/uvm_readahead.h> |
99 | | | 99 | |
100 | #if defined(DDB) || defined(DEBUGPRINT) | | 100 | #if defined(DDB) || defined(DEBUGPRINT) |
101 | #include <uvm/uvm_ddb.h> | | 101 | #include <uvm/uvm_ddb.h> |
102 | #endif | | 102 | #endif |
103 | | | 103 | |
104 | #ifdef UVMHIST | | 104 | #ifdef UVMHIST |
105 | #ifndef UVMHIST_MAPHIST_SIZE | | 105 | #ifndef UVMHIST_MAPHIST_SIZE |
106 | #define UVMHIST_MAPHIST_SIZE 100 | | 106 | #define UVMHIST_MAPHIST_SIZE 100 |
107 | #endif | | 107 | #endif |
108 | #ifndef UVMHIST_PDHIST_SIZE | | 108 | #ifndef UVMHIST_PDHIST_SIZE |
109 | #define UVMHIST_PDHIST_SIZE 100 | | 109 | #define UVMHIST_PDHIST_SIZE 100 |
110 | #endif | | 110 | #endif |
111 | static struct kern_history_ent maphistbuf[UVMHIST_MAPHIST_SIZE]; | | 111 | static struct kern_history_ent maphistbuf[UVMHIST_MAPHIST_SIZE]; |
112 | UVMHIST_DEFINE(maphist) = UVMHIST_INITIALIZER(maphist, maphistbuf); | | 112 | UVMHIST_DEFINE(maphist) = UVMHIST_INITIALIZER(maphist, maphistbuf); |
113 | #endif | | 113 | #endif |
114 | | | 114 | |
115 | #if !defined(UVMMAP_COUNTERS) | | 115 | #if !defined(UVMMAP_COUNTERS) |
116 | | | 116 | |
117 | #define UVMMAP_EVCNT_DEFINE(name) /* nothing */ | | 117 | #define UVMMAP_EVCNT_DEFINE(name) /* nothing */ |
118 | #define UVMMAP_EVCNT_INCR(ev) /* nothing */ | | 118 | #define UVMMAP_EVCNT_INCR(ev) /* nothing */ |
119 | #define UVMMAP_EVCNT_DECR(ev) /* nothing */ | | 119 | #define UVMMAP_EVCNT_DECR(ev) /* nothing */ |
120 | | | 120 | |
121 | #else /* defined(UVMMAP_NOCOUNTERS) */ | | 121 | #else /* defined(UVMMAP_NOCOUNTERS) */ |
122 | | | 122 | |
123 | #include <sys/evcnt.h> | | 123 | #include <sys/evcnt.h> |
124 | #define UVMMAP_EVCNT_DEFINE(name) \ | | 124 | #define UVMMAP_EVCNT_DEFINE(name) \ |
125 | struct evcnt uvmmap_evcnt_##name = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, \ | | 125 | struct evcnt uvmmap_evcnt_##name = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, \ |
126 | "uvmmap", #name); \ | | 126 | "uvmmap", #name); \ |
127 | EVCNT_ATTACH_STATIC(uvmmap_evcnt_##name); | | 127 | EVCNT_ATTACH_STATIC(uvmmap_evcnt_##name); |
128 | #define UVMMAP_EVCNT_INCR(ev) uvmmap_evcnt_##ev.ev_count++ | | 128 | #define UVMMAP_EVCNT_INCR(ev) uvmmap_evcnt_##ev.ev_count++ |
129 | #define UVMMAP_EVCNT_DECR(ev) uvmmap_evcnt_##ev.ev_count-- | | 129 | #define UVMMAP_EVCNT_DECR(ev) uvmmap_evcnt_##ev.ev_count-- |
130 | | | 130 | |
131 | #endif /* defined(UVMMAP_NOCOUNTERS) */ | | 131 | #endif /* defined(UVMMAP_NOCOUNTERS) */ |
132 | | | 132 | |
133 | UVMMAP_EVCNT_DEFINE(ubackmerge) | | 133 | UVMMAP_EVCNT_DEFINE(ubackmerge) |
134 | UVMMAP_EVCNT_DEFINE(uforwmerge) | | 134 | UVMMAP_EVCNT_DEFINE(uforwmerge) |
135 | UVMMAP_EVCNT_DEFINE(ubimerge) | | 135 | UVMMAP_EVCNT_DEFINE(ubimerge) |
136 | UVMMAP_EVCNT_DEFINE(unomerge) | | 136 | UVMMAP_EVCNT_DEFINE(unomerge) |
137 | UVMMAP_EVCNT_DEFINE(kbackmerge) | | 137 | UVMMAP_EVCNT_DEFINE(kbackmerge) |
138 | UVMMAP_EVCNT_DEFINE(kforwmerge) | | 138 | UVMMAP_EVCNT_DEFINE(kforwmerge) |
139 | UVMMAP_EVCNT_DEFINE(kbimerge) | | 139 | UVMMAP_EVCNT_DEFINE(kbimerge) |
140 | UVMMAP_EVCNT_DEFINE(knomerge) | | 140 | UVMMAP_EVCNT_DEFINE(knomerge) |
141 | UVMMAP_EVCNT_DEFINE(map_call) | | 141 | UVMMAP_EVCNT_DEFINE(map_call) |
142 | UVMMAP_EVCNT_DEFINE(mlk_call) | | 142 | UVMMAP_EVCNT_DEFINE(mlk_call) |
143 | UVMMAP_EVCNT_DEFINE(mlk_hint) | | 143 | UVMMAP_EVCNT_DEFINE(mlk_hint) |
144 | UVMMAP_EVCNT_DEFINE(mlk_tree) | | 144 | UVMMAP_EVCNT_DEFINE(mlk_tree) |
145 | UVMMAP_EVCNT_DEFINE(mlk_treeloop) | | 145 | UVMMAP_EVCNT_DEFINE(mlk_treeloop) |
146 | | | 146 | |
147 | const char vmmapbsy[] = "vmmapbsy"; | | 147 | const char vmmapbsy[] = "vmmapbsy"; |
148 | | | 148 | |
149 | /* | | 149 | /* |
150 | * cache for vmspace structures. | | 150 | * cache for vmspace structures. |
151 | */ | | 151 | */ |
152 | | | 152 | |
153 | static struct pool_cache uvm_vmspace_cache; | | 153 | static struct pool_cache uvm_vmspace_cache; |
154 | | | 154 | |
155 | /* | | 155 | /* |
156 | * cache for dynamically-allocated map entries. | | 156 | * cache for dynamically-allocated map entries. |
157 | */ | | 157 | */ |
158 | | | 158 | |
159 | static struct pool_cache uvm_map_entry_cache; | | 159 | static struct pool_cache uvm_map_entry_cache; |
160 | | | 160 | |
161 | #ifdef PMAP_GROWKERNEL | | 161 | #ifdef PMAP_GROWKERNEL |
162 | /* | | 162 | /* |
163 | * This global represents the end of the kernel virtual address | | 163 | * This global represents the end of the kernel virtual address |
164 | * space. If we want to exceed this, we must grow the kernel | | 164 | * space. If we want to exceed this, we must grow the kernel |
165 | * virtual address space dynamically. | | 165 | * virtual address space dynamically. |
166 | * | | 166 | * |
167 | * Note, this variable is locked by kernel_map's lock. | | 167 | * Note, this variable is locked by kernel_map's lock. |
168 | */ | | 168 | */ |
169 | vaddr_t uvm_maxkaddr; | | 169 | vaddr_t uvm_maxkaddr; |
170 | #endif | | 170 | #endif |
171 | | | 171 | |
172 | #ifndef __USER_VA0_IS_SAFE | | 172 | #ifndef __USER_VA0_IS_SAFE |
173 | #ifndef __USER_VA0_DISABLE_DEFAULT | | 173 | #ifndef __USER_VA0_DISABLE_DEFAULT |
174 | #define __USER_VA0_DISABLE_DEFAULT 1 | | 174 | #define __USER_VA0_DISABLE_DEFAULT 1 |
175 | #endif | | 175 | #endif |
176 | #ifdef USER_VA0_DISABLE_DEFAULT /* kernel config option overrides */ | | 176 | #ifdef USER_VA0_DISABLE_DEFAULT /* kernel config option overrides */ |
177 | #undef __USER_VA0_DISABLE_DEFAULT | | 177 | #undef __USER_VA0_DISABLE_DEFAULT |
178 | #define __USER_VA0_DISABLE_DEFAULT USER_VA0_DISABLE_DEFAULT | | 178 | #define __USER_VA0_DISABLE_DEFAULT USER_VA0_DISABLE_DEFAULT |
179 | #endif | | 179 | #endif |
180 | int user_va0_disable = __USER_VA0_DISABLE_DEFAULT; | | 180 | int user_va0_disable = __USER_VA0_DISABLE_DEFAULT; |
181 | #endif | | 181 | #endif |
182 | | | 182 | |
183 | /* | | 183 | /* |
184 | * macros | | 184 | * macros |
185 | */ | | 185 | */ |
186 | | | 186 | |
187 | /* | | 187 | /* |
188 | * uvm_map_align_va: round down or up virtual address | | 188 | * uvm_map_align_va: round down or up virtual address |
189 | */ | | 189 | */ |
190 | static __inline void | | 190 | static __inline void |
191 | uvm_map_align_va(vaddr_t *vap, vsize_t align, int topdown) | | 191 | uvm_map_align_va(vaddr_t *vap, vsize_t align, int topdown) |
192 | { | | 192 | { |
193 | | | 193 | |
194 | KASSERT(powerof2(align)); | | 194 | KASSERT(powerof2(align)); |
195 | | | 195 | |
196 | if (align != 0 && (*vap & (align - 1)) != 0) { | | 196 | if (align != 0 && (*vap & (align - 1)) != 0) { |
197 | if (topdown) | | 197 | if (topdown) |
198 | *vap = rounddown2(*vap, align); | | 198 | *vap = rounddown2(*vap, align); |
199 | else | | 199 | else |
200 | *vap = roundup2(*vap, align); | | 200 | *vap = roundup2(*vap, align); |
201 | } | | 201 | } |
202 | } | | 202 | } |
203 | | | 203 | |
204 | /* | | 204 | /* |
205 | * UVM_ET_ISCOMPATIBLE: check some requirements for map entry merging | | 205 | * UVM_ET_ISCOMPATIBLE: check some requirements for map entry merging |
206 | */ | | 206 | */ |
207 | extern struct vm_map *pager_map; | | 207 | extern struct vm_map *pager_map; |
208 | | | 208 | |
209 | #define UVM_ET_ISCOMPATIBLE(ent, type, uobj, meflags, \ | | 209 | #define UVM_ET_ISCOMPATIBLE(ent, type, uobj, meflags, \ |
210 | prot, maxprot, inh, adv, wire) \ | | 210 | prot, maxprot, inh, adv, wire) \ |
211 | ((ent)->etype == (type) && \ | | 211 | ((ent)->etype == (type) && \ |
212 | (((ent)->flags ^ (meflags)) & (UVM_MAP_NOMERGE)) == 0 && \ | | 212 | (((ent)->flags ^ (meflags)) & (UVM_MAP_NOMERGE)) == 0 && \ |
213 | (ent)->object.uvm_obj == (uobj) && \ | | 213 | (ent)->object.uvm_obj == (uobj) && \ |
214 | (ent)->protection == (prot) && \ | | 214 | (ent)->protection == (prot) && \ |
215 | (ent)->max_protection == (maxprot) && \ | | 215 | (ent)->max_protection == (maxprot) && \ |
216 | (ent)->inheritance == (inh) && \ | | 216 | (ent)->inheritance == (inh) && \ |
217 | (ent)->advice == (adv) && \ | | 217 | (ent)->advice == (adv) && \ |
218 | (ent)->wired_count == (wire)) | | 218 | (ent)->wired_count == (wire)) |
219 | | | 219 | |
220 | /* | | 220 | /* |
221 | * uvm_map_entry_link: insert entry into a map | | 221 | * uvm_map_entry_link: insert entry into a map |
222 | * | | 222 | * |
223 | * => map must be locked | | 223 | * => map must be locked |
224 | */ | | 224 | */ |
225 | #define uvm_map_entry_link(map, after_where, entry) do { \ | | 225 | #define uvm_map_entry_link(map, after_where, entry) do { \ |
226 | uvm_mapent_check(entry); \ | | 226 | uvm_mapent_check(entry); \ |
227 | (map)->nentries++; \ | | 227 | (map)->nentries++; \ |
228 | (entry)->prev = (after_where); \ | | 228 | (entry)->prev = (after_where); \ |
229 | (entry)->next = (after_where)->next; \ | | 229 | (entry)->next = (after_where)->next; \ |
230 | (entry)->prev->next = (entry); \ | | 230 | (entry)->prev->next = (entry); \ |
231 | (entry)->next->prev = (entry); \ | | 231 | (entry)->next->prev = (entry); \ |
232 | uvm_rb_insert((map), (entry)); \ | | 232 | uvm_rb_insert((map), (entry)); \ |
233 | } while (/*CONSTCOND*/ 0) | | 233 | } while (/*CONSTCOND*/ 0) |
234 | | | 234 | |
235 | /* | | 235 | /* |
236 | * uvm_map_entry_unlink: remove entry from a map | | 236 | * uvm_map_entry_unlink: remove entry from a map |
237 | * | | 237 | * |
238 | * => map must be locked | | 238 | * => map must be locked |
239 | */ | | 239 | */ |
240 | #define uvm_map_entry_unlink(map, entry) do { \ | | 240 | #define uvm_map_entry_unlink(map, entry) do { \ |
241 | KASSERT((entry) != (map)->first_free); \ | | 241 | KASSERT((entry) != (map)->first_free); \ |
242 | KASSERT((entry) != (map)->hint); \ | | 242 | KASSERT((entry) != (map)->hint); \ |
243 | uvm_mapent_check(entry); \ | | 243 | uvm_mapent_check(entry); \ |
244 | (map)->nentries--; \ | | 244 | (map)->nentries--; \ |
245 | (entry)->next->prev = (entry)->prev; \ | | 245 | (entry)->next->prev = (entry)->prev; \ |
246 | (entry)->prev->next = (entry)->next; \ | | 246 | (entry)->prev->next = (entry)->next; \ |
247 | uvm_rb_remove((map), (entry)); \ | | 247 | uvm_rb_remove((map), (entry)); \ |
248 | } while (/*CONSTCOND*/ 0) | | 248 | } while (/*CONSTCOND*/ 0) |
249 | | | 249 | |
250 | /* | | 250 | /* |
251 | * SAVE_HINT: saves the specified entry as the hint for future lookups. | | 251 | * SAVE_HINT: saves the specified entry as the hint for future lookups. |
252 | * | | 252 | * |
253 | * => map need not be locked. | | 253 | * => map need not be locked. |
254 | */ | | 254 | */ |
255 | #define SAVE_HINT(map, check, value) do { \ | | 255 | #define SAVE_HINT(map, check, value) do { \ |
256 | if ((map)->hint == (check)) \ | | 256 | if ((map)->hint == (check)) \ |
257 | (map)->hint = (value); \ | | 257 | (map)->hint = (value); \ |
258 | } while (/*CONSTCOND*/ 0) | | 258 | } while (/*CONSTCOND*/ 0) |
259 | | | 259 | |
260 | /* | | 260 | /* |
261 | * clear_hints: ensure that hints don't point to the entry. | | 261 | * clear_hints: ensure that hints don't point to the entry. |
262 | * | | 262 | * |
263 | * => map must be write-locked. | | 263 | * => map must be write-locked. |
264 | */ | | 264 | */ |
265 | static void | | 265 | static void |
266 | clear_hints(struct vm_map *map, struct vm_map_entry *ent) | | 266 | clear_hints(struct vm_map *map, struct vm_map_entry *ent) |
267 | { | | 267 | { |
268 | | | 268 | |
269 | SAVE_HINT(map, ent, ent->prev); | | 269 | SAVE_HINT(map, ent, ent->prev); |
270 | if (map->first_free == ent) { | | 270 | if (map->first_free == ent) { |
271 | map->first_free = ent->prev; | | 271 | map->first_free = ent->prev; |
272 | } | | 272 | } |
273 | } | | 273 | } |
274 | | | 274 | |
275 | /* | | 275 | /* |
276 | * VM_MAP_RANGE_CHECK: check and correct range | | 276 | * VM_MAP_RANGE_CHECK: check and correct range |
277 | * | | 277 | * |
278 | * => map must at least be read locked | | 278 | * => map must at least be read locked |
279 | */ | | 279 | */ |
280 | | | 280 | |
281 | #define VM_MAP_RANGE_CHECK(map, start, end) do { \ | | 281 | #define VM_MAP_RANGE_CHECK(map, start, end) do { \ |
282 | if (start < vm_map_min(map)) \ | | 282 | if (start < vm_map_min(map)) \ |
283 | start = vm_map_min(map); \ | | 283 | start = vm_map_min(map); \ |
284 | if (end > vm_map_max(map)) \ | | 284 | if (end > vm_map_max(map)) \ |
285 | end = vm_map_max(map); \ | | 285 | end = vm_map_max(map); \ |
286 | if (start > end) \ | | 286 | if (start > end) \ |
287 | start = end; \ | | 287 | start = end; \ |
288 | } while (/*CONSTCOND*/ 0) | | 288 | } while (/*CONSTCOND*/ 0) |
289 | | | 289 | |
290 | /* | | 290 | /* |
291 | * local prototypes | | 291 | * local prototypes |
292 | */ | | 292 | */ |
293 | | | 293 | |
294 | static struct vm_map_entry * | | 294 | static struct vm_map_entry * |
295 | uvm_mapent_alloc(struct vm_map *, int); | | 295 | uvm_mapent_alloc(struct vm_map *, int); |
296 | static void uvm_mapent_copy(struct vm_map_entry *, struct vm_map_entry *); | | 296 | static void uvm_mapent_copy(struct vm_map_entry *, struct vm_map_entry *); |
297 | static void uvm_mapent_free(struct vm_map_entry *); | | 297 | static void uvm_mapent_free(struct vm_map_entry *); |
298 | #if defined(DEBUG) | | 298 | #if defined(DEBUG) |
299 | static void _uvm_mapent_check(const struct vm_map_entry *, const char *, | | 299 | static void _uvm_mapent_check(const struct vm_map_entry *, const char *, |
300 | int); | | 300 | int); |
301 | #define uvm_mapent_check(map) _uvm_mapent_check(map, __FILE__, __LINE__) | | 301 | #define uvm_mapent_check(map) _uvm_mapent_check(map, __FILE__, __LINE__) |
302 | #else /* defined(DEBUG) */ | | 302 | #else /* defined(DEBUG) */ |
303 | #define uvm_mapent_check(e) /* nothing */ | | 303 | #define uvm_mapent_check(e) /* nothing */ |
304 | #endif /* defined(DEBUG) */ | | 304 | #endif /* defined(DEBUG) */ |
305 | | | 305 | |
306 | static void uvm_map_entry_unwire(struct vm_map *, struct vm_map_entry *); | | 306 | static void uvm_map_entry_unwire(struct vm_map *, struct vm_map_entry *); |
307 | static void uvm_map_reference_amap(struct vm_map_entry *, int); | | 307 | static void uvm_map_reference_amap(struct vm_map_entry *, int); |
308 | static int uvm_map_space_avail(vaddr_t *, vsize_t, voff_t, vsize_t, int, | | 308 | static int uvm_map_space_avail(vaddr_t *, vsize_t, voff_t, vsize_t, int, |
309 | int, struct vm_map_entry *); | | 309 | int, struct vm_map_entry *); |
310 | static void uvm_map_unreference_amap(struct vm_map_entry *, int); | | 310 | static void uvm_map_unreference_amap(struct vm_map_entry *, int); |
311 | | | 311 | |
312 | int _uvm_map_sanity(struct vm_map *); | | 312 | int _uvm_map_sanity(struct vm_map *); |
313 | int _uvm_tree_sanity(struct vm_map *); | | 313 | int _uvm_tree_sanity(struct vm_map *); |
314 | static vsize_t uvm_rb_maxgap(const struct vm_map_entry *); | | 314 | static vsize_t uvm_rb_maxgap(const struct vm_map_entry *); |
315 | | | 315 | |
316 | #define ROOT_ENTRY(map) ((struct vm_map_entry *)(map)->rb_tree.rbt_root) | | 316 | #define ROOT_ENTRY(map) ((struct vm_map_entry *)(map)->rb_tree.rbt_root) |
317 | #define LEFT_ENTRY(entry) ((struct vm_map_entry *)(entry)->rb_node.rb_left) | | 317 | #define LEFT_ENTRY(entry) ((struct vm_map_entry *)(entry)->rb_node.rb_left) |
318 | #define RIGHT_ENTRY(entry) ((struct vm_map_entry *)(entry)->rb_node.rb_right) | | 318 | #define RIGHT_ENTRY(entry) ((struct vm_map_entry *)(entry)->rb_node.rb_right) |
319 | #define PARENT_ENTRY(map, entry) \ | | 319 | #define PARENT_ENTRY(map, entry) \ |
320 | (ROOT_ENTRY(map) == (entry) \ | | 320 | (ROOT_ENTRY(map) == (entry) \ |
321 | ? NULL : (struct vm_map_entry *)RB_FATHER(&(entry)->rb_node)) | | 321 | ? NULL : (struct vm_map_entry *)RB_FATHER(&(entry)->rb_node)) |
322 | | | 322 | |
323 | /* | | 323 | /* |
324 | * These get filled in if/when SYSVSHM shared memory code is loaded | | 324 | * These get filled in if/when SYSVSHM shared memory code is loaded |
325 | * | | 325 | * |
326 | * We do this with function pointers rather the #ifdef SYSVSHM so the | | 326 | * We do this with function pointers rather the #ifdef SYSVSHM so the |
327 | * SYSVSHM code can be loaded and unloaded | | 327 | * SYSVSHM code can be loaded and unloaded |
328 | */ | | 328 | */ |
329 | void (*uvm_shmexit)(struct vmspace *) = NULL; | | 329 | void (*uvm_shmexit)(struct vmspace *) = NULL; |
330 | void (*uvm_shmfork)(struct vmspace *, struct vmspace *) = NULL; | | 330 | void (*uvm_shmfork)(struct vmspace *, struct vmspace *) = NULL; |
331 | | | 331 | |
332 | static int | | 332 | static int |
333 | uvm_map_compare_nodes(void *ctx, const void *nparent, const void *nkey) | | 333 | uvm_map_compare_nodes(void *ctx, const void *nparent, const void *nkey) |
334 | { | | 334 | { |
335 | const struct vm_map_entry *eparent = nparent; | | 335 | const struct vm_map_entry *eparent = nparent; |
336 | const struct vm_map_entry *ekey = nkey; | | 336 | const struct vm_map_entry *ekey = nkey; |
337 | | | 337 | |
338 | KASSERT(eparent->start < ekey->start || eparent->start >= ekey->end); | | 338 | KASSERT(eparent->start < ekey->start || eparent->start >= ekey->end); |
339 | KASSERT(ekey->start < eparent->start || ekey->start >= eparent->end); | | 339 | KASSERT(ekey->start < eparent->start || ekey->start >= eparent->end); |
340 | | | 340 | |
341 | if (eparent->start < ekey->start) | | 341 | if (eparent->start < ekey->start) |
342 | return -1; | | 342 | return -1; |
343 | if (eparent->end >= ekey->start) | | 343 | if (eparent->end >= ekey->start) |
344 | return 1; | | 344 | return 1; |
345 | return 0; | | 345 | return 0; |
346 | } | | 346 | } |
347 | | | 347 | |
348 | static int | | 348 | static int |
349 | uvm_map_compare_key(void *ctx, const void *nparent, const void *vkey) | | 349 | uvm_map_compare_key(void *ctx, const void *nparent, const void *vkey) |
350 | { | | 350 | { |
351 | const struct vm_map_entry *eparent = nparent; | | 351 | const struct vm_map_entry *eparent = nparent; |
352 | const vaddr_t va = *(const vaddr_t *) vkey; | | 352 | const vaddr_t va = *(const vaddr_t *) vkey; |
353 | | | 353 | |
354 | if (eparent->start < va) | | 354 | if (eparent->start < va) |
355 | return -1; | | 355 | return -1; |
356 | if (eparent->end >= va) | | 356 | if (eparent->end >= va) |
357 | return 1; | | 357 | return 1; |
358 | return 0; | | 358 | return 0; |
359 | } | | 359 | } |
360 | | | 360 | |
361 | static const rb_tree_ops_t uvm_map_tree_ops = { | | 361 | static const rb_tree_ops_t uvm_map_tree_ops = { |
362 | .rbto_compare_nodes = uvm_map_compare_nodes, | | 362 | .rbto_compare_nodes = uvm_map_compare_nodes, |
363 | .rbto_compare_key = uvm_map_compare_key, | | 363 | .rbto_compare_key = uvm_map_compare_key, |
364 | .rbto_node_offset = offsetof(struct vm_map_entry, rb_node), | | 364 | .rbto_node_offset = offsetof(struct vm_map_entry, rb_node), |
365 | .rbto_context = NULL | | 365 | .rbto_context = NULL |
366 | }; | | 366 | }; |
367 | | | 367 | |
368 | /* | | 368 | /* |
369 | * uvm_rb_gap: return the gap size between our entry and next entry. | | 369 | * uvm_rb_gap: return the gap size between our entry and next entry. |
370 | */ | | 370 | */ |
371 | static inline vsize_t | | 371 | static inline vsize_t |
372 | uvm_rb_gap(const struct vm_map_entry *entry) | | 372 | uvm_rb_gap(const struct vm_map_entry *entry) |
373 | { | | 373 | { |
374 | | | 374 | |
375 | KASSERT(entry->next != NULL); | | 375 | KASSERT(entry->next != NULL); |
376 | return entry->next->start - entry->end; | | 376 | return entry->next->start - entry->end; |
377 | } | | 377 | } |
378 | | | 378 | |
379 | static vsize_t | | 379 | static vsize_t |
380 | uvm_rb_maxgap(const struct vm_map_entry *entry) | | 380 | uvm_rb_maxgap(const struct vm_map_entry *entry) |
381 | { | | 381 | { |
382 | struct vm_map_entry *child; | | 382 | struct vm_map_entry *child; |
383 | vsize_t maxgap = entry->gap; | | 383 | vsize_t maxgap = entry->gap; |
384 | | | 384 | |
385 | /* | | 385 | /* |
386 | * We need maxgap to be the largest gap of us or any of our | | 386 | * We need maxgap to be the largest gap of us or any of our |
387 | * descendents. Since each of our children's maxgap is the | | 387 | * descendents. Since each of our children's maxgap is the |
388 | * cached value of their largest gap of themselves or their | | 388 | * cached value of their largest gap of themselves or their |
389 | * descendents, we can just use that value and avoid recursing | | 389 | * descendents, we can just use that value and avoid recursing |
390 | * down the tree to calculate it. | | 390 | * down the tree to calculate it. |
391 | */ | | 391 | */ |
392 | if ((child = LEFT_ENTRY(entry)) != NULL && maxgap < child->maxgap) | | 392 | if ((child = LEFT_ENTRY(entry)) != NULL && maxgap < child->maxgap) |
393 | maxgap = child->maxgap; | | 393 | maxgap = child->maxgap; |
394 | | | 394 | |
395 | if ((child = RIGHT_ENTRY(entry)) != NULL && maxgap < child->maxgap) | | 395 | if ((child = RIGHT_ENTRY(entry)) != NULL && maxgap < child->maxgap) |
396 | maxgap = child->maxgap; | | 396 | maxgap = child->maxgap; |
397 | | | 397 | |
398 | return maxgap; | | 398 | return maxgap; |
399 | } | | 399 | } |
400 | | | 400 | |
401 | static void | | 401 | static void |
402 | uvm_rb_fixup(struct vm_map *map, struct vm_map_entry *entry) | | 402 | uvm_rb_fixup(struct vm_map *map, struct vm_map_entry *entry) |
403 | { | | 403 | { |
404 | struct vm_map_entry *parent; | | 404 | struct vm_map_entry *parent; |
405 | | | 405 | |
406 | KASSERT(entry->gap == uvm_rb_gap(entry)); | | 406 | KASSERT(entry->gap == uvm_rb_gap(entry)); |
407 | entry->maxgap = uvm_rb_maxgap(entry); | | 407 | entry->maxgap = uvm_rb_maxgap(entry); |
408 | | | 408 | |
409 | while ((parent = PARENT_ENTRY(map, entry)) != NULL) { | | 409 | while ((parent = PARENT_ENTRY(map, entry)) != NULL) { |
410 | struct vm_map_entry *brother; | | 410 | struct vm_map_entry *brother; |
411 | vsize_t maxgap = parent->gap; | | 411 | vsize_t maxgap = parent->gap; |
412 | unsigned int which; | | 412 | unsigned int which; |
413 | | | 413 | |
414 | KDASSERT(parent->gap == uvm_rb_gap(parent)); | | 414 | KDASSERT(parent->gap == uvm_rb_gap(parent)); |
415 | if (maxgap < entry->maxgap) | | 415 | if (maxgap < entry->maxgap) |
416 | maxgap = entry->maxgap; | | 416 | maxgap = entry->maxgap; |
417 | /* | | 417 | /* |
418 | * Since we work towards the root, we know entry's maxgap | | 418 | * Since we work towards the root, we know entry's maxgap |
419 | * value is OK, but its brothers may now be out-of-date due | | 419 | * value is OK, but its brothers may now be out-of-date due |
420 | * to rebalancing. So refresh it. | | 420 | * to rebalancing. So refresh it. |
421 | */ | | 421 | */ |
422 | which = RB_POSITION(&entry->rb_node) ^ RB_DIR_OTHER; | | 422 | which = RB_POSITION(&entry->rb_node) ^ RB_DIR_OTHER; |
423 | brother = (struct vm_map_entry *)parent->rb_node.rb_nodes[which]; | | 423 | brother = (struct vm_map_entry *)parent->rb_node.rb_nodes[which]; |
424 | if (brother != NULL) { | | 424 | if (brother != NULL) { |
425 | KDASSERT(brother->gap == uvm_rb_gap(brother)); | | 425 | KDASSERT(brother->gap == uvm_rb_gap(brother)); |
426 | brother->maxgap = uvm_rb_maxgap(brother); | | 426 | brother->maxgap = uvm_rb_maxgap(brother); |
427 | if (maxgap < brother->maxgap) | | 427 | if (maxgap < brother->maxgap) |
428 | maxgap = brother->maxgap; | | 428 | maxgap = brother->maxgap; |
429 | } | | 429 | } |
430 | | | 430 | |
431 | parent->maxgap = maxgap; | | 431 | parent->maxgap = maxgap; |
432 | entry = parent; | | 432 | entry = parent; |
433 | } | | 433 | } |
434 | } | | 434 | } |
435 | | | 435 | |
436 | static void | | 436 | static void |
437 | uvm_rb_insert(struct vm_map *map, struct vm_map_entry *entry) | | 437 | uvm_rb_insert(struct vm_map *map, struct vm_map_entry *entry) |
438 | { | | 438 | { |
439 | struct vm_map_entry *ret __diagused; | | 439 | struct vm_map_entry *ret __diagused; |
440 | | | 440 | |
441 | entry->gap = entry->maxgap = uvm_rb_gap(entry); | | 441 | entry->gap = entry->maxgap = uvm_rb_gap(entry); |
442 | if (entry->prev != &map->header) | | 442 | if (entry->prev != &map->header) |
443 | entry->prev->gap = uvm_rb_gap(entry->prev); | | 443 | entry->prev->gap = uvm_rb_gap(entry->prev); |
444 | | | 444 | |
445 | ret = rb_tree_insert_node(&map->rb_tree, entry); | | 445 | ret = rb_tree_insert_node(&map->rb_tree, entry); |
446 | KASSERTMSG(ret == entry, | | 446 | KASSERTMSG(ret == entry, |
447 | "uvm_rb_insert: map %p: duplicate entry %p", map, ret); | | 447 | "uvm_rb_insert: map %p: duplicate entry %p", map, ret); |
448 | | | 448 | |
449 | /* | | 449 | /* |
450 | * If the previous entry is not our immediate left child, then it's an | | 450 | * If the previous entry is not our immediate left child, then it's an |
451 | * ancestor and will be fixed up on the way to the root. We don't | | 451 | * ancestor and will be fixed up on the way to the root. We don't |
452 | * have to check entry->prev against &map->header since &map->header | | 452 | * have to check entry->prev against &map->header since &map->header |
453 | * will never be in the tree. | | 453 | * will never be in the tree. |
454 | */ | | 454 | */ |
455 | uvm_rb_fixup(map, | | 455 | uvm_rb_fixup(map, |
456 | LEFT_ENTRY(entry) == entry->prev ? entry->prev : entry); | | 456 | LEFT_ENTRY(entry) == entry->prev ? entry->prev : entry); |
457 | } | | 457 | } |
458 | | | 458 | |
459 | static void | | 459 | static void |
460 | uvm_rb_remove(struct vm_map *map, struct vm_map_entry *entry) | | 460 | uvm_rb_remove(struct vm_map *map, struct vm_map_entry *entry) |
461 | { | | 461 | { |
462 | struct vm_map_entry *prev_parent = NULL, *next_parent = NULL; | | 462 | struct vm_map_entry *prev_parent = NULL, *next_parent = NULL; |
463 | | | 463 | |
464 | /* | | 464 | /* |
465 | * If we are removing an interior node, then an adjacent node will | | 465 | * If we are removing an interior node, then an adjacent node will |
466 | * be used to replace its position in the tree. Therefore we will | | 466 | * be used to replace its position in the tree. Therefore we will |
467 | * need to fixup the tree starting at the parent of the replacement | | 467 | * need to fixup the tree starting at the parent of the replacement |
468 | * node. So record their parents for later use. | | 468 | * node. So record their parents for later use. |
469 | */ | | 469 | */ |
470 | if (entry->prev != &map->header) | | 470 | if (entry->prev != &map->header) |
471 | prev_parent = PARENT_ENTRY(map, entry->prev); | | 471 | prev_parent = PARENT_ENTRY(map, entry->prev); |
472 | if (entry->next != &map->header) | | 472 | if (entry->next != &map->header) |
473 | next_parent = PARENT_ENTRY(map, entry->next); | | 473 | next_parent = PARENT_ENTRY(map, entry->next); |
474 | | | 474 | |
475 | rb_tree_remove_node(&map->rb_tree, entry); | | 475 | rb_tree_remove_node(&map->rb_tree, entry); |
476 | | | 476 | |
477 | /* | | 477 | /* |
478 | * If the previous node has a new parent, fixup the tree starting | | 478 | * If the previous node has a new parent, fixup the tree starting |
479 | * at the previous node's old parent. | | 479 | * at the previous node's old parent. |
480 | */ | | 480 | */ |
481 | if (entry->prev != &map->header) { | | 481 | if (entry->prev != &map->header) { |
482 | /* | | 482 | /* |
483 | * Update the previous entry's gap due to our absence. | | 483 | * Update the previous entry's gap due to our absence. |
484 | */ | | 484 | */ |
485 | entry->prev->gap = uvm_rb_gap(entry->prev); | | 485 | entry->prev->gap = uvm_rb_gap(entry->prev); |
486 | uvm_rb_fixup(map, entry->prev); | | 486 | uvm_rb_fixup(map, entry->prev); |
487 | if (prev_parent != NULL | | 487 | if (prev_parent != NULL |
488 | && prev_parent != entry | | 488 | && prev_parent != entry |
489 | && prev_parent != PARENT_ENTRY(map, entry->prev)) | | 489 | && prev_parent != PARENT_ENTRY(map, entry->prev)) |
490 | uvm_rb_fixup(map, prev_parent); | | 490 | uvm_rb_fixup(map, prev_parent); |
491 | } | | 491 | } |
492 | | | 492 | |
493 | /* | | 493 | /* |
494 | * If the next node has a new parent, fixup the tree starting | | 494 | * If the next node has a new parent, fixup the tree starting |
495 | * at the next node's old parent. | | 495 | * at the next node's old parent. |
496 | */ | | 496 | */ |
497 | if (entry->next != &map->header) { | | 497 | if (entry->next != &map->header) { |
498 | uvm_rb_fixup(map, entry->next); | | 498 | uvm_rb_fixup(map, entry->next); |
499 | if (next_parent != NULL | | 499 | if (next_parent != NULL |
500 | && next_parent != entry | | 500 | && next_parent != entry |
501 | && next_parent != PARENT_ENTRY(map, entry->next)) | | 501 | && next_parent != PARENT_ENTRY(map, entry->next)) |
502 | uvm_rb_fixup(map, next_parent); | | 502 | uvm_rb_fixup(map, next_parent); |
503 | } | | 503 | } |
504 | } | | 504 | } |
505 | | | 505 | |
506 | #if defined(DEBUG) | | 506 | #if defined(DEBUG) |
507 | int uvm_debug_check_map = 0; | | 507 | int uvm_debug_check_map = 0; |
508 | int uvm_debug_check_rbtree = 0; | | 508 | int uvm_debug_check_rbtree = 0; |
509 | #define uvm_map_check(map, name) \ | | 509 | #define uvm_map_check(map, name) \ |
510 | _uvm_map_check((map), (name), __FILE__, __LINE__) | | 510 | _uvm_map_check((map), (name), __FILE__, __LINE__) |
511 | static void | | 511 | static void |
512 | _uvm_map_check(struct vm_map *map, const char *name, | | 512 | _uvm_map_check(struct vm_map *map, const char *name, |
513 | const char *file, int line) | | 513 | const char *file, int line) |
514 | { | | 514 | { |
515 | | | 515 | |
516 | if ((uvm_debug_check_map && _uvm_map_sanity(map)) || | | 516 | if ((uvm_debug_check_map && _uvm_map_sanity(map)) || |
517 | (uvm_debug_check_rbtree && _uvm_tree_sanity(map))) { | | 517 | (uvm_debug_check_rbtree && _uvm_tree_sanity(map))) { |
518 | panic("uvm_map_check failed: \"%s\" map=%p (%s:%d)", | | 518 | panic("uvm_map_check failed: \"%s\" map=%p (%s:%d)", |
519 | name, map, file, line); | | 519 | name, map, file, line); |
520 | } | | 520 | } |
521 | } | | 521 | } |
522 | #else /* defined(DEBUG) */ | | 522 | #else /* defined(DEBUG) */ |
523 | #define uvm_map_check(map, name) /* nothing */ | | 523 | #define uvm_map_check(map, name) /* nothing */ |
524 | #endif /* defined(DEBUG) */ | | 524 | #endif /* defined(DEBUG) */ |
525 | | | 525 | |
526 | #if defined(DEBUG) || defined(DDB) | | 526 | #if defined(DEBUG) || defined(DDB) |
527 | int | | 527 | int |
528 | _uvm_map_sanity(struct vm_map *map) | | 528 | _uvm_map_sanity(struct vm_map *map) |
529 | { | | 529 | { |
530 | bool first_free_found = false; | | 530 | bool first_free_found = false; |
531 | bool hint_found = false; | | 531 | bool hint_found = false; |
532 | const struct vm_map_entry *e; | | 532 | const struct vm_map_entry *e; |
533 | struct vm_map_entry *hint = map->hint; | | 533 | struct vm_map_entry *hint = map->hint; |
534 | | | 534 | |
535 | e = &map->header; | | 535 | e = &map->header; |
536 | for (;;) { | | 536 | for (;;) { |
537 | if (map->first_free == e) { | | 537 | if (map->first_free == e) { |
538 | first_free_found = true; | | 538 | first_free_found = true; |
539 | } else if (!first_free_found && e->next->start > e->end) { | | 539 | } else if (!first_free_found && e->next->start > e->end) { |
540 | printf("first_free %p should be %p\n", | | 540 | printf("first_free %p should be %p\n", |
541 | map->first_free, e); | | 541 | map->first_free, e); |
542 | return -1; | | 542 | return -1; |
543 | } | | 543 | } |
544 | if (hint == e) { | | 544 | if (hint == e) { |
545 | hint_found = true; | | 545 | hint_found = true; |
546 | } | | 546 | } |
547 | | | 547 | |
548 | e = e->next; | | 548 | e = e->next; |
549 | if (e == &map->header) { | | 549 | if (e == &map->header) { |
550 | break; | | 550 | break; |
551 | } | | 551 | } |
552 | } | | 552 | } |
553 | if (!first_free_found) { | | 553 | if (!first_free_found) { |
554 | printf("stale first_free\n"); | | 554 | printf("stale first_free\n"); |
555 | return -1; | | 555 | return -1; |
556 | } | | 556 | } |
557 | if (!hint_found) { | | 557 | if (!hint_found) { |
558 | printf("stale hint\n"); | | 558 | printf("stale hint\n"); |
559 | return -1; | | 559 | return -1; |
560 | } | | 560 | } |
561 | return 0; | | 561 | return 0; |
562 | } | | 562 | } |
563 | | | 563 | |
564 | int | | 564 | int |
565 | _uvm_tree_sanity(struct vm_map *map) | | 565 | _uvm_tree_sanity(struct vm_map *map) |
566 | { | | 566 | { |
567 | struct vm_map_entry *tmp, *trtmp; | | 567 | struct vm_map_entry *tmp, *trtmp; |
568 | int n = 0, i = 1; | | 568 | int n = 0, i = 1; |
569 | | | 569 | |
570 | for (tmp = map->header.next; tmp != &map->header; tmp = tmp->next) { | | 570 | for (tmp = map->header.next; tmp != &map->header; tmp = tmp->next) { |
571 | if (tmp->gap != uvm_rb_gap(tmp)) { | | 571 | if (tmp->gap != uvm_rb_gap(tmp)) { |
572 | printf("%d/%d gap %#lx != %#lx %s\n", | | 572 | printf("%d/%d gap %#lx != %#lx %s\n", |
573 | n + 1, map->nentries, | | 573 | n + 1, map->nentries, |
574 | (ulong)tmp->gap, (ulong)uvm_rb_gap(tmp), | | 574 | (ulong)tmp->gap, (ulong)uvm_rb_gap(tmp), |
575 | tmp->next == &map->header ? "(last)" : ""); | | 575 | tmp->next == &map->header ? "(last)" : ""); |
576 | goto error; | | 576 | goto error; |
577 | } | | 577 | } |
578 | /* | | 578 | /* |
579 | * If any entries are out of order, tmp->gap will be unsigned | | 579 | * If any entries are out of order, tmp->gap will be unsigned |
580 | * and will likely exceed the size of the map. | | 580 | * and will likely exceed the size of the map. |
581 | */ | | 581 | */ |
582 | if (tmp->gap >= vm_map_max(map) - vm_map_min(map)) { | | 582 | if (tmp->gap >= vm_map_max(map) - vm_map_min(map)) { |
583 | printf("too large gap %zu\n", (size_t)tmp->gap); | | 583 | printf("too large gap %zu\n", (size_t)tmp->gap); |
584 | goto error; | | 584 | goto error; |
585 | } | | 585 | } |
586 | n++; | | 586 | n++; |
587 | } | | 587 | } |
588 | | | 588 | |
589 | if (n != map->nentries) { | | 589 | if (n != map->nentries) { |
590 | printf("nentries: %d vs %d\n", n, map->nentries); | | 590 | printf("nentries: %d vs %d\n", n, map->nentries); |
591 | goto error; | | 591 | goto error; |
592 | } | | 592 | } |
593 | | | 593 | |
594 | trtmp = NULL; | | 594 | trtmp = NULL; |
595 | for (tmp = map->header.next; tmp != &map->header; tmp = tmp->next) { | | 595 | for (tmp = map->header.next; tmp != &map->header; tmp = tmp->next) { |
596 | if (tmp->maxgap != uvm_rb_maxgap(tmp)) { | | 596 | if (tmp->maxgap != uvm_rb_maxgap(tmp)) { |
597 | printf("maxgap %#lx != %#lx\n", | | 597 | printf("maxgap %#lx != %#lx\n", |
598 | (ulong)tmp->maxgap, | | 598 | (ulong)tmp->maxgap, |
599 | (ulong)uvm_rb_maxgap(tmp)); | | 599 | (ulong)uvm_rb_maxgap(tmp)); |
600 | goto error; | | 600 | goto error; |
601 | } | | 601 | } |
602 | if (trtmp != NULL && trtmp->start >= tmp->start) { | | 602 | if (trtmp != NULL && trtmp->start >= tmp->start) { |
603 | printf("corrupt: 0x%"PRIxVADDR"x >= 0x%"PRIxVADDR"x\n", | | 603 | printf("corrupt: 0x%"PRIxVADDR"x >= 0x%"PRIxVADDR"x\n", |
604 | trtmp->start, tmp->start); | | 604 | trtmp->start, tmp->start); |
605 | goto error; | | 605 | goto error; |
606 | } | | 606 | } |
607 | | | 607 | |
608 | trtmp = tmp; | | 608 | trtmp = tmp; |
609 | } | | 609 | } |
610 | | | 610 | |
611 | for (tmp = map->header.next; tmp != &map->header; | | 611 | for (tmp = map->header.next; tmp != &map->header; |
612 | tmp = tmp->next, i++) { | | 612 | tmp = tmp->next, i++) { |
613 | trtmp = rb_tree_iterate(&map->rb_tree, tmp, RB_DIR_LEFT); | | 613 | trtmp = rb_tree_iterate(&map->rb_tree, tmp, RB_DIR_LEFT); |
614 | if (trtmp == NULL) | | 614 | if (trtmp == NULL) |
615 | trtmp = &map->header; | | 615 | trtmp = &map->header; |
616 | if (tmp->prev != trtmp) { | | 616 | if (tmp->prev != trtmp) { |
617 | printf("lookup: %d: %p->prev=%p: %p\n", | | 617 | printf("lookup: %d: %p->prev=%p: %p\n", |
618 | i, tmp, tmp->prev, trtmp); | | 618 | i, tmp, tmp->prev, trtmp); |
619 | goto error; | | 619 | goto error; |
620 | } | | 620 | } |
621 | trtmp = rb_tree_iterate(&map->rb_tree, tmp, RB_DIR_RIGHT); | | 621 | trtmp = rb_tree_iterate(&map->rb_tree, tmp, RB_DIR_RIGHT); |
622 | if (trtmp == NULL) | | 622 | if (trtmp == NULL) |
623 | trtmp = &map->header; | | 623 | trtmp = &map->header; |
624 | if (tmp->next != trtmp) { | | 624 | if (tmp->next != trtmp) { |
625 | printf("lookup: %d: %p->next=%p: %p\n", | | 625 | printf("lookup: %d: %p->next=%p: %p\n", |
626 | i, tmp, tmp->next, trtmp); | | 626 | i, tmp, tmp->next, trtmp); |
627 | goto error; | | 627 | goto error; |
628 | } | | 628 | } |
629 | trtmp = rb_tree_find_node(&map->rb_tree, &tmp->start); | | 629 | trtmp = rb_tree_find_node(&map->rb_tree, &tmp->start); |
630 | if (trtmp != tmp) { | | 630 | if (trtmp != tmp) { |
631 | printf("lookup: %d: %p - %p: %p\n", i, tmp, trtmp, | | 631 | printf("lookup: %d: %p - %p: %p\n", i, tmp, trtmp, |
632 | PARENT_ENTRY(map, tmp)); | | 632 | PARENT_ENTRY(map, tmp)); |
633 | goto error; | | 633 | goto error; |
634 | } | | 634 | } |
635 | } | | 635 | } |
636 | | | 636 | |
637 | return (0); | | 637 | return (0); |
638 | error: | | 638 | error: |
639 | return (-1); | | 639 | return (-1); |
640 | } | | 640 | } |
641 | #endif /* defined(DEBUG) || defined(DDB) */ | | 641 | #endif /* defined(DEBUG) || defined(DDB) */ |
642 | | | 642 | |
643 | /* | | 643 | /* |
644 | * vm_map_lock: acquire an exclusive (write) lock on a map. | | 644 | * vm_map_lock: acquire an exclusive (write) lock on a map. |
645 | * | | 645 | * |
646 | * => The locking protocol provides for guaranteed upgrade from shared -> | | 646 | * => The locking protocol provides for guaranteed upgrade from shared -> |
647 | * exclusive by whichever thread currently has the map marked busy. | | 647 | * exclusive by whichever thread currently has the map marked busy. |
648 | * See "LOCKING PROTOCOL NOTES" in uvm_map.h. This is horrible; among | | 648 | * See "LOCKING PROTOCOL NOTES" in uvm_map.h. This is horrible; among |
649 | * other problems, it defeats any fairness guarantees provided by RW | | 649 | * other problems, it defeats any fairness guarantees provided by RW |
650 | * locks. | | 650 | * locks. |
651 | */ | | 651 | */ |
652 | | | 652 | |
653 | void | | 653 | void |
654 | vm_map_lock(struct vm_map *map) | | 654 | vm_map_lock(struct vm_map *map) |
655 | { | | 655 | { |
656 | | | 656 | |
657 | for (;;) { | | 657 | for (;;) { |
658 | rw_enter(&map->lock, RW_WRITER); | | 658 | rw_enter(&map->lock, RW_WRITER); |
659 | if (map->busy == NULL || map->busy == curlwp) { | | 659 | if (map->busy == NULL || map->busy == curlwp) { |
660 | break; | | 660 | break; |
661 | } | | 661 | } |
662 | mutex_enter(&map->misc_lock); | | 662 | mutex_enter(&map->misc_lock); |
663 | rw_exit(&map->lock); | | 663 | rw_exit(&map->lock); |
664 | if (map->busy != NULL) { | | 664 | if (map->busy != NULL) { |
665 | cv_wait(&map->cv, &map->misc_lock); | | 665 | cv_wait(&map->cv, &map->misc_lock); |
666 | } | | 666 | } |
667 | mutex_exit(&map->misc_lock); | | 667 | mutex_exit(&map->misc_lock); |
668 | } | | 668 | } |
669 | map->timestamp++; | | 669 | map->timestamp++; |
670 | } | | 670 | } |
671 | | | 671 | |
672 | /* | | 672 | /* |
673 | * vm_map_lock_try: try to lock a map, failing if it is already locked. | | 673 | * vm_map_lock_try: try to lock a map, failing if it is already locked. |
674 | */ | | 674 | */ |
675 | | | 675 | |
676 | bool | | 676 | bool |
677 | vm_map_lock_try(struct vm_map *map) | | 677 | vm_map_lock_try(struct vm_map *map) |
678 | { | | 678 | { |
679 | | | 679 | |
680 | if (!rw_tryenter(&map->lock, RW_WRITER)) { | | 680 | if (!rw_tryenter(&map->lock, RW_WRITER)) { |
681 | return false; | | 681 | return false; |
682 | } | | 682 | } |
683 | if (map->busy != NULL) { | | 683 | if (map->busy != NULL) { |
684 | rw_exit(&map->lock); | | 684 | rw_exit(&map->lock); |
685 | return false; | | 685 | return false; |
686 | } | | 686 | } |
687 | map->timestamp++; | | 687 | map->timestamp++; |
688 | return true; | | 688 | return true; |
689 | } | | 689 | } |
690 | | | 690 | |
691 | /* | | 691 | /* |
692 | * vm_map_unlock: release an exclusive lock on a map. | | 692 | * vm_map_unlock: release an exclusive lock on a map. |
693 | */ | | 693 | */ |
694 | | | 694 | |
695 | void | | 695 | void |
696 | vm_map_unlock(struct vm_map *map) | | 696 | vm_map_unlock(struct vm_map *map) |
697 | { | | 697 | { |
698 | | | 698 | |
699 | KASSERT(rw_write_held(&map->lock)); | | 699 | KASSERT(rw_write_held(&map->lock)); |
700 | KASSERT(map->busy == NULL || map->busy == curlwp); | | 700 | KASSERT(map->busy == NULL || map->busy == curlwp); |
701 | rw_exit(&map->lock); | | 701 | rw_exit(&map->lock); |
702 | } | | 702 | } |
703 | | | 703 | |
704 | /* | | 704 | /* |
705 | * vm_map_unbusy: mark the map as unbusy, and wake any waiters that | | 705 | * vm_map_unbusy: mark the map as unbusy, and wake any waiters that |
706 | * want an exclusive lock. | | 706 | * want an exclusive lock. |
707 | */ | | 707 | */ |
708 | | | 708 | |
709 | void | | 709 | void |
710 | vm_map_unbusy(struct vm_map *map) | | 710 | vm_map_unbusy(struct vm_map *map) |
711 | { | | 711 | { |
712 | | | 712 | |
713 | KASSERT(map->busy == curlwp); | | 713 | KASSERT(map->busy == curlwp); |
714 | | | 714 | |
715 | /* | | 715 | /* |
716 | * Safe to clear 'busy' and 'waiters' with only a read lock held: | | 716 | * Safe to clear 'busy' and 'waiters' with only a read lock held: |
717 | * | | 717 | * |
718 | * o they can only be set with a write lock held | | 718 | * o they can only be set with a write lock held |
719 | * o writers are blocked out with a read or write hold | | 719 | * o writers are blocked out with a read or write hold |
720 | * o at any time, only one thread owns the set of values | | 720 | * o at any time, only one thread owns the set of values |
721 | */ | | 721 | */ |
722 | mutex_enter(&map->misc_lock); | | 722 | mutex_enter(&map->misc_lock); |
723 | map->busy = NULL; | | 723 | map->busy = NULL; |
724 | cv_broadcast(&map->cv); | | 724 | cv_broadcast(&map->cv); |
725 | mutex_exit(&map->misc_lock); | | 725 | mutex_exit(&map->misc_lock); |
726 | } | | 726 | } |
727 | | | 727 | |
728 | /* | | 728 | /* |
729 | * vm_map_lock_read: acquire a shared (read) lock on a map. | | 729 | * vm_map_lock_read: acquire a shared (read) lock on a map. |
730 | */ | | 730 | */ |
731 | | | 731 | |
732 | void | | 732 | void |
733 | vm_map_lock_read(struct vm_map *map) | | 733 | vm_map_lock_read(struct vm_map *map) |
734 | { | | 734 | { |
735 | | | 735 | |
736 | rw_enter(&map->lock, RW_READER); | | 736 | rw_enter(&map->lock, RW_READER); |
737 | } | | 737 | } |
738 | | | 738 | |
739 | /* | | 739 | /* |
740 | * vm_map_unlock_read: release a shared lock on a map. | | 740 | * vm_map_unlock_read: release a shared lock on a map. |
741 | */ | | 741 | */ |
742 | | | 742 | |
743 | void | | 743 | void |
744 | vm_map_unlock_read(struct vm_map *map) | | 744 | vm_map_unlock_read(struct vm_map *map) |
745 | { | | 745 | { |
746 | | | 746 | |
747 | rw_exit(&map->lock); | | 747 | rw_exit(&map->lock); |
748 | } | | 748 | } |
749 | | | 749 | |
750 | /* | | 750 | /* |
751 | * vm_map_busy: mark a map as busy. | | 751 | * vm_map_busy: mark a map as busy. |
752 | * | | 752 | * |
753 | * => the caller must hold the map write locked | | 753 | * => the caller must hold the map write locked |
754 | */ | | 754 | */ |
755 | | | 755 | |
756 | void | | 756 | void |
757 | vm_map_busy(struct vm_map *map) | | 757 | vm_map_busy(struct vm_map *map) |
758 | { | | 758 | { |
759 | | | 759 | |
760 | KASSERT(rw_write_held(&map->lock)); | | 760 | KASSERT(rw_write_held(&map->lock)); |
761 | KASSERT(map->busy == NULL); | | 761 | KASSERT(map->busy == NULL); |
762 | | | 762 | |
763 | map->busy = curlwp; | | 763 | map->busy = curlwp; |
764 | } | | 764 | } |
765 | | | 765 | |
766 | /* | | 766 | /* |
767 | * vm_map_locked_p: return true if the map is write locked. | | 767 | * vm_map_locked_p: return true if the map is write locked. |
768 | * | | 768 | * |
769 | * => only for debug purposes like KASSERTs. | | 769 | * => only for debug purposes like KASSERTs. |
770 | * => should not be used to verify that a map is not locked. | | 770 | * => should not be used to verify that a map is not locked. |
771 | */ | | 771 | */ |
772 | | | 772 | |
773 | bool | | 773 | bool |
774 | vm_map_locked_p(struct vm_map *map) | | 774 | vm_map_locked_p(struct vm_map *map) |
775 | { | | 775 | { |
776 | | | 776 | |
777 | return rw_write_held(&map->lock); | | 777 | return rw_write_held(&map->lock); |
778 | } | | 778 | } |
779 | | | 779 | |
780 | /* | | 780 | /* |
781 | * uvm_mapent_alloc: allocate a map entry | | 781 | * uvm_mapent_alloc: allocate a map entry |
782 | */ | | 782 | */ |
783 | | | 783 | |
784 | static struct vm_map_entry * | | 784 | static struct vm_map_entry * |
785 | uvm_mapent_alloc(struct vm_map *map, int flags) | | 785 | uvm_mapent_alloc(struct vm_map *map, int flags) |
786 | { | | 786 | { |
787 | struct vm_map_entry *me; | | 787 | struct vm_map_entry *me; |
788 | int pflags = (flags & UVM_FLAG_NOWAIT) ? PR_NOWAIT : PR_WAITOK; | | 788 | int pflags = (flags & UVM_FLAG_NOWAIT) ? PR_NOWAIT : PR_WAITOK; |
789 | UVMHIST_FUNC("uvm_mapent_alloc"); UVMHIST_CALLED(maphist); | | 789 | UVMHIST_FUNC("uvm_mapent_alloc"); UVMHIST_CALLED(maphist); |
790 | | | 790 | |
791 | me = pool_cache_get(&uvm_map_entry_cache, pflags); | | 791 | me = pool_cache_get(&uvm_map_entry_cache, pflags); |
792 | if (__predict_false(me == NULL)) { | | 792 | if (__predict_false(me == NULL)) { |
793 | return NULL; | | 793 | return NULL; |
794 | } | | 794 | } |
795 | me->flags = 0; | | 795 | me->flags = 0; |
796 | | | 796 | |
797 | UVMHIST_LOG(maphist, "<- new entry=%#jx [kentry=%jd]", (uintptr_t)me, | | 797 | UVMHIST_LOG(maphist, "<- new entry=%#jx [kentry=%jd]", (uintptr_t)me, |
798 | (map == kernel_map), 0, 0); | | 798 | (map == kernel_map), 0, 0); |
799 | return me; | | 799 | return me; |
800 | } | | 800 | } |
801 | | | 801 | |
802 | /* | | 802 | /* |
803 | * uvm_mapent_free: free map entry | | 803 | * uvm_mapent_free: free map entry |
804 | */ | | 804 | */ |
805 | | | 805 | |
806 | static void | | 806 | static void |
807 | uvm_mapent_free(struct vm_map_entry *me) | | 807 | uvm_mapent_free(struct vm_map_entry *me) |
808 | { | | 808 | { |
809 | UVMHIST_FUNC("uvm_mapent_free"); UVMHIST_CALLED(maphist); | | 809 | UVMHIST_FUNC("uvm_mapent_free"); UVMHIST_CALLED(maphist); |
810 | | | 810 | |
811 | UVMHIST_LOG(maphist,"<- freeing map entry=%#jx [flags=%jd]", | | 811 | UVMHIST_LOG(maphist,"<- freeing map entry=%#jx [flags=%jd]", |
812 | (uintptr_t)me, me->flags, 0, 0); | | 812 | (uintptr_t)me, me->flags, 0, 0); |
813 | pool_cache_put(&uvm_map_entry_cache, me); | | 813 | pool_cache_put(&uvm_map_entry_cache, me); |
814 | } | | 814 | } |
815 | | | 815 | |
816 | /* | | 816 | /* |
817 | * uvm_mapent_copy: copy a map entry, preserving flags | | 817 | * uvm_mapent_copy: copy a map entry, preserving flags |
818 | */ | | 818 | */ |
819 | | | 819 | |
820 | static inline void | | 820 | static inline void |
821 | uvm_mapent_copy(struct vm_map_entry *src, struct vm_map_entry *dst) | | 821 | uvm_mapent_copy(struct vm_map_entry *src, struct vm_map_entry *dst) |
822 | { | | 822 | { |
823 | | | 823 | |
824 | memcpy(dst, src, sizeof(*dst)); | | 824 | memcpy(dst, src, sizeof(*dst)); |
825 | dst->flags = 0; | | 825 | dst->flags = 0; |
826 | } | | 826 | } |
827 | | | 827 | |
828 | #if defined(DEBUG) | | 828 | #if defined(DEBUG) |
829 | static void | | 829 | static void |
830 | _uvm_mapent_check(const struct vm_map_entry *entry, const char *file, int line) | | 830 | _uvm_mapent_check(const struct vm_map_entry *entry, const char *file, int line) |
831 | { | | 831 | { |
832 | | | 832 | |
833 | if (entry->start >= entry->end) { | | 833 | if (entry->start >= entry->end) { |
834 | goto bad; | | 834 | goto bad; |
835 | } | | 835 | } |
836 | if (UVM_ET_ISOBJ(entry)) { | | 836 | if (UVM_ET_ISOBJ(entry)) { |
837 | if (entry->object.uvm_obj == NULL) { | | 837 | if (entry->object.uvm_obj == NULL) { |
838 | goto bad; | | 838 | goto bad; |
839 | } | | 839 | } |
840 | } else if (UVM_ET_ISSUBMAP(entry)) { | | 840 | } else if (UVM_ET_ISSUBMAP(entry)) { |
841 | if (entry->object.sub_map == NULL) { | | 841 | if (entry->object.sub_map == NULL) { |
842 | goto bad; | | 842 | goto bad; |
843 | } | | 843 | } |
844 | } else { | | 844 | } else { |
845 | if (entry->object.uvm_obj != NULL || | | 845 | if (entry->object.uvm_obj != NULL || |
846 | entry->object.sub_map != NULL) { | | 846 | entry->object.sub_map != NULL) { |
847 | goto bad; | | 847 | goto bad; |
848 | } | | 848 | } |
849 | } | | 849 | } |
850 | if (!UVM_ET_ISOBJ(entry)) { | | 850 | if (!UVM_ET_ISOBJ(entry)) { |
851 | if (entry->offset != 0) { | | 851 | if (entry->offset != 0) { |
852 | goto bad; | | 852 | goto bad; |
853 | } | | 853 | } |
854 | } | | 854 | } |
855 | | | 855 | |
856 | return; | | 856 | return; |
857 | | | 857 | |
858 | bad: | | 858 | bad: |
859 | panic("%s: bad entry %p (%s:%d)", __func__, entry, file, line); | | 859 | panic("%s: bad entry %p (%s:%d)", __func__, entry, file, line); |
860 | } | | 860 | } |
861 | #endif /* defined(DEBUG) */ | | 861 | #endif /* defined(DEBUG) */ |
862 | | | 862 | |
863 | /* | | 863 | /* |
864 | * uvm_map_entry_unwire: unwire a map entry | | 864 | * uvm_map_entry_unwire: unwire a map entry |
865 | * | | 865 | * |
866 | * => map should be locked by caller | | 866 | * => map should be locked by caller |
867 | */ | | 867 | */ |
868 | | | 868 | |
869 | static inline void | | 869 | static inline void |
870 | uvm_map_entry_unwire(struct vm_map *map, struct vm_map_entry *entry) | | 870 | uvm_map_entry_unwire(struct vm_map *map, struct vm_map_entry *entry) |
871 | { | | 871 | { |
872 | | | 872 | |
873 | entry->wired_count = 0; | | 873 | entry->wired_count = 0; |
874 | uvm_fault_unwire_locked(map, entry->start, entry->end); | | 874 | uvm_fault_unwire_locked(map, entry->start, entry->end); |
875 | } | | 875 | } |
876 | | | 876 | |
877 | | | 877 | |
878 | /* | | 878 | /* |
879 | * wrapper for calling amap_ref() | | 879 | * wrapper for calling amap_ref() |
880 | */ | | 880 | */ |
881 | static inline void | | 881 | static inline void |
882 | uvm_map_reference_amap(struct vm_map_entry *entry, int flags) | | 882 | uvm_map_reference_amap(struct vm_map_entry *entry, int flags) |
883 | { | | 883 | { |
884 | | | 884 | |
885 | amap_ref(entry->aref.ar_amap, entry->aref.ar_pageoff, | | 885 | amap_ref(entry->aref.ar_amap, entry->aref.ar_pageoff, |
886 | (entry->end - entry->start) >> PAGE_SHIFT, flags); | | 886 | (entry->end - entry->start) >> PAGE_SHIFT, flags); |
887 | } | | 887 | } |
888 | | | 888 | |
889 | | | 889 | |
890 | /* | | 890 | /* |
891 | * wrapper for calling amap_unref() | | 891 | * wrapper for calling amap_unref() |
892 | */ | | 892 | */ |
893 | static inline void | | 893 | static inline void |
894 | uvm_map_unreference_amap(struct vm_map_entry *entry, int flags) | | 894 | uvm_map_unreference_amap(struct vm_map_entry *entry, int flags) |
895 | { | | 895 | { |
896 | | | 896 | |
897 | amap_unref(entry->aref.ar_amap, entry->aref.ar_pageoff, | | 897 | amap_unref(entry->aref.ar_amap, entry->aref.ar_pageoff, |
898 | (entry->end - entry->start) >> PAGE_SHIFT, flags); | | 898 | (entry->end - entry->start) >> PAGE_SHIFT, flags); |
899 | } | | 899 | } |
900 | | | 900 | |
901 | | | 901 | |
902 | /* | | 902 | /* |
903 | * uvm_map_init: init mapping system at boot time. | | 903 | * uvm_map_init: init mapping system at boot time. |
904 | */ | | 904 | */ |
905 | | | 905 | |
906 | void | | 906 | void |
907 | uvm_map_init(void) | | 907 | uvm_map_init(void) |
908 | { | | 908 | { |
909 | #if defined(UVMHIST) | | 909 | #if defined(UVMHIST) |
910 | static struct kern_history_ent pdhistbuf[UVMHIST_PDHIST_SIZE]; | | 910 | static struct kern_history_ent pdhistbuf[UVMHIST_PDHIST_SIZE]; |
911 | #endif | | 911 | #endif |
912 | | | 912 | |
913 | /* | | 913 | /* |
914 | * first, init logging system. | | 914 | * first, init logging system. |
915 | */ | | 915 | */ |
916 | | | 916 | |
917 | UVMHIST_FUNC("uvm_map_init"); | | 917 | UVMHIST_FUNC("uvm_map_init"); |
918 | UVMHIST_LINK_STATIC(maphist); | | 918 | UVMHIST_LINK_STATIC(maphist); |
919 | UVMHIST_INIT_STATIC(pdhist, pdhistbuf); | | 919 | UVMHIST_INIT_STATIC(pdhist, pdhistbuf); |
920 | UVMHIST_CALLED(maphist); | | 920 | UVMHIST_CALLED(maphist); |
921 | UVMHIST_LOG(maphist,"<starting uvm map system>", 0, 0, 0, 0); | | 921 | UVMHIST_LOG(maphist,"<starting uvm map system>", 0, 0, 0, 0); |
922 | | | 922 | |
923 | /* | | 923 | /* |
924 | * initialize the global lock for kernel map entry. | | 924 | * initialize the global lock for kernel map entry. |
925 | */ | | 925 | */ |
926 | | | 926 | |
927 | mutex_init(&uvm_kentry_lock, MUTEX_DRIVER, IPL_VM); | | 927 | mutex_init(&uvm_kentry_lock, MUTEX_DRIVER, IPL_VM); |
928 | } | | 928 | } |
929 | | | 929 | |
930 | /* | | 930 | /* |
931 | * uvm_map_init_caches: init mapping system caches. | | 931 | * uvm_map_init_caches: init mapping system caches. |
932 | */ | | 932 | */ |
933 | void | | 933 | void |
934 | uvm_map_init_caches(void) | | 934 | uvm_map_init_caches(void) |
935 | { | | 935 | { |
936 | /* | | 936 | /* |
937 | * initialize caches. | | 937 | * initialize caches. |
938 | */ | | 938 | */ |
939 | | | 939 | |
940 | pool_cache_bootstrap(&uvm_map_entry_cache, sizeof(struct vm_map_entry), | | 940 | pool_cache_bootstrap(&uvm_map_entry_cache, sizeof(struct vm_map_entry), |
941 | coherency_unit, 0, PR_LARGECACHE, "vmmpepl", NULL, IPL_NONE, NULL, | | 941 | coherency_unit, 0, PR_LARGECACHE, "vmmpepl", NULL, IPL_NONE, NULL, |
942 | NULL, NULL); | | 942 | NULL, NULL); |
943 | pool_cache_bootstrap(&uvm_vmspace_cache, sizeof(struct vmspace), | | 943 | pool_cache_bootstrap(&uvm_vmspace_cache, sizeof(struct vmspace), |
944 | 0, 0, 0, "vmsppl", NULL, IPL_NONE, NULL, NULL, NULL); | | 944 | 0, 0, 0, "vmsppl", NULL, IPL_NONE, NULL, NULL, NULL); |
945 | } | | 945 | } |
946 | | | 946 | |
947 | /* | | 947 | /* |
948 | * clippers | | 948 | * clippers |
949 | */ | | 949 | */ |
950 | | | 950 | |
951 | /* | | 951 | /* |
952 | * uvm_mapent_splitadj: adjust map entries for splitting, after uvm_mapent_copy. | | 952 | * uvm_mapent_splitadj: adjust map entries for splitting, after uvm_mapent_copy. |
953 | */ | | 953 | */ |
954 | | | 954 | |
955 | static void | | 955 | static void |
956 | uvm_mapent_splitadj(struct vm_map_entry *entry1, struct vm_map_entry *entry2, | | 956 | uvm_mapent_splitadj(struct vm_map_entry *entry1, struct vm_map_entry *entry2, |
957 | vaddr_t splitat) | | 957 | vaddr_t splitat) |
958 | { | | 958 | { |
959 | vaddr_t adj; | | 959 | vaddr_t adj; |
960 | | | 960 | |
961 | KASSERT(entry1->start < splitat); | | 961 | KASSERT(entry1->start < splitat); |
962 | KASSERT(splitat < entry1->end); | | 962 | KASSERT(splitat < entry1->end); |
963 | | | 963 | |
964 | adj = splitat - entry1->start; | | 964 | adj = splitat - entry1->start; |
965 | entry1->end = entry2->start = splitat; | | 965 | entry1->end = entry2->start = splitat; |
966 | | | 966 | |
967 | if (entry1->aref.ar_amap) { | | 967 | if (entry1->aref.ar_amap) { |
968 | amap_splitref(&entry1->aref, &entry2->aref, adj); | | 968 | amap_splitref(&entry1->aref, &entry2->aref, adj); |
969 | } | | 969 | } |
970 | if (UVM_ET_ISSUBMAP(entry1)) { | | 970 | if (UVM_ET_ISSUBMAP(entry1)) { |
971 | /* ... unlikely to happen, but play it safe */ | | 971 | /* ... unlikely to happen, but play it safe */ |
972 | uvm_map_reference(entry1->object.sub_map); | | 972 | uvm_map_reference(entry1->object.sub_map); |
973 | } else if (UVM_ET_ISOBJ(entry1)) { | | 973 | } else if (UVM_ET_ISOBJ(entry1)) { |
974 | KASSERT(entry1->object.uvm_obj != NULL); /* suppress coverity */ | | 974 | KASSERT(entry1->object.uvm_obj != NULL); /* suppress coverity */ |
975 | entry2->offset += adj; | | 975 | entry2->offset += adj; |
976 | if (entry1->object.uvm_obj->pgops && | | 976 | if (entry1->object.uvm_obj->pgops && |
977 | entry1->object.uvm_obj->pgops->pgo_reference) | | 977 | entry1->object.uvm_obj->pgops->pgo_reference) |
978 | entry1->object.uvm_obj->pgops->pgo_reference( | | 978 | entry1->object.uvm_obj->pgops->pgo_reference( |
979 | entry1->object.uvm_obj); | | 979 | entry1->object.uvm_obj); |
980 | } | | 980 | } |
981 | } | | 981 | } |
982 | | | 982 | |
983 | /* | | 983 | /* |
984 | * uvm_map_clip_start: ensure that the entry begins at or after | | 984 | * uvm_map_clip_start: ensure that the entry begins at or after |
985 | * the starting address, if it doesn't we split the entry. | | 985 | * the starting address, if it doesn't we split the entry. |
986 | * | | 986 | * |
987 | * => caller should use UVM_MAP_CLIP_START macro rather than calling | | 987 | * => caller should use UVM_MAP_CLIP_START macro rather than calling |
988 | * this directly | | 988 | * this directly |
989 | * => map must be locked by caller | | 989 | * => map must be locked by caller |
990 | */ | | 990 | */ |
991 | | | 991 | |
992 | void | | 992 | void |
993 | uvm_map_clip_start(struct vm_map *map, struct vm_map_entry *entry, | | 993 | uvm_map_clip_start(struct vm_map *map, struct vm_map_entry *entry, |
994 | vaddr_t start) | | 994 | vaddr_t start) |
995 | { | | 995 | { |
996 | struct vm_map_entry *new_entry; | | 996 | struct vm_map_entry *new_entry; |
997 | | | 997 | |
998 | /* uvm_map_simplify_entry(map, entry); */ /* XXX */ | | 998 | /* uvm_map_simplify_entry(map, entry); */ /* XXX */ |
999 | | | 999 | |
1000 | uvm_map_check(map, "clip_start entry"); | | 1000 | uvm_map_check(map, "clip_start entry"); |
1001 | uvm_mapent_check(entry); | | 1001 | uvm_mapent_check(entry); |
1002 | | | 1002 | |
1003 | /* | | 1003 | /* |
1004 | * Split off the front portion. note that we must insert the new | | 1004 | * Split off the front portion. note that we must insert the new |
1005 | * entry BEFORE this one, so that this entry has the specified | | 1005 | * entry BEFORE this one, so that this entry has the specified |
1006 | * starting address. | | 1006 | * starting address. |
1007 | */ | | 1007 | */ |
1008 | new_entry = uvm_mapent_alloc(map, 0); | | 1008 | new_entry = uvm_mapent_alloc(map, 0); |
1009 | uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */ | | 1009 | uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */ |
1010 | uvm_mapent_splitadj(new_entry, entry, start); | | 1010 | uvm_mapent_splitadj(new_entry, entry, start); |
1011 | uvm_map_entry_link(map, entry->prev, new_entry); | | 1011 | uvm_map_entry_link(map, entry->prev, new_entry); |
1012 | | | 1012 | |
1013 | uvm_map_check(map, "clip_start leave"); | | 1013 | uvm_map_check(map, "clip_start leave"); |
1014 | } | | 1014 | } |
1015 | | | 1015 | |
1016 | /* | | 1016 | /* |
1017 | * uvm_map_clip_end: ensure that the entry ends at or before | | 1017 | * uvm_map_clip_end: ensure that the entry ends at or before |
1018 | * the ending address, if it does't we split the reference | | 1018 | * the ending address, if it does't we split the reference |
1019 | * | | 1019 | * |
1020 | * => caller should use UVM_MAP_CLIP_END macro rather than calling | | 1020 | * => caller should use UVM_MAP_CLIP_END macro rather than calling |
1021 | * this directly | | 1021 | * this directly |
1022 | * => map must be locked by caller | | 1022 | * => map must be locked by caller |
1023 | */ | | 1023 | */ |
1024 | | | 1024 | |
1025 | void | | 1025 | void |
1026 | uvm_map_clip_end(struct vm_map *map, struct vm_map_entry *entry, vaddr_t end) | | 1026 | uvm_map_clip_end(struct vm_map *map, struct vm_map_entry *entry, vaddr_t end) |
1027 | { | | 1027 | { |
1028 | struct vm_map_entry *new_entry; | | 1028 | struct vm_map_entry *new_entry; |
1029 | | | 1029 | |
1030 | uvm_map_check(map, "clip_end entry"); | | 1030 | uvm_map_check(map, "clip_end entry"); |
1031 | uvm_mapent_check(entry); | | 1031 | uvm_mapent_check(entry); |
1032 | | | 1032 | |
1033 | /* | | 1033 | /* |
1034 | * Create a new entry and insert it | | 1034 | * Create a new entry and insert it |
1035 | * AFTER the specified entry | | 1035 | * AFTER the specified entry |
1036 | */ | | 1036 | */ |
1037 | new_entry = uvm_mapent_alloc(map, 0); | | 1037 | new_entry = uvm_mapent_alloc(map, 0); |
1038 | uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */ | | 1038 | uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */ |
1039 | uvm_mapent_splitadj(entry, new_entry, end); | | 1039 | uvm_mapent_splitadj(entry, new_entry, end); |
1040 | uvm_map_entry_link(map, entry, new_entry); | | 1040 | uvm_map_entry_link(map, entry, new_entry); |
1041 | | | 1041 | |
1042 | uvm_map_check(map, "clip_end leave"); | | 1042 | uvm_map_check(map, "clip_end leave"); |
1043 | } | | 1043 | } |
1044 | | | 1044 | |
1045 | /* | | 1045 | /* |
1046 | * M A P - m a i n e n t r y p o i n t | | 1046 | * M A P - m a i n e n t r y p o i n t |
1047 | */ | | 1047 | */ |
1048 | /* | | 1048 | /* |
1049 | * uvm_map: establish a valid mapping in a map | | 1049 | * uvm_map: establish a valid mapping in a map |
1050 | * | | 1050 | * |
1051 | * => assume startp is page aligned. | | 1051 | * => assume startp is page aligned. |
1052 | * => assume size is a multiple of PAGE_SIZE. | | 1052 | * => assume size is a multiple of PAGE_SIZE. |
1053 | * => assume sys_mmap provides enough of a "hint" to have us skip | | 1053 | * => assume sys_mmap provides enough of a "hint" to have us skip |
1054 | * over text/data/bss area. | | 1054 | * over text/data/bss area. |
1055 | * => map must be unlocked (we will lock it) | | 1055 | * => map must be unlocked (we will lock it) |
1056 | * => <uobj,uoffset> value meanings (4 cases): | | 1056 | * => <uobj,uoffset> value meanings (4 cases): |
1057 | * [1] <NULL,uoffset> == uoffset is a hint for PMAP_PREFER | | 1057 | * [1] <NULL,uoffset> == uoffset is a hint for PMAP_PREFER |
1058 | * [2] <NULL,UVM_UNKNOWN_OFFSET> == don't PMAP_PREFER | | 1058 | * [2] <NULL,UVM_UNKNOWN_OFFSET> == don't PMAP_PREFER |
1059 | * [3] <uobj,uoffset> == normal mapping | | 1059 | * [3] <uobj,uoffset> == normal mapping |
1060 | * [4] <uobj,UVM_UNKNOWN_OFFSET> == uvm_map finds offset based on VA | | 1060 | * [4] <uobj,UVM_UNKNOWN_OFFSET> == uvm_map finds offset based on VA |
1061 | * | | 1061 | * |
1062 | * case [4] is for kernel mappings where we don't know the offset until | | 1062 | * case [4] is for kernel mappings where we don't know the offset until |
1063 | * we've found a virtual address. note that kernel object offsets are | | 1063 | * we've found a virtual address. note that kernel object offsets are |
1064 | * always relative to vm_map_min(kernel_map). | | 1064 | * always relative to vm_map_min(kernel_map). |
1065 | * | | 1065 | * |
1066 | * => if `align' is non-zero, we align the virtual address to the specified | | 1066 | * => if `align' is non-zero, we align the virtual address to the specified |
1067 | * alignment. | | 1067 | * alignment. |
1068 | * this is provided as a mechanism for large pages. | | 1068 | * this is provided as a mechanism for large pages. |
| @@ -1617,3715 +1617,3715 @@ nomerge: | | | @@ -1617,3715 +1617,3715 @@ nomerge: |
1617 | } | | 1617 | } |
1618 | | | 1618 | |
1619 | map->size += size; | | 1619 | map->size += size; |
1620 | | | 1620 | |
1621 | UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0); | | 1621 | UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0); |
1622 | | | 1622 | |
1623 | error = 0; | | 1623 | error = 0; |
1624 | | | 1624 | |
1625 | done: | | 1625 | done: |
1626 | vm_map_unlock(map); | | 1626 | vm_map_unlock(map); |
1627 | | | 1627 | |
1628 | if (new_entry) { | | 1628 | if (new_entry) { |
1629 | uvm_mapent_free(new_entry); | | 1629 | uvm_mapent_free(new_entry); |
1630 | } | | 1630 | } |
1631 | if (dead) { | | 1631 | if (dead) { |
1632 | KDASSERT(merged); | | 1632 | KDASSERT(merged); |
1633 | uvm_mapent_free(dead); | | 1633 | uvm_mapent_free(dead); |
1634 | } | | 1634 | } |
1635 | if (dead_entries) | | 1635 | if (dead_entries) |
1636 | uvm_unmap_detach(dead_entries, 0); | | 1636 | uvm_unmap_detach(dead_entries, 0); |
1637 | | | 1637 | |
1638 | return error; | | 1638 | return error; |
1639 | } | | 1639 | } |
1640 | | | 1640 | |
1641 | /* | | 1641 | /* |
1642 | * uvm_map_lookup_entry_bytree: lookup an entry in tree | | 1642 | * uvm_map_lookup_entry_bytree: lookup an entry in tree |
1643 | */ | | 1643 | */ |
1644 | | | 1644 | |
1645 | static inline bool | | 1645 | static inline bool |
1646 | uvm_map_lookup_entry_bytree(struct vm_map *map, vaddr_t address, | | 1646 | uvm_map_lookup_entry_bytree(struct vm_map *map, vaddr_t address, |
1647 | struct vm_map_entry **entry /* OUT */) | | 1647 | struct vm_map_entry **entry /* OUT */) |
1648 | { | | 1648 | { |
1649 | struct vm_map_entry *prev = &map->header; | | 1649 | struct vm_map_entry *prev = &map->header; |
1650 | struct vm_map_entry *cur = ROOT_ENTRY(map); | | 1650 | struct vm_map_entry *cur = ROOT_ENTRY(map); |
1651 | | | 1651 | |
1652 | while (cur) { | | 1652 | while (cur) { |
1653 | UVMMAP_EVCNT_INCR(mlk_treeloop); | | 1653 | UVMMAP_EVCNT_INCR(mlk_treeloop); |
1654 | if (address >= cur->start) { | | 1654 | if (address >= cur->start) { |
1655 | if (address < cur->end) { | | 1655 | if (address < cur->end) { |
1656 | *entry = cur; | | 1656 | *entry = cur; |
1657 | return true; | | 1657 | return true; |
1658 | } | | 1658 | } |
1659 | prev = cur; | | 1659 | prev = cur; |
1660 | cur = RIGHT_ENTRY(cur); | | 1660 | cur = RIGHT_ENTRY(cur); |
1661 | } else | | 1661 | } else |
1662 | cur = LEFT_ENTRY(cur); | | 1662 | cur = LEFT_ENTRY(cur); |
1663 | } | | 1663 | } |
1664 | *entry = prev; | | 1664 | *entry = prev; |
1665 | return false; | | 1665 | return false; |
1666 | } | | 1666 | } |
1667 | | | 1667 | |
1668 | /* | | 1668 | /* |
1669 | * uvm_map_lookup_entry: find map entry at or before an address | | 1669 | * uvm_map_lookup_entry: find map entry at or before an address |
1670 | * | | 1670 | * |
1671 | * => map must at least be read-locked by caller | | 1671 | * => map must at least be read-locked by caller |
1672 | * => entry is returned in "entry" | | 1672 | * => entry is returned in "entry" |
1673 | * => return value is true if address is in the returned entry | | 1673 | * => return value is true if address is in the returned entry |
1674 | */ | | 1674 | */ |
1675 | | | 1675 | |
1676 | bool | | 1676 | bool |
1677 | uvm_map_lookup_entry(struct vm_map *map, vaddr_t address, | | 1677 | uvm_map_lookup_entry(struct vm_map *map, vaddr_t address, |
1678 | struct vm_map_entry **entry /* OUT */) | | 1678 | struct vm_map_entry **entry /* OUT */) |
1679 | { | | 1679 | { |
1680 | struct vm_map_entry *cur; | | 1680 | struct vm_map_entry *cur; |
1681 | UVMHIST_FUNC("uvm_map_lookup_entry"); | | 1681 | UVMHIST_FUNC("uvm_map_lookup_entry"); |
1682 | UVMHIST_CALLED(maphist); | | 1682 | UVMHIST_CALLED(maphist); |
1683 | | | 1683 | |
1684 | UVMHIST_LOG(maphist,"(map=%#jx,addr=%#jx,ent=%#jx)", | | 1684 | UVMHIST_LOG(maphist,"(map=%#jx,addr=%#jx,ent=%#jx)", |
1685 | (uintptr_t)map, address, (uintptr_t)entry, 0); | | 1685 | (uintptr_t)map, address, (uintptr_t)entry, 0); |
1686 | | | 1686 | |
1687 | /* | | 1687 | /* |
1688 | * make a quick check to see if we are already looking at | | 1688 | * make a quick check to see if we are already looking at |
1689 | * the entry we want (which is usually the case). note also | | 1689 | * the entry we want (which is usually the case). note also |
1690 | * that we don't need to save the hint here... it is the | | 1690 | * that we don't need to save the hint here... it is the |
1691 | * same hint (unless we are at the header, in which case the | | 1691 | * same hint (unless we are at the header, in which case the |
1692 | * hint didn't buy us anything anyway). | | 1692 | * hint didn't buy us anything anyway). |
1693 | */ | | 1693 | */ |
1694 | | | 1694 | |
1695 | cur = map->hint; | | 1695 | cur = map->hint; |
1696 | UVMMAP_EVCNT_INCR(mlk_call); | | 1696 | UVMMAP_EVCNT_INCR(mlk_call); |
1697 | if (cur != &map->header && | | 1697 | if (cur != &map->header && |
1698 | address >= cur->start && cur->end > address) { | | 1698 | address >= cur->start && cur->end > address) { |
1699 | UVMMAP_EVCNT_INCR(mlk_hint); | | 1699 | UVMMAP_EVCNT_INCR(mlk_hint); |
1700 | *entry = cur; | | 1700 | *entry = cur; |
1701 | UVMHIST_LOG(maphist,"<- got it via hint (%#jx)", | | 1701 | UVMHIST_LOG(maphist,"<- got it via hint (%#jx)", |
1702 | (uintptr_t)cur, 0, 0, 0); | | 1702 | (uintptr_t)cur, 0, 0, 0); |
1703 | uvm_mapent_check(*entry); | | 1703 | uvm_mapent_check(*entry); |
1704 | return (true); | | 1704 | return (true); |
1705 | } | | 1705 | } |
1706 | uvm_map_check(map, __func__); | | 1706 | uvm_map_check(map, __func__); |
1707 | | | 1707 | |
1708 | /* | | 1708 | /* |
1709 | * lookup in the tree. | | 1709 | * lookup in the tree. |
1710 | */ | | 1710 | */ |
1711 | | | 1711 | |
1712 | UVMMAP_EVCNT_INCR(mlk_tree); | | 1712 | UVMMAP_EVCNT_INCR(mlk_tree); |
1713 | if (__predict_true(uvm_map_lookup_entry_bytree(map, address, entry))) { | | 1713 | if (__predict_true(uvm_map_lookup_entry_bytree(map, address, entry))) { |
1714 | SAVE_HINT(map, map->hint, *entry); | | 1714 | SAVE_HINT(map, map->hint, *entry); |
1715 | UVMHIST_LOG(maphist,"<- search got it (%#jx)", | | 1715 | UVMHIST_LOG(maphist,"<- search got it (%#jx)", |
1716 | (uintptr_t)cur, 0, 0, 0); | | 1716 | (uintptr_t)cur, 0, 0, 0); |
1717 | KDASSERT((*entry)->start <= address); | | 1717 | KDASSERT((*entry)->start <= address); |
1718 | KDASSERT(address < (*entry)->end); | | 1718 | KDASSERT(address < (*entry)->end); |
1719 | uvm_mapent_check(*entry); | | 1719 | uvm_mapent_check(*entry); |
1720 | return (true); | | 1720 | return (true); |
1721 | } | | 1721 | } |
1722 | | | 1722 | |
1723 | SAVE_HINT(map, map->hint, *entry); | | 1723 | SAVE_HINT(map, map->hint, *entry); |
1724 | UVMHIST_LOG(maphist,"<- failed!",0,0,0,0); | | 1724 | UVMHIST_LOG(maphist,"<- failed!",0,0,0,0); |
1725 | KDASSERT((*entry) == &map->header || (*entry)->end <= address); | | 1725 | KDASSERT((*entry) == &map->header || (*entry)->end <= address); |
1726 | KDASSERT((*entry)->next == &map->header || | | 1726 | KDASSERT((*entry)->next == &map->header || |
1727 | address < (*entry)->next->start); | | 1727 | address < (*entry)->next->start); |
1728 | return (false); | | 1728 | return (false); |
1729 | } | | 1729 | } |
1730 | | | 1730 | |
1731 | /* | | 1731 | /* |
1732 | * See if the range between start and start + length fits in the gap | | 1732 | * See if the range between start and start + length fits in the gap |
1733 | * entry->next->start and entry->end. Returns 1 if fits, 0 if doesn't | | 1733 | * entry->next->start and entry->end. Returns 1 if fits, 0 if doesn't |
1734 | * fit, and -1 address wraps around. | | 1734 | * fit, and -1 address wraps around. |
1735 | */ | | 1735 | */ |
1736 | static int | | 1736 | static int |
1737 | uvm_map_space_avail(vaddr_t *start, vsize_t length, voff_t uoffset, | | 1737 | uvm_map_space_avail(vaddr_t *start, vsize_t length, voff_t uoffset, |
1738 | vsize_t align, int flags, int topdown, struct vm_map_entry *entry) | | 1738 | vsize_t align, int flags, int topdown, struct vm_map_entry *entry) |
1739 | { | | 1739 | { |
1740 | vaddr_t end; | | 1740 | vaddr_t end; |
1741 | | | 1741 | |
1742 | #ifdef PMAP_PREFER | | 1742 | #ifdef PMAP_PREFER |
1743 | /* | | 1743 | /* |
1744 | * push start address forward as needed to avoid VAC alias problems. | | 1744 | * push start address forward as needed to avoid VAC alias problems. |
1745 | * we only do this if a valid offset is specified. | | 1745 | * we only do this if a valid offset is specified. |
1746 | */ | | 1746 | */ |
1747 | | | 1747 | |
1748 | if (uoffset != UVM_UNKNOWN_OFFSET) | | 1748 | if (uoffset != UVM_UNKNOWN_OFFSET) |
1749 | PMAP_PREFER(uoffset, start, length, topdown); | | 1749 | PMAP_PREFER(uoffset, start, length, topdown); |
1750 | #endif | | 1750 | #endif |
1751 | if ((flags & UVM_FLAG_COLORMATCH) != 0) { | | 1751 | if ((flags & UVM_FLAG_COLORMATCH) != 0) { |
1752 | KASSERT(align < uvmexp.ncolors); | | 1752 | KASSERT(align < uvmexp.ncolors); |
1753 | if (uvmexp.ncolors > 1) { | | 1753 | if (uvmexp.ncolors > 1) { |
1754 | const u_int colormask = uvmexp.colormask; | | 1754 | const u_int colormask = uvmexp.colormask; |
1755 | const u_int colorsize = colormask + 1; | | 1755 | const u_int colorsize = colormask + 1; |
1756 | vaddr_t hint = atop(*start); | | 1756 | vaddr_t hint = atop(*start); |
1757 | const u_int color = hint & colormask; | | 1757 | const u_int color = hint & colormask; |
1758 | if (color != align) { | | 1758 | if (color != align) { |
1759 | hint -= color; /* adjust to color boundary */ | | 1759 | hint -= color; /* adjust to color boundary */ |
1760 | KASSERT((hint & colormask) == 0); | | 1760 | KASSERT((hint & colormask) == 0); |
1761 | if (topdown) { | | 1761 | if (topdown) { |
1762 | if (align > color) | | 1762 | if (align > color) |
1763 | hint -= colorsize; | | 1763 | hint -= colorsize; |
1764 | } else { | | 1764 | } else { |
1765 | if (align < color) | | 1765 | if (align < color) |
1766 | hint += colorsize; | | 1766 | hint += colorsize; |
1767 | } | | 1767 | } |
1768 | *start = ptoa(hint + align); /* adjust to color */ | | 1768 | *start = ptoa(hint + align); /* adjust to color */ |
1769 | } | | 1769 | } |
1770 | } | | 1770 | } |
1771 | } else { | | 1771 | } else { |
1772 | KASSERT(powerof2(align)); | | 1772 | KASSERT(powerof2(align)); |
1773 | uvm_map_align_va(start, align, topdown); | | 1773 | uvm_map_align_va(start, align, topdown); |
1774 | /* | | 1774 | /* |
1775 | * XXX Should we PMAP_PREFER() here again? | | 1775 | * XXX Should we PMAP_PREFER() here again? |
1776 | * eh...i think we're okay | | 1776 | * eh...i think we're okay |
1777 | */ | | 1777 | */ |
1778 | } | | 1778 | } |
1779 | | | 1779 | |
1780 | /* | | 1780 | /* |
1781 | * Find the end of the proposed new region. Be sure we didn't | | 1781 | * Find the end of the proposed new region. Be sure we didn't |
1782 | * wrap around the address; if so, we lose. Otherwise, if the | | 1782 | * wrap around the address; if so, we lose. Otherwise, if the |
1783 | * proposed new region fits before the next entry, we win. | | 1783 | * proposed new region fits before the next entry, we win. |
1784 | */ | | 1784 | */ |
1785 | | | 1785 | |
1786 | end = *start + length; | | 1786 | end = *start + length; |
1787 | if (end < *start) | | 1787 | if (end < *start) |
1788 | return (-1); | | 1788 | return (-1); |
1789 | | | 1789 | |
1790 | if (entry->next->start >= end && *start >= entry->end) | | 1790 | if (entry->next->start >= end && *start >= entry->end) |
1791 | return (1); | | 1791 | return (1); |
1792 | | | 1792 | |
1793 | return (0); | | 1793 | return (0); |
1794 | } | | 1794 | } |
1795 | | | 1795 | |
1796 | /* | | 1796 | /* |
1797 | * uvm_map_findspace: find "length" sized space in "map". | | 1797 | * uvm_map_findspace: find "length" sized space in "map". |
1798 | * | | 1798 | * |
1799 | * => "hint" is a hint about where we want it, unless UVM_FLAG_FIXED is | | 1799 | * => "hint" is a hint about where we want it, unless UVM_FLAG_FIXED is |
1800 | * set in "flags" (in which case we insist on using "hint"). | | 1800 | * set in "flags" (in which case we insist on using "hint"). |
1801 | * => "result" is VA returned | | 1801 | * => "result" is VA returned |
1802 | * => uobj/uoffset are to be used to handle VAC alignment, if required | | 1802 | * => uobj/uoffset are to be used to handle VAC alignment, if required |
1803 | * => if "align" is non-zero, we attempt to align to that value. | | 1803 | * => if "align" is non-zero, we attempt to align to that value. |
1804 | * => caller must at least have read-locked map | | 1804 | * => caller must at least have read-locked map |
1805 | * => returns NULL on failure, or pointer to prev. map entry if success | | 1805 | * => returns NULL on failure, or pointer to prev. map entry if success |
1806 | * => note this is a cross between the old vm_map_findspace and vm_map_find | | 1806 | * => note this is a cross between the old vm_map_findspace and vm_map_find |
1807 | */ | | 1807 | */ |
1808 | | | 1808 | |
1809 | struct vm_map_entry * | | 1809 | struct vm_map_entry * |
1810 | uvm_map_findspace(struct vm_map *map, vaddr_t hint, vsize_t length, | | 1810 | uvm_map_findspace(struct vm_map *map, vaddr_t hint, vsize_t length, |
1811 | vaddr_t *result /* OUT */, struct uvm_object *uobj, voff_t uoffset, | | 1811 | vaddr_t *result /* OUT */, struct uvm_object *uobj, voff_t uoffset, |
1812 | vsize_t align, int flags) | | 1812 | vsize_t align, int flags) |
1813 | { | | 1813 | { |
1814 | struct vm_map_entry *entry; | | 1814 | struct vm_map_entry *entry; |
1815 | struct vm_map_entry *child, *prev, *tmp; | | 1815 | struct vm_map_entry *child, *prev, *tmp; |
1816 | vaddr_t orig_hint __diagused; | | 1816 | vaddr_t orig_hint __diagused; |
1817 | const int topdown = map->flags & VM_MAP_TOPDOWN; | | 1817 | const int topdown = map->flags & VM_MAP_TOPDOWN; |
1818 | UVMHIST_FUNC("uvm_map_findspace"); | | 1818 | UVMHIST_FUNC("uvm_map_findspace"); |
1819 | UVMHIST_CALLED(maphist); | | 1819 | UVMHIST_CALLED(maphist); |
1820 | | | 1820 | |
1821 | UVMHIST_LOG(maphist, "(map=%#jx, hint=%#jx, len=%ju, flags=%#jx)", | | 1821 | UVMHIST_LOG(maphist, "(map=%#jx, hint=%#jx, len=%ju, flags=%#jx)", |
1822 | (uintptr_t)map, hint, length, flags); | | 1822 | (uintptr_t)map, hint, length, flags); |
1823 | KASSERT((flags & UVM_FLAG_COLORMATCH) != 0 || powerof2(align)); | | 1823 | KASSERT((flags & UVM_FLAG_COLORMATCH) != 0 || powerof2(align)); |
1824 | KASSERT((flags & UVM_FLAG_COLORMATCH) == 0 || align < uvmexp.ncolors); | | 1824 | KASSERT((flags & UVM_FLAG_COLORMATCH) == 0 || align < uvmexp.ncolors); |
1825 | KASSERT((flags & UVM_FLAG_FIXED) == 0 || align == 0); | | 1825 | KASSERT((flags & UVM_FLAG_FIXED) == 0 || align == 0); |
1826 | | | 1826 | |
1827 | uvm_map_check(map, "map_findspace entry"); | | 1827 | uvm_map_check(map, "map_findspace entry"); |
1828 | | | 1828 | |
1829 | /* | | 1829 | /* |
1830 | * remember the original hint. if we are aligning, then we | | 1830 | * remember the original hint. if we are aligning, then we |
1831 | * may have to try again with no alignment constraint if | | 1831 | * may have to try again with no alignment constraint if |
1832 | * we fail the first time. | | 1832 | * we fail the first time. |
1833 | */ | | 1833 | */ |
1834 | | | 1834 | |
1835 | orig_hint = hint; | | 1835 | orig_hint = hint; |
1836 | if (hint < vm_map_min(map)) { /* check ranges ... */ | | 1836 | if (hint < vm_map_min(map)) { /* check ranges ... */ |
1837 | if (flags & UVM_FLAG_FIXED) { | | 1837 | if (flags & UVM_FLAG_FIXED) { |
1838 | UVMHIST_LOG(maphist,"<- VA below map range",0,0,0,0); | | 1838 | UVMHIST_LOG(maphist,"<- VA below map range",0,0,0,0); |
1839 | return (NULL); | | 1839 | return (NULL); |
1840 | } | | 1840 | } |
1841 | hint = vm_map_min(map); | | 1841 | hint = vm_map_min(map); |
1842 | } | | 1842 | } |
1843 | if (hint > vm_map_max(map)) { | | 1843 | if (hint > vm_map_max(map)) { |
1844 | UVMHIST_LOG(maphist,"<- VA %#jx > range [%#jx->%#jx]", | | 1844 | UVMHIST_LOG(maphist,"<- VA %#jx > range [%#jx->%#jx]", |
1845 | hint, vm_map_min(map), vm_map_max(map), 0); | | 1845 | hint, vm_map_min(map), vm_map_max(map), 0); |
1846 | return (NULL); | | 1846 | return (NULL); |
1847 | } | | 1847 | } |
1848 | | | 1848 | |
1849 | /* | | 1849 | /* |
1850 | * hint may not be aligned properly; we need round up or down it | | 1850 | * hint may not be aligned properly; we need round up or down it |
1851 | * before proceeding further. | | 1851 | * before proceeding further. |
1852 | */ | | 1852 | */ |
1853 | if ((flags & UVM_FLAG_COLORMATCH) == 0) | | 1853 | if ((flags & UVM_FLAG_COLORMATCH) == 0) |
1854 | uvm_map_align_va(&hint, align, topdown); | | 1854 | uvm_map_align_va(&hint, align, topdown); |
1855 | | | 1855 | |
1856 | /* | | 1856 | /* |
1857 | * Look for the first possible address; if there's already | | 1857 | * Look for the first possible address; if there's already |
1858 | * something at this address, we have to start after it. | | 1858 | * something at this address, we have to start after it. |
1859 | */ | | 1859 | */ |
1860 | | | 1860 | |
1861 | /* | | 1861 | /* |
1862 | * @@@: there are four, no, eight cases to consider. | | 1862 | * @@@: there are four, no, eight cases to consider. |
1863 | * | | 1863 | * |
1864 | * 0: found, fixed, bottom up -> fail | | 1864 | * 0: found, fixed, bottom up -> fail |
1865 | * 1: found, fixed, top down -> fail | | 1865 | * 1: found, fixed, top down -> fail |
1866 | * 2: found, not fixed, bottom up -> start after entry->end, | | 1866 | * 2: found, not fixed, bottom up -> start after entry->end, |
1867 | * loop up | | 1867 | * loop up |
1868 | * 3: found, not fixed, top down -> start before entry->start, | | 1868 | * 3: found, not fixed, top down -> start before entry->start, |
1869 | * loop down | | 1869 | * loop down |
1870 | * 4: not found, fixed, bottom up -> check entry->next->start, fail | | 1870 | * 4: not found, fixed, bottom up -> check entry->next->start, fail |
1871 | * 5: not found, fixed, top down -> check entry->next->start, fail | | 1871 | * 5: not found, fixed, top down -> check entry->next->start, fail |
1872 | * 6: not found, not fixed, bottom up -> check entry->next->start, | | 1872 | * 6: not found, not fixed, bottom up -> check entry->next->start, |
1873 | * loop up | | 1873 | * loop up |
1874 | * 7: not found, not fixed, top down -> check entry->next->start, | | 1874 | * 7: not found, not fixed, top down -> check entry->next->start, |
1875 | * loop down | | 1875 | * loop down |
1876 | * | | 1876 | * |
1877 | * as you can see, it reduces to roughly five cases, and that | | 1877 | * as you can see, it reduces to roughly five cases, and that |
1878 | * adding top down mapping only adds one unique case (without | | 1878 | * adding top down mapping only adds one unique case (without |
1879 | * it, there would be four cases). | | 1879 | * it, there would be four cases). |
1880 | */ | | 1880 | */ |
1881 | | | 1881 | |
1882 | if ((flags & UVM_FLAG_FIXED) == 0 && hint == vm_map_min(map)) { | | 1882 | if ((flags & UVM_FLAG_FIXED) == 0 && hint == vm_map_min(map)) { |
1883 | entry = map->first_free; | | 1883 | entry = map->first_free; |
1884 | } else { | | 1884 | } else { |
1885 | if (uvm_map_lookup_entry(map, hint, &entry)) { | | 1885 | if (uvm_map_lookup_entry(map, hint, &entry)) { |
1886 | /* "hint" address already in use ... */ | | 1886 | /* "hint" address already in use ... */ |
1887 | if (flags & UVM_FLAG_FIXED) { | | 1887 | if (flags & UVM_FLAG_FIXED) { |
1888 | UVMHIST_LOG(maphist, "<- fixed & VA in use", | | 1888 | UVMHIST_LOG(maphist, "<- fixed & VA in use", |
1889 | 0, 0, 0, 0); | | 1889 | 0, 0, 0, 0); |
1890 | return (NULL); | | 1890 | return (NULL); |
1891 | } | | 1891 | } |
1892 | if (topdown) | | 1892 | if (topdown) |
1893 | /* Start from lower gap. */ | | 1893 | /* Start from lower gap. */ |
1894 | entry = entry->prev; | | 1894 | entry = entry->prev; |
1895 | } else if (flags & UVM_FLAG_FIXED) { | | 1895 | } else if (flags & UVM_FLAG_FIXED) { |
1896 | if (entry->next->start >= hint + length && | | 1896 | if (entry->next->start >= hint + length && |
1897 | hint + length > hint) | | 1897 | hint + length > hint) |
1898 | goto found; | | 1898 | goto found; |
1899 | | | 1899 | |
1900 | /* "hint" address is gap but too small */ | | 1900 | /* "hint" address is gap but too small */ |
1901 | UVMHIST_LOG(maphist, "<- fixed mapping failed", | | 1901 | UVMHIST_LOG(maphist, "<- fixed mapping failed", |
1902 | 0, 0, 0, 0); | | 1902 | 0, 0, 0, 0); |
1903 | return (NULL); /* only one shot at it ... */ | | 1903 | return (NULL); /* only one shot at it ... */ |
1904 | } else { | | 1904 | } else { |
1905 | /* | | 1905 | /* |
1906 | * See if given hint fits in this gap. | | 1906 | * See if given hint fits in this gap. |
1907 | */ | | 1907 | */ |
1908 | switch (uvm_map_space_avail(&hint, length, | | 1908 | switch (uvm_map_space_avail(&hint, length, |
1909 | uoffset, align, flags, topdown, entry)) { | | 1909 | uoffset, align, flags, topdown, entry)) { |
1910 | case 1: | | 1910 | case 1: |
1911 | goto found; | | 1911 | goto found; |
1912 | case -1: | | 1912 | case -1: |
1913 | goto wraparound; | | 1913 | goto wraparound; |
1914 | } | | 1914 | } |
1915 | | | 1915 | |
1916 | if (topdown) { | | 1916 | if (topdown) { |
1917 | /* | | 1917 | /* |
1918 | * Still there is a chance to fit | | 1918 | * Still there is a chance to fit |
1919 | * if hint > entry->end. | | 1919 | * if hint > entry->end. |
1920 | */ | | 1920 | */ |
1921 | } else { | | 1921 | } else { |
1922 | /* Start from higher gap. */ | | 1922 | /* Start from higher gap. */ |
1923 | entry = entry->next; | | 1923 | entry = entry->next; |
1924 | if (entry == &map->header) | | 1924 | if (entry == &map->header) |
1925 | goto notfound; | | 1925 | goto notfound; |
1926 | goto nextgap; | | 1926 | goto nextgap; |
1927 | } | | 1927 | } |
1928 | } | | 1928 | } |
1929 | } | | 1929 | } |
1930 | | | 1930 | |
1931 | /* | | 1931 | /* |
1932 | * Note that all UVM_FLAGS_FIXED case is already handled. | | 1932 | * Note that all UVM_FLAGS_FIXED case is already handled. |
1933 | */ | | 1933 | */ |
1934 | KDASSERT((flags & UVM_FLAG_FIXED) == 0); | | 1934 | KDASSERT((flags & UVM_FLAG_FIXED) == 0); |
1935 | | | 1935 | |
1936 | /* Try to find the space in the red-black tree */ | | 1936 | /* Try to find the space in the red-black tree */ |
1937 | | | 1937 | |
1938 | /* Check slot before any entry */ | | 1938 | /* Check slot before any entry */ |
1939 | hint = topdown ? entry->next->start - length : entry->end; | | 1939 | hint = topdown ? entry->next->start - length : entry->end; |
1940 | switch (uvm_map_space_avail(&hint, length, uoffset, align, flags, | | 1940 | switch (uvm_map_space_avail(&hint, length, uoffset, align, flags, |
1941 | topdown, entry)) { | | 1941 | topdown, entry)) { |
1942 | case 1: | | 1942 | case 1: |
1943 | goto found; | | 1943 | goto found; |
1944 | case -1: | | 1944 | case -1: |
1945 | goto wraparound; | | 1945 | goto wraparound; |
1946 | } | | 1946 | } |
1947 | | | 1947 | |
1948 | nextgap: | | 1948 | nextgap: |
1949 | KDASSERT((flags & UVM_FLAG_FIXED) == 0); | | 1949 | KDASSERT((flags & UVM_FLAG_FIXED) == 0); |
1950 | /* If there is not enough space in the whole tree, we fail */ | | 1950 | /* If there is not enough space in the whole tree, we fail */ |
1951 | tmp = ROOT_ENTRY(map); | | 1951 | tmp = ROOT_ENTRY(map); |
1952 | if (tmp == NULL || tmp->maxgap < length) | | 1952 | if (tmp == NULL || tmp->maxgap < length) |
1953 | goto notfound; | | 1953 | goto notfound; |
1954 | | | 1954 | |
1955 | prev = NULL; /* previous candidate */ | | 1955 | prev = NULL; /* previous candidate */ |
1956 | | | 1956 | |
1957 | /* Find an entry close to hint that has enough space */ | | 1957 | /* Find an entry close to hint that has enough space */ |
1958 | for (; tmp;) { | | 1958 | for (; tmp;) { |
1959 | KASSERT(tmp->next->start == tmp->end + tmp->gap); | | 1959 | KASSERT(tmp->next->start == tmp->end + tmp->gap); |
1960 | if (topdown) { | | 1960 | if (topdown) { |
1961 | if (tmp->next->start < hint + length && | | 1961 | if (tmp->next->start < hint + length && |
1962 | (prev == NULL || tmp->end > prev->end)) { | | 1962 | (prev == NULL || tmp->end > prev->end)) { |
1963 | if (tmp->gap >= length) | | 1963 | if (tmp->gap >= length) |
1964 | prev = tmp; | | 1964 | prev = tmp; |
1965 | else if ((child = LEFT_ENTRY(tmp)) != NULL | | 1965 | else if ((child = LEFT_ENTRY(tmp)) != NULL |
1966 | && child->maxgap >= length) | | 1966 | && child->maxgap >= length) |
1967 | prev = tmp; | | 1967 | prev = tmp; |
1968 | } | | 1968 | } |
1969 | } else { | | 1969 | } else { |
1970 | if (tmp->end >= hint && | | 1970 | if (tmp->end >= hint && |
1971 | (prev == NULL || tmp->end < prev->end)) { | | 1971 | (prev == NULL || tmp->end < prev->end)) { |
1972 | if (tmp->gap >= length) | | 1972 | if (tmp->gap >= length) |
1973 | prev = tmp; | | 1973 | prev = tmp; |
1974 | else if ((child = RIGHT_ENTRY(tmp)) != NULL | | 1974 | else if ((child = RIGHT_ENTRY(tmp)) != NULL |
1975 | && child->maxgap >= length) | | 1975 | && child->maxgap >= length) |
1976 | prev = tmp; | | 1976 | prev = tmp; |
1977 | } | | 1977 | } |
1978 | } | | 1978 | } |
1979 | if (tmp->next->start < hint + length) | | 1979 | if (tmp->next->start < hint + length) |
1980 | child = RIGHT_ENTRY(tmp); | | 1980 | child = RIGHT_ENTRY(tmp); |
1981 | else if (tmp->end > hint) | | 1981 | else if (tmp->end > hint) |
1982 | child = LEFT_ENTRY(tmp); | | 1982 | child = LEFT_ENTRY(tmp); |
1983 | else { | | 1983 | else { |
1984 | if (tmp->gap >= length) | | 1984 | if (tmp->gap >= length) |
1985 | break; | | 1985 | break; |
1986 | if (topdown) | | 1986 | if (topdown) |
1987 | child = LEFT_ENTRY(tmp); | | 1987 | child = LEFT_ENTRY(tmp); |
1988 | else | | 1988 | else |
1989 | child = RIGHT_ENTRY(tmp); | | 1989 | child = RIGHT_ENTRY(tmp); |
1990 | } | | 1990 | } |
1991 | if (child == NULL || child->maxgap < length) | | 1991 | if (child == NULL || child->maxgap < length) |
1992 | break; | | 1992 | break; |
1993 | tmp = child; | | 1993 | tmp = child; |
1994 | } | | 1994 | } |
1995 | | | 1995 | |
1996 | if (tmp != NULL && tmp->start < hint && hint < tmp->next->start) { | | 1996 | if (tmp != NULL && tmp->start < hint && hint < tmp->next->start) { |
1997 | /* | | 1997 | /* |
1998 | * Check if the entry that we found satifies the | | 1998 | * Check if the entry that we found satifies the |
1999 | * space requirement | | 1999 | * space requirement |
2000 | */ | | 2000 | */ |
2001 | if (topdown) { | | 2001 | if (topdown) { |
2002 | if (hint > tmp->next->start - length) | | 2002 | if (hint > tmp->next->start - length) |
2003 | hint = tmp->next->start - length; | | 2003 | hint = tmp->next->start - length; |
2004 | } else { | | 2004 | } else { |
2005 | if (hint < tmp->end) | | 2005 | if (hint < tmp->end) |
2006 | hint = tmp->end; | | 2006 | hint = tmp->end; |
2007 | } | | 2007 | } |
2008 | switch (uvm_map_space_avail(&hint, length, uoffset, align, | | 2008 | switch (uvm_map_space_avail(&hint, length, uoffset, align, |
2009 | flags, topdown, tmp)) { | | 2009 | flags, topdown, tmp)) { |
2010 | case 1: | | 2010 | case 1: |
2011 | entry = tmp; | | 2011 | entry = tmp; |
2012 | goto found; | | 2012 | goto found; |
2013 | case -1: | | 2013 | case -1: |
2014 | goto wraparound; | | 2014 | goto wraparound; |
2015 | } | | 2015 | } |
2016 | if (tmp->gap >= length) | | 2016 | if (tmp->gap >= length) |
2017 | goto listsearch; | | 2017 | goto listsearch; |
2018 | } | | 2018 | } |
2019 | if (prev == NULL) | | 2019 | if (prev == NULL) |
2020 | goto notfound; | | 2020 | goto notfound; |
2021 | | | 2021 | |
2022 | if (topdown) { | | 2022 | if (topdown) { |
2023 | KASSERT(orig_hint >= prev->next->start - length || | | 2023 | KASSERT(orig_hint >= prev->next->start - length || |
2024 | prev->next->start - length > prev->next->start); | | 2024 | prev->next->start - length > prev->next->start); |
2025 | hint = prev->next->start - length; | | 2025 | hint = prev->next->start - length; |
2026 | } else { | | 2026 | } else { |
2027 | KASSERT(orig_hint <= prev->end); | | 2027 | KASSERT(orig_hint <= prev->end); |
2028 | hint = prev->end; | | 2028 | hint = prev->end; |
2029 | } | | 2029 | } |
2030 | switch (uvm_map_space_avail(&hint, length, uoffset, align, | | 2030 | switch (uvm_map_space_avail(&hint, length, uoffset, align, |
2031 | flags, topdown, prev)) { | | 2031 | flags, topdown, prev)) { |
2032 | case 1: | | 2032 | case 1: |
2033 | entry = prev; | | 2033 | entry = prev; |
2034 | goto found; | | 2034 | goto found; |
2035 | case -1: | | 2035 | case -1: |
2036 | goto wraparound; | | 2036 | goto wraparound; |
2037 | } | | 2037 | } |
2038 | if (prev->gap >= length) | | 2038 | if (prev->gap >= length) |
2039 | goto listsearch; | | 2039 | goto listsearch; |
2040 | | | 2040 | |
2041 | if (topdown) | | 2041 | if (topdown) |
2042 | tmp = LEFT_ENTRY(prev); | | 2042 | tmp = LEFT_ENTRY(prev); |
2043 | else | | 2043 | else |
2044 | tmp = RIGHT_ENTRY(prev); | | 2044 | tmp = RIGHT_ENTRY(prev); |
2045 | for (;;) { | | 2045 | for (;;) { |
2046 | KASSERT(tmp && tmp->maxgap >= length); | | 2046 | KASSERT(tmp && tmp->maxgap >= length); |
2047 | if (topdown) | | 2047 | if (topdown) |
2048 | child = RIGHT_ENTRY(tmp); | | 2048 | child = RIGHT_ENTRY(tmp); |
2049 | else | | 2049 | else |
2050 | child = LEFT_ENTRY(tmp); | | 2050 | child = LEFT_ENTRY(tmp); |
2051 | if (child && child->maxgap >= length) { | | 2051 | if (child && child->maxgap >= length) { |
2052 | tmp = child; | | 2052 | tmp = child; |
2053 | continue; | | 2053 | continue; |
2054 | } | | 2054 | } |
2055 | if (tmp->gap >= length) | | 2055 | if (tmp->gap >= length) |
2056 | break; | | 2056 | break; |
2057 | if (topdown) | | 2057 | if (topdown) |
2058 | tmp = LEFT_ENTRY(tmp); | | 2058 | tmp = LEFT_ENTRY(tmp); |
2059 | else | | 2059 | else |
2060 | tmp = RIGHT_ENTRY(tmp); | | 2060 | tmp = RIGHT_ENTRY(tmp); |
2061 | } | | 2061 | } |
2062 | | | 2062 | |
2063 | if (topdown) { | | 2063 | if (topdown) { |
2064 | KASSERT(orig_hint >= tmp->next->start - length || | | 2064 | KASSERT(orig_hint >= tmp->next->start - length || |
2065 | tmp->next->start - length > tmp->next->start); | | 2065 | tmp->next->start - length > tmp->next->start); |
2066 | hint = tmp->next->start - length; | | 2066 | hint = tmp->next->start - length; |
2067 | } else { | | 2067 | } else { |
2068 | KASSERT(orig_hint <= tmp->end); | | 2068 | KASSERT(orig_hint <= tmp->end); |
2069 | hint = tmp->end; | | 2069 | hint = tmp->end; |
2070 | } | | 2070 | } |
2071 | switch (uvm_map_space_avail(&hint, length, uoffset, align, | | 2071 | switch (uvm_map_space_avail(&hint, length, uoffset, align, |
2072 | flags, topdown, tmp)) { | | 2072 | flags, topdown, tmp)) { |
2073 | case 1: | | 2073 | case 1: |
2074 | entry = tmp; | | 2074 | entry = tmp; |
2075 | goto found; | | 2075 | goto found; |
2076 | case -1: | | 2076 | case -1: |
2077 | goto wraparound; | | 2077 | goto wraparound; |
2078 | } | | 2078 | } |
2079 | | | 2079 | |
2080 | /* | | 2080 | /* |
2081 | * The tree fails to find an entry because of offset or alignment | | 2081 | * The tree fails to find an entry because of offset or alignment |
2082 | * restrictions. Search the list instead. | | 2082 | * restrictions. Search the list instead. |
2083 | */ | | 2083 | */ |
2084 | listsearch: | | 2084 | listsearch: |
2085 | /* | | 2085 | /* |
2086 | * Look through the rest of the map, trying to fit a new region in | | 2086 | * Look through the rest of the map, trying to fit a new region in |
2087 | * the gap between existing regions, or after the very last region. | | 2087 | * the gap between existing regions, or after the very last region. |
2088 | * note: entry->end = base VA of current gap, | | 2088 | * note: entry->end = base VA of current gap, |
2089 | * entry->next->start = VA of end of current gap | | 2089 | * entry->next->start = VA of end of current gap |
2090 | */ | | 2090 | */ |
2091 | | | 2091 | |
2092 | for (;;) { | | 2092 | for (;;) { |
2093 | /* Update hint for current gap. */ | | 2093 | /* Update hint for current gap. */ |
2094 | hint = topdown ? entry->next->start - length : entry->end; | | 2094 | hint = topdown ? entry->next->start - length : entry->end; |
2095 | | | 2095 | |
2096 | /* See if it fits. */ | | 2096 | /* See if it fits. */ |
2097 | switch (uvm_map_space_avail(&hint, length, uoffset, align, | | 2097 | switch (uvm_map_space_avail(&hint, length, uoffset, align, |
2098 | flags, topdown, entry)) { | | 2098 | flags, topdown, entry)) { |
2099 | case 1: | | 2099 | case 1: |
2100 | goto found; | | 2100 | goto found; |
2101 | case -1: | | 2101 | case -1: |
2102 | goto wraparound; | | 2102 | goto wraparound; |
2103 | } | | 2103 | } |
2104 | | | 2104 | |
2105 | /* Advance to next/previous gap */ | | 2105 | /* Advance to next/previous gap */ |
2106 | if (topdown) { | | 2106 | if (topdown) { |
2107 | if (entry == &map->header) { | | 2107 | if (entry == &map->header) { |
2108 | UVMHIST_LOG(maphist, "<- failed (off start)", | | 2108 | UVMHIST_LOG(maphist, "<- failed (off start)", |
2109 | 0,0,0,0); | | 2109 | 0,0,0,0); |
2110 | goto notfound; | | 2110 | goto notfound; |
2111 | } | | 2111 | } |
2112 | entry = entry->prev; | | 2112 | entry = entry->prev; |
2113 | } else { | | 2113 | } else { |
2114 | entry = entry->next; | | 2114 | entry = entry->next; |
2115 | if (entry == &map->header) { | | 2115 | if (entry == &map->header) { |
2116 | UVMHIST_LOG(maphist, "<- failed (off end)", | | 2116 | UVMHIST_LOG(maphist, "<- failed (off end)", |
2117 | 0,0,0,0); | | 2117 | 0,0,0,0); |
2118 | goto notfound; | | 2118 | goto notfound; |
2119 | } | | 2119 | } |
2120 | } | | 2120 | } |
2121 | } | | 2121 | } |
2122 | | | 2122 | |
2123 | found: | | 2123 | found: |
2124 | SAVE_HINT(map, map->hint, entry); | | 2124 | SAVE_HINT(map, map->hint, entry); |
2125 | *result = hint; | | 2125 | *result = hint; |
2126 | UVMHIST_LOG(maphist,"<- got it! (result=%#jx)", hint, 0,0,0); | | 2126 | UVMHIST_LOG(maphist,"<- got it! (result=%#jx)", hint, 0,0,0); |
2127 | KASSERTMSG( topdown || hint >= orig_hint, "hint: %jx, orig_hint: %jx", | | 2127 | KASSERTMSG( topdown || hint >= orig_hint, "hint: %jx, orig_hint: %jx", |
2128 | (uintmax_t)hint, (uintmax_t)orig_hint); | | 2128 | (uintmax_t)hint, (uintmax_t)orig_hint); |
2129 | KASSERTMSG(!topdown || hint <= orig_hint, "hint: %jx, orig_hint: %jx", | | 2129 | KASSERTMSG(!topdown || hint <= orig_hint, "hint: %jx, orig_hint: %jx", |
2130 | (uintmax_t)hint, (uintmax_t)orig_hint); | | 2130 | (uintmax_t)hint, (uintmax_t)orig_hint); |
2131 | KASSERT(entry->end <= hint); | | 2131 | KASSERT(entry->end <= hint); |
2132 | KASSERT(hint + length <= entry->next->start); | | 2132 | KASSERT(hint + length <= entry->next->start); |
2133 | return (entry); | | 2133 | return (entry); |
2134 | | | 2134 | |
2135 | wraparound: | | 2135 | wraparound: |
2136 | UVMHIST_LOG(maphist, "<- failed (wrap around)", 0,0,0,0); | | 2136 | UVMHIST_LOG(maphist, "<- failed (wrap around)", 0,0,0,0); |
2137 | | | 2137 | |
2138 | return (NULL); | | 2138 | return (NULL); |
2139 | | | 2139 | |
2140 | notfound: | | 2140 | notfound: |
2141 | UVMHIST_LOG(maphist, "<- failed (notfound)", 0,0,0,0); | | 2141 | UVMHIST_LOG(maphist, "<- failed (notfound)", 0,0,0,0); |
2142 | | | 2142 | |
2143 | return (NULL); | | 2143 | return (NULL); |
2144 | } | | 2144 | } |
2145 | | | 2145 | |
2146 | /* | | 2146 | /* |
2147 | * U N M A P - m a i n h e l p e r f u n c t i o n s | | 2147 | * U N M A P - m a i n h e l p e r f u n c t i o n s |
2148 | */ | | 2148 | */ |
2149 | | | 2149 | |
2150 | /* | | 2150 | /* |
2151 | * uvm_unmap_remove: remove mappings from a vm_map (from "start" up to "stop") | | 2151 | * uvm_unmap_remove: remove mappings from a vm_map (from "start" up to "stop") |
2152 | * | | 2152 | * |
2153 | * => caller must check alignment and size | | 2153 | * => caller must check alignment and size |
2154 | * => map must be locked by caller | | 2154 | * => map must be locked by caller |
2155 | * => we return a list of map entries that we've remove from the map | | 2155 | * => we return a list of map entries that we've remove from the map |
2156 | * in "entry_list" | | 2156 | * in "entry_list" |
2157 | */ | | 2157 | */ |
2158 | | | 2158 | |
2159 | void | | 2159 | void |
2160 | uvm_unmap_remove(struct vm_map *map, vaddr_t start, vaddr_t end, | | 2160 | uvm_unmap_remove(struct vm_map *map, vaddr_t start, vaddr_t end, |
2161 | struct vm_map_entry **entry_list /* OUT */, int flags) | | 2161 | struct vm_map_entry **entry_list /* OUT */, int flags) |
2162 | { | | 2162 | { |
2163 | struct vm_map_entry *entry, *first_entry, *next; | | 2163 | struct vm_map_entry *entry, *first_entry, *next; |
2164 | vaddr_t len; | | 2164 | vaddr_t len; |
2165 | UVMHIST_FUNC("uvm_unmap_remove"); UVMHIST_CALLED(maphist); | | 2165 | UVMHIST_FUNC("uvm_unmap_remove"); UVMHIST_CALLED(maphist); |
2166 | | | 2166 | |
2167 | UVMHIST_LOG(maphist,"(map=%#jx, start=%#jx, end=%#jx)", | | 2167 | UVMHIST_LOG(maphist,"(map=%#jx, start=%#jx, end=%#jx)", |
2168 | (uintptr_t)map, start, end, 0); | | 2168 | (uintptr_t)map, start, end, 0); |
2169 | VM_MAP_RANGE_CHECK(map, start, end); | | 2169 | VM_MAP_RANGE_CHECK(map, start, end); |
2170 | | | 2170 | |
2171 | uvm_map_check(map, "unmap_remove entry"); | | 2171 | uvm_map_check(map, "unmap_remove entry"); |
2172 | | | 2172 | |
2173 | /* | | 2173 | /* |
2174 | * find first entry | | 2174 | * find first entry |
2175 | */ | | 2175 | */ |
2176 | | | 2176 | |
2177 | if (uvm_map_lookup_entry(map, start, &first_entry) == true) { | | 2177 | if (uvm_map_lookup_entry(map, start, &first_entry) == true) { |
2178 | /* clip and go... */ | | 2178 | /* clip and go... */ |
2179 | entry = first_entry; | | 2179 | entry = first_entry; |
2180 | UVM_MAP_CLIP_START(map, entry, start); | | 2180 | UVM_MAP_CLIP_START(map, entry, start); |
2181 | /* critical! prevents stale hint */ | | 2181 | /* critical! prevents stale hint */ |
2182 | SAVE_HINT(map, entry, entry->prev); | | 2182 | SAVE_HINT(map, entry, entry->prev); |
2183 | } else { | | 2183 | } else { |
2184 | entry = first_entry->next; | | 2184 | entry = first_entry->next; |
2185 | } | | 2185 | } |
2186 | | | 2186 | |
2187 | /* | | 2187 | /* |
2188 | * save the free space hint | | 2188 | * save the free space hint |
2189 | */ | | 2189 | */ |
2190 | | | 2190 | |
2191 | if (map->first_free != &map->header && map->first_free->start >= start) | | 2191 | if (map->first_free != &map->header && map->first_free->start >= start) |
2192 | map->first_free = entry->prev; | | 2192 | map->first_free = entry->prev; |
2193 | | | 2193 | |
2194 | /* | | 2194 | /* |
2195 | * note: we now re-use first_entry for a different task. we remove | | 2195 | * note: we now re-use first_entry for a different task. we remove |
2196 | * a number of map entries from the map and save them in a linked | | 2196 | * a number of map entries from the map and save them in a linked |
2197 | * list headed by "first_entry". once we remove them from the map | | 2197 | * list headed by "first_entry". once we remove them from the map |
2198 | * the caller should unlock the map and drop the references to the | | 2198 | * the caller should unlock the map and drop the references to the |
2199 | * backing objects [c.f. uvm_unmap_detach]. the object is to | | 2199 | * backing objects [c.f. uvm_unmap_detach]. the object is to |
2200 | * separate unmapping from reference dropping. why? | | 2200 | * separate unmapping from reference dropping. why? |
2201 | * [1] the map has to be locked for unmapping | | 2201 | * [1] the map has to be locked for unmapping |
2202 | * [2] the map need not be locked for reference dropping | | 2202 | * [2] the map need not be locked for reference dropping |
2203 | * [3] dropping references may trigger pager I/O, and if we hit | | 2203 | * [3] dropping references may trigger pager I/O, and if we hit |
2204 | * a pager that does synchronous I/O we may have to wait for it. | | 2204 | * a pager that does synchronous I/O we may have to wait for it. |
2205 | * [4] we would like all waiting for I/O to occur with maps unlocked | | 2205 | * [4] we would like all waiting for I/O to occur with maps unlocked |
2206 | * so that we don't block other threads. | | 2206 | * so that we don't block other threads. |
2207 | */ | | 2207 | */ |
2208 | | | 2208 | |
2209 | first_entry = NULL; | | 2209 | first_entry = NULL; |
2210 | *entry_list = NULL; | | 2210 | *entry_list = NULL; |
2211 | | | 2211 | |
2212 | /* | | 2212 | /* |
2213 | * break up the area into map entry sized regions and unmap. note | | 2213 | * break up the area into map entry sized regions and unmap. note |
2214 | * that all mappings have to be removed before we can even consider | | 2214 | * that all mappings have to be removed before we can even consider |
2215 | * dropping references to amaps or VM objects (otherwise we could end | | 2215 | * dropping references to amaps or VM objects (otherwise we could end |
2216 | * up with a mapping to a page on the free list which would be very bad) | | 2216 | * up with a mapping to a page on the free list which would be very bad) |
2217 | */ | | 2217 | */ |
2218 | | | 2218 | |
2219 | while ((entry != &map->header) && (entry->start < end)) { | | 2219 | while ((entry != &map->header) && (entry->start < end)) { |
2220 | KASSERT((entry->flags & UVM_MAP_STATIC) == 0); | | 2220 | KASSERT((entry->flags & UVM_MAP_STATIC) == 0); |
2221 | | | 2221 | |
2222 | UVM_MAP_CLIP_END(map, entry, end); | | 2222 | UVM_MAP_CLIP_END(map, entry, end); |
2223 | next = entry->next; | | 2223 | next = entry->next; |
2224 | len = entry->end - entry->start; | | 2224 | len = entry->end - entry->start; |
2225 | | | 2225 | |
2226 | /* | | 2226 | /* |
2227 | * unwire before removing addresses from the pmap; otherwise | | 2227 | * unwire before removing addresses from the pmap; otherwise |
2228 | * unwiring will put the entries back into the pmap (XXX). | | 2228 | * unwiring will put the entries back into the pmap (XXX). |
2229 | */ | | 2229 | */ |
2230 | | | 2230 | |
2231 | if (VM_MAPENT_ISWIRED(entry)) { | | 2231 | if (VM_MAPENT_ISWIRED(entry)) { |
2232 | uvm_map_entry_unwire(map, entry); | | 2232 | uvm_map_entry_unwire(map, entry); |
2233 | } | | 2233 | } |
2234 | if (flags & UVM_FLAG_VAONLY) { | | 2234 | if (flags & UVM_FLAG_VAONLY) { |
2235 | | | 2235 | |
2236 | /* nothing */ | | 2236 | /* nothing */ |
2237 | | | 2237 | |
2238 | } else if ((map->flags & VM_MAP_PAGEABLE) == 0) { | | 2238 | } else if ((map->flags & VM_MAP_PAGEABLE) == 0) { |
2239 | | | 2239 | |
2240 | /* | | 2240 | /* |
2241 | * if the map is non-pageable, any pages mapped there | | 2241 | * if the map is non-pageable, any pages mapped there |
2242 | * must be wired and entered with pmap_kenter_pa(), | | 2242 | * must be wired and entered with pmap_kenter_pa(), |
2243 | * and we should free any such pages immediately. | | 2243 | * and we should free any such pages immediately. |
2244 | * this is mostly used for kmem_map. | | 2244 | * this is mostly used for kmem_map. |
2245 | */ | | 2245 | */ |
2246 | KASSERT(vm_map_pmap(map) == pmap_kernel()); | | 2246 | KASSERT(vm_map_pmap(map) == pmap_kernel()); |
2247 | | | 2247 | |
2248 | uvm_km_pgremove_intrsafe(map, entry->start, entry->end); | | 2248 | uvm_km_pgremove_intrsafe(map, entry->start, entry->end); |
2249 | } else if (UVM_ET_ISOBJ(entry) && | | 2249 | } else if (UVM_ET_ISOBJ(entry) && |
2250 | UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj)) { | | 2250 | UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj)) { |
2251 | panic("%s: kernel object %p %p\n", | | 2251 | panic("%s: kernel object %p %p\n", |
2252 | __func__, map, entry); | | 2252 | __func__, map, entry); |
2253 | } else if (UVM_ET_ISOBJ(entry) || entry->aref.ar_amap) { | | 2253 | } else if (UVM_ET_ISOBJ(entry) || entry->aref.ar_amap) { |
2254 | /* | | 2254 | /* |
2255 | * remove mappings the standard way. lock object | | 2255 | * remove mappings the standard way. lock object |
2256 | * and/or amap to ensure vm_page state does not | | 2256 | * and/or amap to ensure vm_page state does not |
2257 | * change while in pmap_remove(). | | 2257 | * change while in pmap_remove(). |
2258 | */ | | 2258 | */ |
2259 | | | 2259 | |
2260 | #ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */ | | 2260 | #ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */ |
2261 | uvm_map_lock_entry(entry, RW_WRITER); | | 2261 | uvm_map_lock_entry(entry, RW_WRITER); |
2262 | #else | | 2262 | #else |
2263 | uvm_map_lock_entry(entry, RW_READER); | | 2263 | uvm_map_lock_entry(entry, RW_READER); |
2264 | #endif | | 2264 | #endif |
2265 | pmap_remove(map->pmap, entry->start, entry->end); | | 2265 | pmap_remove(map->pmap, entry->start, entry->end); |
2266 | | | 2266 | |
2267 | /* | | 2267 | /* |
2268 | * note: if map is dying, leave pmap_update() for | | 2268 | * note: if map is dying, leave pmap_update() for |
2269 | * later. if the map is to be reused (exec) then | | 2269 | * later. if the map is to be reused (exec) then |
2270 | * pmap_update() will be called. if the map is | | 2270 | * pmap_update() will be called. if the map is |
2271 | * being disposed of (exit) then pmap_destroy() | | 2271 | * being disposed of (exit) then pmap_destroy() |
2272 | * will be called. | | 2272 | * will be called. |
2273 | */ | | 2273 | */ |
2274 | | | 2274 | |
2275 | if ((map->flags & VM_MAP_DYING) == 0) { | | 2275 | if ((map->flags & VM_MAP_DYING) == 0) { |
2276 | pmap_update(vm_map_pmap(map)); | | 2276 | pmap_update(vm_map_pmap(map)); |
2277 | } else { | | 2277 | } else { |
2278 | KASSERT(vm_map_pmap(map) != pmap_kernel()); | | 2278 | KASSERT(vm_map_pmap(map) != pmap_kernel()); |
2279 | } | | 2279 | } |
2280 | | | 2280 | |
2281 | uvm_map_unlock_entry(entry); | | 2281 | uvm_map_unlock_entry(entry); |
2282 | } | | 2282 | } |
2283 | | | 2283 | |
2284 | #if defined(UVMDEBUG) | | 2284 | #if defined(UVMDEBUG) |
2285 | /* | | 2285 | /* |
2286 | * check if there's remaining mapping, | | 2286 | * check if there's remaining mapping, |
2287 | * which is a bug in caller. | | 2287 | * which is a bug in caller. |
2288 | */ | | 2288 | */ |
2289 | | | 2289 | |
2290 | vaddr_t va; | | 2290 | vaddr_t va; |
2291 | for (va = entry->start; va < entry->end; | | 2291 | for (va = entry->start; va < entry->end; |
2292 | va += PAGE_SIZE) { | | 2292 | va += PAGE_SIZE) { |
2293 | if (pmap_extract(vm_map_pmap(map), va, NULL)) { | | 2293 | if (pmap_extract(vm_map_pmap(map), va, NULL)) { |
2294 | panic("%s: %#"PRIxVADDR" has mapping", | | 2294 | panic("%s: %#"PRIxVADDR" has mapping", |
2295 | __func__, va); | | 2295 | __func__, va); |
2296 | } | | 2296 | } |
2297 | } | | 2297 | } |
2298 | | | 2298 | |
2299 | if (VM_MAP_IS_KERNEL(map) && (flags & UVM_FLAG_NOWAIT) == 0) { | | 2299 | if (VM_MAP_IS_KERNEL(map) && (flags & UVM_FLAG_NOWAIT) == 0) { |
2300 | uvm_km_check_empty(map, entry->start, | | 2300 | uvm_km_check_empty(map, entry->start, |
2301 | entry->end); | | 2301 | entry->end); |
2302 | } | | 2302 | } |
2303 | #endif /* defined(UVMDEBUG) */ | | 2303 | #endif /* defined(UVMDEBUG) */ |
2304 | | | 2304 | |
2305 | /* | | 2305 | /* |
2306 | * remove entry from map and put it on our list of entries | | 2306 | * remove entry from map and put it on our list of entries |
2307 | * that we've nuked. then go to next entry. | | 2307 | * that we've nuked. then go to next entry. |
2308 | */ | | 2308 | */ |
2309 | | | 2309 | |
2310 | UVMHIST_LOG(maphist, " removed map entry %#jx", | | 2310 | UVMHIST_LOG(maphist, " removed map entry %#jx", |
2311 | (uintptr_t)entry, 0, 0, 0); | | 2311 | (uintptr_t)entry, 0, 0, 0); |
2312 | | | 2312 | |
2313 | /* critical! prevents stale hint */ | | 2313 | /* critical! prevents stale hint */ |
2314 | SAVE_HINT(map, entry, entry->prev); | | 2314 | SAVE_HINT(map, entry, entry->prev); |
2315 | | | 2315 | |
2316 | uvm_map_entry_unlink(map, entry); | | 2316 | uvm_map_entry_unlink(map, entry); |
2317 | KASSERT(map->size >= len); | | 2317 | KASSERT(map->size >= len); |
2318 | map->size -= len; | | 2318 | map->size -= len; |
2319 | entry->prev = NULL; | | 2319 | entry->prev = NULL; |
2320 | entry->next = first_entry; | | 2320 | entry->next = first_entry; |
2321 | first_entry = entry; | | 2321 | first_entry = entry; |
2322 | entry = next; | | 2322 | entry = next; |
2323 | } | | 2323 | } |
2324 | | | 2324 | |
2325 | uvm_map_check(map, "unmap_remove leave"); | | 2325 | uvm_map_check(map, "unmap_remove leave"); |
2326 | | | 2326 | |
2327 | /* | | 2327 | /* |
2328 | * now we've cleaned up the map and are ready for the caller to drop | | 2328 | * now we've cleaned up the map and are ready for the caller to drop |
2329 | * references to the mapped objects. | | 2329 | * references to the mapped objects. |
2330 | */ | | 2330 | */ |
2331 | | | 2331 | |
2332 | *entry_list = first_entry; | | 2332 | *entry_list = first_entry; |
2333 | UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0); | | 2333 | UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0); |
2334 | | | 2334 | |
2335 | if (map->flags & VM_MAP_WANTVA) { | | 2335 | if (map->flags & VM_MAP_WANTVA) { |
2336 | mutex_enter(&map->misc_lock); | | 2336 | mutex_enter(&map->misc_lock); |
2337 | map->flags &= ~VM_MAP_WANTVA; | | 2337 | map->flags &= ~VM_MAP_WANTVA; |
2338 | cv_broadcast(&map->cv); | | 2338 | cv_broadcast(&map->cv); |
2339 | mutex_exit(&map->misc_lock); | | 2339 | mutex_exit(&map->misc_lock); |
2340 | } | | 2340 | } |
2341 | } | | 2341 | } |
2342 | | | 2342 | |
2343 | /* | | 2343 | /* |
2344 | * uvm_unmap_detach: drop references in a chain of map entries | | 2344 | * uvm_unmap_detach: drop references in a chain of map entries |
2345 | * | | 2345 | * |
2346 | * => we will free the map entries as we traverse the list. | | 2346 | * => we will free the map entries as we traverse the list. |
2347 | */ | | 2347 | */ |
2348 | | | 2348 | |
2349 | void | | 2349 | void |
2350 | uvm_unmap_detach(struct vm_map_entry *first_entry, int flags) | | 2350 | uvm_unmap_detach(struct vm_map_entry *first_entry, int flags) |
2351 | { | | 2351 | { |
2352 | struct vm_map_entry *next_entry; | | 2352 | struct vm_map_entry *next_entry; |
2353 | UVMHIST_FUNC("uvm_unmap_detach"); UVMHIST_CALLED(maphist); | | 2353 | UVMHIST_FUNC("uvm_unmap_detach"); UVMHIST_CALLED(maphist); |
2354 | | | 2354 | |
2355 | while (first_entry) { | | 2355 | while (first_entry) { |
2356 | KASSERT(!VM_MAPENT_ISWIRED(first_entry)); | | 2356 | KASSERT(!VM_MAPENT_ISWIRED(first_entry)); |
2357 | UVMHIST_LOG(maphist, | | 2357 | UVMHIST_LOG(maphist, |
2358 | " detach %#jx: amap=%#jx, obj=%#jx, submap?=%jd", | | 2358 | " detach %#jx: amap=%#jx, obj=%#jx, submap?=%jd", |
2359 | (uintptr_t)first_entry, | | 2359 | (uintptr_t)first_entry, |
2360 | (uintptr_t)first_entry->aref.ar_amap, | | 2360 | (uintptr_t)first_entry->aref.ar_amap, |
2361 | (uintptr_t)first_entry->object.uvm_obj, | | 2361 | (uintptr_t)first_entry->object.uvm_obj, |
2362 | UVM_ET_ISSUBMAP(first_entry)); | | 2362 | UVM_ET_ISSUBMAP(first_entry)); |
2363 | | | 2363 | |
2364 | /* | | 2364 | /* |
2365 | * drop reference to amap, if we've got one | | 2365 | * drop reference to amap, if we've got one |
2366 | */ | | 2366 | */ |
2367 | | | 2367 | |
2368 | if (first_entry->aref.ar_amap) | | 2368 | if (first_entry->aref.ar_amap) |
2369 | uvm_map_unreference_amap(first_entry, flags); | | 2369 | uvm_map_unreference_amap(first_entry, flags); |
2370 | | | 2370 | |
2371 | /* | | 2371 | /* |
2372 | * drop reference to our backing object, if we've got one | | 2372 | * drop reference to our backing object, if we've got one |
2373 | */ | | 2373 | */ |
2374 | | | 2374 | |
2375 | KASSERT(!UVM_ET_ISSUBMAP(first_entry)); | | 2375 | KASSERT(!UVM_ET_ISSUBMAP(first_entry)); |
2376 | if (UVM_ET_ISOBJ(first_entry) && | | 2376 | if (UVM_ET_ISOBJ(first_entry) && |
2377 | first_entry->object.uvm_obj->pgops->pgo_detach) { | | 2377 | first_entry->object.uvm_obj->pgops->pgo_detach) { |
2378 | (*first_entry->object.uvm_obj->pgops->pgo_detach) | | 2378 | (*first_entry->object.uvm_obj->pgops->pgo_detach) |
2379 | (first_entry->object.uvm_obj); | | 2379 | (first_entry->object.uvm_obj); |
2380 | } | | 2380 | } |
2381 | next_entry = first_entry->next; | | 2381 | next_entry = first_entry->next; |
2382 | uvm_mapent_free(first_entry); | | 2382 | uvm_mapent_free(first_entry); |
2383 | first_entry = next_entry; | | 2383 | first_entry = next_entry; |
2384 | } | | 2384 | } |
2385 | UVMHIST_LOG(maphist, "<- done", 0,0,0,0); | | 2385 | UVMHIST_LOG(maphist, "<- done", 0,0,0,0); |
2386 | } | | 2386 | } |
2387 | | | 2387 | |
2388 | /* | | 2388 | /* |
2389 | * E X T R A C T I O N F U N C T I O N S | | 2389 | * E X T R A C T I O N F U N C T I O N S |
2390 | */ | | 2390 | */ |
2391 | | | 2391 | |
2392 | /* | | 2392 | /* |
2393 | * uvm_map_reserve: reserve space in a vm_map for future use. | | 2393 | * uvm_map_reserve: reserve space in a vm_map for future use. |
2394 | * | | 2394 | * |
2395 | * => we reserve space in a map by putting a dummy map entry in the | | 2395 | * => we reserve space in a map by putting a dummy map entry in the |
2396 | * map (dummy means obj=NULL, amap=NULL, prot=VM_PROT_NONE) | | 2396 | * map (dummy means obj=NULL, amap=NULL, prot=VM_PROT_NONE) |
2397 | * => map should be unlocked (we will write lock it) | | 2397 | * => map should be unlocked (we will write lock it) |
2398 | * => we return true if we were able to reserve space | | 2398 | * => we return true if we were able to reserve space |
2399 | * => XXXCDC: should be inline? | | 2399 | * => XXXCDC: should be inline? |
2400 | */ | | 2400 | */ |
2401 | | | 2401 | |
2402 | int | | 2402 | int |
2403 | uvm_map_reserve(struct vm_map *map, vsize_t size, | | 2403 | uvm_map_reserve(struct vm_map *map, vsize_t size, |
2404 | vaddr_t offset /* hint for pmap_prefer */, | | 2404 | vaddr_t offset /* hint for pmap_prefer */, |
2405 | vsize_t align /* alignment */, | | 2405 | vsize_t align /* alignment */, |
2406 | vaddr_t *raddr /* IN:hint, OUT: reserved VA */, | | 2406 | vaddr_t *raddr /* IN:hint, OUT: reserved VA */, |
2407 | uvm_flag_t flags /* UVM_FLAG_FIXED or UVM_FLAG_COLORMATCH or 0 */) | | 2407 | uvm_flag_t flags /* UVM_FLAG_FIXED or UVM_FLAG_COLORMATCH or 0 */) |
2408 | { | | 2408 | { |
2409 | UVMHIST_FUNC("uvm_map_reserve"); UVMHIST_CALLED(maphist); | | 2409 | UVMHIST_FUNC("uvm_map_reserve"); UVMHIST_CALLED(maphist); |
2410 | | | 2410 | |
2411 | UVMHIST_LOG(maphist, "(map=%#jx, size=%#jx, offset=%#jx, addr=%#jx)", | | 2411 | UVMHIST_LOG(maphist, "(map=%#jx, size=%#jx, offset=%#jx, addr=%#jx)", |
2412 | (uintptr_t)map, size, offset, (uintptr_t)raddr); | | 2412 | (uintptr_t)map, size, offset, (uintptr_t)raddr); |
2413 | | | 2413 | |
2414 | size = round_page(size); | | 2414 | size = round_page(size); |
2415 | | | 2415 | |
2416 | /* | | 2416 | /* |
2417 | * reserve some virtual space. | | 2417 | * reserve some virtual space. |
2418 | */ | | 2418 | */ |
2419 | | | 2419 | |
2420 | if (uvm_map(map, raddr, size, NULL, offset, align, | | 2420 | if (uvm_map(map, raddr, size, NULL, offset, align, |
2421 | UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE, | | 2421 | UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE, |
2422 | UVM_ADV_RANDOM, UVM_FLAG_NOMERGE|flags)) != 0) { | | 2422 | UVM_ADV_RANDOM, UVM_FLAG_NOMERGE|flags)) != 0) { |
2423 | UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0); | | 2423 | UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0); |
2424 | return (false); | | 2424 | return (false); |
2425 | } | | 2425 | } |
2426 | | | 2426 | |
2427 | UVMHIST_LOG(maphist, "<- done (*raddr=%#jx)", *raddr,0,0,0); | | 2427 | UVMHIST_LOG(maphist, "<- done (*raddr=%#jx)", *raddr,0,0,0); |
2428 | return (true); | | 2428 | return (true); |
2429 | } | | 2429 | } |
2430 | | | 2430 | |
2431 | /* | | 2431 | /* |
2432 | * uvm_map_replace: replace a reserved (blank) area of memory with | | 2432 | * uvm_map_replace: replace a reserved (blank) area of memory with |
2433 | * real mappings. | | 2433 | * real mappings. |
2434 | * | | 2434 | * |
2435 | * => caller must WRITE-LOCK the map | | 2435 | * => caller must WRITE-LOCK the map |
2436 | * => we return true if replacement was a success | | 2436 | * => we return true if replacement was a success |
2437 | * => we expect the newents chain to have nnewents entrys on it and | | 2437 | * => we expect the newents chain to have nnewents entrys on it and |
2438 | * we expect newents->prev to point to the last entry on the list | | 2438 | * we expect newents->prev to point to the last entry on the list |
2439 | * => note newents is allowed to be NULL | | 2439 | * => note newents is allowed to be NULL |
2440 | */ | | 2440 | */ |
2441 | | | 2441 | |
2442 | static int | | 2442 | static int |
2443 | uvm_map_replace(struct vm_map *map, vaddr_t start, vaddr_t end, | | 2443 | uvm_map_replace(struct vm_map *map, vaddr_t start, vaddr_t end, |
2444 | struct vm_map_entry *newents, int nnewents, vsize_t nsize, | | 2444 | struct vm_map_entry *newents, int nnewents, vsize_t nsize, |
2445 | struct vm_map_entry **oldentryp) | | 2445 | struct vm_map_entry **oldentryp) |
2446 | { | | 2446 | { |
2447 | struct vm_map_entry *oldent, *last; | | 2447 | struct vm_map_entry *oldent, *last; |
2448 | | | 2448 | |
2449 | uvm_map_check(map, "map_replace entry"); | | 2449 | uvm_map_check(map, "map_replace entry"); |
2450 | | | 2450 | |
2451 | /* | | 2451 | /* |
2452 | * first find the blank map entry at the specified address | | 2452 | * first find the blank map entry at the specified address |
2453 | */ | | 2453 | */ |
2454 | | | 2454 | |
2455 | if (!uvm_map_lookup_entry(map, start, &oldent)) { | | 2455 | if (!uvm_map_lookup_entry(map, start, &oldent)) { |
2456 | return (false); | | 2456 | return (false); |
2457 | } | | 2457 | } |
2458 | | | 2458 | |
2459 | /* | | 2459 | /* |
2460 | * check to make sure we have a proper blank entry | | 2460 | * check to make sure we have a proper blank entry |
2461 | */ | | 2461 | */ |
2462 | | | 2462 | |
2463 | if (end < oldent->end) { | | 2463 | if (end < oldent->end) { |
2464 | UVM_MAP_CLIP_END(map, oldent, end); | | 2464 | UVM_MAP_CLIP_END(map, oldent, end); |
2465 | } | | 2465 | } |
2466 | if (oldent->start != start || oldent->end != end || | | 2466 | if (oldent->start != start || oldent->end != end || |
2467 | oldent->object.uvm_obj != NULL || oldent->aref.ar_amap != NULL) { | | 2467 | oldent->object.uvm_obj != NULL || oldent->aref.ar_amap != NULL) { |
2468 | return (false); | | 2468 | return (false); |
2469 | } | | 2469 | } |
2470 | | | 2470 | |
2471 | #ifdef DIAGNOSTIC | | 2471 | #ifdef DIAGNOSTIC |
2472 | | | 2472 | |
2473 | /* | | 2473 | /* |
2474 | * sanity check the newents chain | | 2474 | * sanity check the newents chain |
2475 | */ | | 2475 | */ |
2476 | | | 2476 | |
2477 | { | | 2477 | { |
2478 | struct vm_map_entry *tmpent = newents; | | 2478 | struct vm_map_entry *tmpent = newents; |
2479 | int nent = 0; | | 2479 | int nent = 0; |
2480 | vsize_t sz = 0; | | 2480 | vsize_t sz = 0; |
2481 | vaddr_t cur = start; | | 2481 | vaddr_t cur = start; |
2482 | | | 2482 | |
2483 | while (tmpent) { | | 2483 | while (tmpent) { |
2484 | nent++; | | 2484 | nent++; |
2485 | sz += tmpent->end - tmpent->start; | | 2485 | sz += tmpent->end - tmpent->start; |
2486 | if (tmpent->start < cur) | | 2486 | if (tmpent->start < cur) |
2487 | panic("uvm_map_replace1"); | | 2487 | panic("uvm_map_replace1"); |
2488 | if (tmpent->start >= tmpent->end || tmpent->end > end) { | | 2488 | if (tmpent->start >= tmpent->end || tmpent->end > end) { |
2489 | panic("uvm_map_replace2: " | | 2489 | panic("uvm_map_replace2: " |
2490 | "tmpent->start=%#"PRIxVADDR | | 2490 | "tmpent->start=%#"PRIxVADDR |
2491 | ", tmpent->end=%#"PRIxVADDR | | 2491 | ", tmpent->end=%#"PRIxVADDR |
2492 | ", end=%#"PRIxVADDR, | | 2492 | ", end=%#"PRIxVADDR, |
2493 | tmpent->start, tmpent->end, end); | | 2493 | tmpent->start, tmpent->end, end); |
2494 | } | | 2494 | } |
2495 | cur = tmpent->end; | | 2495 | cur = tmpent->end; |
2496 | if (tmpent->next) { | | 2496 | if (tmpent->next) { |
2497 | if (tmpent->next->prev != tmpent) | | 2497 | if (tmpent->next->prev != tmpent) |
2498 | panic("uvm_map_replace3"); | | 2498 | panic("uvm_map_replace3"); |
2499 | } else { | | 2499 | } else { |
2500 | if (newents->prev != tmpent) | | 2500 | if (newents->prev != tmpent) |
2501 | panic("uvm_map_replace4"); | | 2501 | panic("uvm_map_replace4"); |
2502 | } | | 2502 | } |
2503 | tmpent = tmpent->next; | | 2503 | tmpent = tmpent->next; |
2504 | } | | 2504 | } |
2505 | if (nent != nnewents) | | 2505 | if (nent != nnewents) |
2506 | panic("uvm_map_replace5"); | | 2506 | panic("uvm_map_replace5"); |
2507 | if (sz != nsize) | | 2507 | if (sz != nsize) |
2508 | panic("uvm_map_replace6"); | | 2508 | panic("uvm_map_replace6"); |
2509 | } | | 2509 | } |
2510 | #endif | | 2510 | #endif |
2511 | | | 2511 | |
2512 | /* | | 2512 | /* |
2513 | * map entry is a valid blank! replace it. (this does all the | | 2513 | * map entry is a valid blank! replace it. (this does all the |
2514 | * work of map entry link/unlink...). | | 2514 | * work of map entry link/unlink...). |
2515 | */ | | 2515 | */ |
2516 | | | 2516 | |
2517 | if (newents) { | | 2517 | if (newents) { |
2518 | last = newents->prev; | | 2518 | last = newents->prev; |
2519 | | | 2519 | |
2520 | /* critical: flush stale hints out of map */ | | 2520 | /* critical: flush stale hints out of map */ |
2521 | SAVE_HINT(map, map->hint, newents); | | 2521 | SAVE_HINT(map, map->hint, newents); |
2522 | if (map->first_free == oldent) | | 2522 | if (map->first_free == oldent) |
2523 | map->first_free = last; | | 2523 | map->first_free = last; |
2524 | | | 2524 | |
2525 | last->next = oldent->next; | | 2525 | last->next = oldent->next; |
2526 | last->next->prev = last; | | 2526 | last->next->prev = last; |
2527 | | | 2527 | |
2528 | /* Fix RB tree */ | | 2528 | /* Fix RB tree */ |
2529 | uvm_rb_remove(map, oldent); | | 2529 | uvm_rb_remove(map, oldent); |
2530 | | | 2530 | |
2531 | newents->prev = oldent->prev; | | 2531 | newents->prev = oldent->prev; |
2532 | newents->prev->next = newents; | | 2532 | newents->prev->next = newents; |
2533 | map->nentries = map->nentries + (nnewents - 1); | | 2533 | map->nentries = map->nentries + (nnewents - 1); |
2534 | | | 2534 | |
2535 | /* Fixup the RB tree */ | | 2535 | /* Fixup the RB tree */ |
2536 | { | | 2536 | { |
2537 | int i; | | 2537 | int i; |
2538 | struct vm_map_entry *tmp; | | 2538 | struct vm_map_entry *tmp; |
2539 | | | 2539 | |
2540 | tmp = newents; | | 2540 | tmp = newents; |
2541 | for (i = 0; i < nnewents && tmp; i++) { | | 2541 | for (i = 0; i < nnewents && tmp; i++) { |
2542 | uvm_rb_insert(map, tmp); | | 2542 | uvm_rb_insert(map, tmp); |
2543 | tmp = tmp->next; | | 2543 | tmp = tmp->next; |
2544 | } | | 2544 | } |
2545 | } | | 2545 | } |
2546 | } else { | | 2546 | } else { |
2547 | /* NULL list of new entries: just remove the old one */ | | 2547 | /* NULL list of new entries: just remove the old one */ |
2548 | clear_hints(map, oldent); | | 2548 | clear_hints(map, oldent); |
2549 | uvm_map_entry_unlink(map, oldent); | | 2549 | uvm_map_entry_unlink(map, oldent); |
2550 | } | | 2550 | } |
2551 | map->size -= end - start - nsize; | | 2551 | map->size -= end - start - nsize; |
2552 | | | 2552 | |
2553 | uvm_map_check(map, "map_replace leave"); | | 2553 | uvm_map_check(map, "map_replace leave"); |
2554 | | | 2554 | |
2555 | /* | | 2555 | /* |
2556 | * now we can free the old blank entry and return. | | 2556 | * now we can free the old blank entry and return. |
2557 | */ | | 2557 | */ |
2558 | | | 2558 | |
2559 | *oldentryp = oldent; | | 2559 | *oldentryp = oldent; |
2560 | return (true); | | 2560 | return (true); |
2561 | } | | 2561 | } |
2562 | | | 2562 | |
2563 | /* | | 2563 | /* |
2564 | * uvm_map_extract: extract a mapping from a map and put it somewhere | | 2564 | * uvm_map_extract: extract a mapping from a map and put it somewhere |
2565 | * (maybe removing the old mapping) | | 2565 | * (maybe removing the old mapping) |
2566 | * | | 2566 | * |
2567 | * => maps should be unlocked (we will write lock them) | | 2567 | * => maps should be unlocked (we will write lock them) |
2568 | * => returns 0 on success, error code otherwise | | 2568 | * => returns 0 on success, error code otherwise |
2569 | * => start must be page aligned | | 2569 | * => start must be page aligned |
2570 | * => len must be page sized | | 2570 | * => len must be page sized |
2571 | * => flags: | | 2571 | * => flags: |
2572 | * UVM_EXTRACT_REMOVE: remove mappings from srcmap | | 2572 | * UVM_EXTRACT_REMOVE: remove mappings from srcmap |
2573 | * UVM_EXTRACT_CONTIG: abort if unmapped area (advisory only) | | 2573 | * UVM_EXTRACT_CONTIG: abort if unmapped area (advisory only) |
2574 | * UVM_EXTRACT_QREF: for a temporary extraction do quick obj refs | | 2574 | * UVM_EXTRACT_QREF: for a temporary extraction do quick obj refs |
2575 | * UVM_EXTRACT_FIXPROT: set prot to maxprot as we go | | 2575 | * UVM_EXTRACT_FIXPROT: set prot to maxprot as we go |
2576 | * UVM_EXTRACT_PROT_ALL: set prot to UVM_PROT_ALL as we go | | 2576 | * UVM_EXTRACT_PROT_ALL: set prot to UVM_PROT_ALL as we go |
2577 | * >>>NOTE: if you set REMOVE, you are not allowed to use CONTIG or QREF!<<< | | 2577 | * >>>NOTE: if you set REMOVE, you are not allowed to use CONTIG or QREF!<<< |
2578 | * >>>NOTE: QREF's must be unmapped via the QREF path, thus should only | | 2578 | * >>>NOTE: QREF's must be unmapped via the QREF path, thus should only |
2579 | * be used from within the kernel in a kernel level map <<< | | 2579 | * be used from within the kernel in a kernel level map <<< |
2580 | */ | | 2580 | */ |
2581 | | | 2581 | |
2582 | int | | 2582 | int |
2583 | uvm_map_extract(struct vm_map *srcmap, vaddr_t start, vsize_t len, | | 2583 | uvm_map_extract(struct vm_map *srcmap, vaddr_t start, vsize_t len, |
2584 | struct vm_map *dstmap, vaddr_t *dstaddrp, int flags) | | 2584 | struct vm_map *dstmap, vaddr_t *dstaddrp, int flags) |
2585 | { | | 2585 | { |
2586 | vaddr_t dstaddr, end, newend, oldoffset, fudge, orig_fudge; | | 2586 | vaddr_t dstaddr, end, newend, oldoffset, fudge, orig_fudge; |
2587 | struct vm_map_entry *chain, *endchain, *entry, *orig_entry, *newentry, | | 2587 | struct vm_map_entry *chain, *endchain, *entry, *orig_entry, *newentry, |
2588 | *deadentry, *oldentry; | | 2588 | *deadentry, *oldentry; |
2589 | struct vm_map_entry *resentry = NULL; /* a dummy reservation entry */ | | 2589 | struct vm_map_entry *resentry = NULL; /* a dummy reservation entry */ |
2590 | vsize_t elen __unused; | | 2590 | vsize_t elen __unused; |
2591 | int nchain, error, copy_ok; | | 2591 | int nchain, error, copy_ok; |
2592 | vsize_t nsize; | | 2592 | vsize_t nsize; |
2593 | UVMHIST_FUNC("uvm_map_extract"); UVMHIST_CALLED(maphist); | | 2593 | UVMHIST_FUNC("uvm_map_extract"); UVMHIST_CALLED(maphist); |
2594 | | | 2594 | |
2595 | UVMHIST_LOG(maphist,"(srcmap=%#jx,start=%#jx, len=%#jx", | | 2595 | UVMHIST_LOG(maphist,"(srcmap=%#jx,start=%#jx, len=%#jx", |
2596 | (uintptr_t)srcmap, start, len, 0); | | 2596 | (uintptr_t)srcmap, start, len, 0); |
2597 | UVMHIST_LOG(maphist," ...,dstmap=%#jx, flags=%#jx)", | | 2597 | UVMHIST_LOG(maphist," ...,dstmap=%#jx, flags=%#jx)", |
2598 | (uintptr_t)dstmap, flags, 0, 0); | | 2598 | (uintptr_t)dstmap, flags, 0, 0); |
2599 | | | 2599 | |
2600 | /* | | 2600 | /* |
2601 | * step 0: sanity check: start must be on a page boundary, length | | 2601 | * step 0: sanity check: start must be on a page boundary, length |
2602 | * must be page sized. can't ask for CONTIG/QREF if you asked for | | 2602 | * must be page sized. can't ask for CONTIG/QREF if you asked for |
2603 | * REMOVE. | | 2603 | * REMOVE. |
2604 | */ | | 2604 | */ |
2605 | | | 2605 | |
2606 | KASSERT((start & PAGE_MASK) == 0 && (len & PAGE_MASK) == 0); | | 2606 | KASSERT((start & PAGE_MASK) == 0 && (len & PAGE_MASK) == 0); |
2607 | KASSERT((flags & UVM_EXTRACT_REMOVE) == 0 || | | 2607 | KASSERT((flags & UVM_EXTRACT_REMOVE) == 0 || |
2608 | (flags & (UVM_EXTRACT_CONTIG|UVM_EXTRACT_QREF)) == 0); | | 2608 | (flags & (UVM_EXTRACT_CONTIG|UVM_EXTRACT_QREF)) == 0); |
2609 | | | 2609 | |
2610 | /* | | 2610 | /* |
2611 | * step 1: reserve space in the target map for the extracted area | | 2611 | * step 1: reserve space in the target map for the extracted area |
2612 | */ | | 2612 | */ |
2613 | | | 2613 | |
2614 | if ((flags & UVM_EXTRACT_RESERVED) == 0) { | | 2614 | if ((flags & UVM_EXTRACT_RESERVED) == 0) { |
2615 | dstaddr = vm_map_min(dstmap); | | 2615 | dstaddr = vm_map_min(dstmap); |
2616 | if (!uvm_map_reserve(dstmap, len, start, | | 2616 | if (!uvm_map_reserve(dstmap, len, start, |
2617 | atop(start) & uvmexp.colormask, &dstaddr, | | 2617 | atop(start) & uvmexp.colormask, &dstaddr, |
2618 | UVM_FLAG_COLORMATCH)) | | 2618 | UVM_FLAG_COLORMATCH)) |
2619 | return (ENOMEM); | | 2619 | return (ENOMEM); |
2620 | KASSERT((atop(start ^ dstaddr) & uvmexp.colormask) == 0); | | 2620 | KASSERT((atop(start ^ dstaddr) & uvmexp.colormask) == 0); |
2621 | *dstaddrp = dstaddr; /* pass address back to caller */ | | 2621 | *dstaddrp = dstaddr; /* pass address back to caller */ |
2622 | UVMHIST_LOG(maphist, " dstaddr=%#jx", dstaddr,0,0,0); | | 2622 | UVMHIST_LOG(maphist, " dstaddr=%#jx", dstaddr,0,0,0); |
2623 | } else { | | 2623 | } else { |
2624 | dstaddr = *dstaddrp; | | 2624 | dstaddr = *dstaddrp; |
2625 | } | | 2625 | } |
2626 | | | 2626 | |
2627 | /* | | 2627 | /* |
2628 | * step 2: setup for the extraction process loop by init'ing the | | 2628 | * step 2: setup for the extraction process loop by init'ing the |
2629 | * map entry chain, locking src map, and looking up the first useful | | 2629 | * map entry chain, locking src map, and looking up the first useful |
2630 | * entry in the map. | | 2630 | * entry in the map. |
2631 | */ | | 2631 | */ |
2632 | | | 2632 | |
2633 | end = start + len; | | 2633 | end = start + len; |
2634 | newend = dstaddr + len; | | 2634 | newend = dstaddr + len; |
2635 | chain = endchain = NULL; | | 2635 | chain = endchain = NULL; |
2636 | nchain = 0; | | 2636 | nchain = 0; |
2637 | nsize = 0; | | 2637 | nsize = 0; |
2638 | vm_map_lock(srcmap); | | 2638 | vm_map_lock(srcmap); |
2639 | | | 2639 | |
2640 | if (uvm_map_lookup_entry(srcmap, start, &entry)) { | | 2640 | if (uvm_map_lookup_entry(srcmap, start, &entry)) { |
2641 | | | 2641 | |
2642 | /* "start" is within an entry */ | | 2642 | /* "start" is within an entry */ |
2643 | if (flags & UVM_EXTRACT_QREF) { | | 2643 | if (flags & UVM_EXTRACT_QREF) { |
2644 | | | 2644 | |
2645 | /* | | 2645 | /* |
2646 | * for quick references we don't clip the entry, so | | 2646 | * for quick references we don't clip the entry, so |
2647 | * the entry may map space "before" the starting | | 2647 | * the entry may map space "before" the starting |
2648 | * virtual address... this is the "fudge" factor | | 2648 | * virtual address... this is the "fudge" factor |
2649 | * (which can be non-zero only the first time | | 2649 | * (which can be non-zero only the first time |
2650 | * through the "while" loop in step 3). | | 2650 | * through the "while" loop in step 3). |
2651 | */ | | 2651 | */ |
2652 | | | 2652 | |
2653 | fudge = start - entry->start; | | 2653 | fudge = start - entry->start; |
2654 | } else { | | 2654 | } else { |
2655 | | | 2655 | |
2656 | /* | | 2656 | /* |
2657 | * normal reference: we clip the map to fit (thus | | 2657 | * normal reference: we clip the map to fit (thus |
2658 | * fudge is zero) | | 2658 | * fudge is zero) |
2659 | */ | | 2659 | */ |
2660 | | | 2660 | |
2661 | UVM_MAP_CLIP_START(srcmap, entry, start); | | 2661 | UVM_MAP_CLIP_START(srcmap, entry, start); |
2662 | SAVE_HINT(srcmap, srcmap->hint, entry->prev); | | 2662 | SAVE_HINT(srcmap, srcmap->hint, entry->prev); |
2663 | fudge = 0; | | 2663 | fudge = 0; |
2664 | } | | 2664 | } |
2665 | } else { | | 2665 | } else { |
2666 | | | 2666 | |
2667 | /* "start" is not within an entry ... skip to next entry */ | | 2667 | /* "start" is not within an entry ... skip to next entry */ |
2668 | if (flags & UVM_EXTRACT_CONTIG) { | | 2668 | if (flags & UVM_EXTRACT_CONTIG) { |
2669 | error = EINVAL; | | 2669 | error = EINVAL; |
2670 | goto bad; /* definite hole here ... */ | | 2670 | goto bad; /* definite hole here ... */ |
2671 | } | | 2671 | } |
2672 | | | 2672 | |
2673 | entry = entry->next; | | 2673 | entry = entry->next; |
2674 | fudge = 0; | | 2674 | fudge = 0; |
2675 | } | | 2675 | } |
2676 | | | 2676 | |
2677 | /* save values from srcmap for step 6 */ | | 2677 | /* save values from srcmap for step 6 */ |
2678 | orig_entry = entry; | | 2678 | orig_entry = entry; |
2679 | orig_fudge = fudge; | | 2679 | orig_fudge = fudge; |
2680 | | | 2680 | |
2681 | /* | | 2681 | /* |
2682 | * step 3: now start looping through the map entries, extracting | | 2682 | * step 3: now start looping through the map entries, extracting |
2683 | * as we go. | | 2683 | * as we go. |
2684 | */ | | 2684 | */ |
2685 | | | 2685 | |
2686 | while (entry->start < end && entry != &srcmap->header) { | | 2686 | while (entry->start < end && entry != &srcmap->header) { |
2687 | | | 2687 | |
2688 | /* if we are not doing a quick reference, clip it */ | | 2688 | /* if we are not doing a quick reference, clip it */ |
2689 | if ((flags & UVM_EXTRACT_QREF) == 0) | | 2689 | if ((flags & UVM_EXTRACT_QREF) == 0) |
2690 | UVM_MAP_CLIP_END(srcmap, entry, end); | | 2690 | UVM_MAP_CLIP_END(srcmap, entry, end); |
2691 | | | 2691 | |
2692 | /* clear needs_copy (allow chunking) */ | | 2692 | /* clear needs_copy (allow chunking) */ |
2693 | if (UVM_ET_ISNEEDSCOPY(entry)) { | | 2693 | if (UVM_ET_ISNEEDSCOPY(entry)) { |
2694 | amap_copy(srcmap, entry, | | 2694 | amap_copy(srcmap, entry, |
2695 | AMAP_COPY_NOWAIT|AMAP_COPY_NOMERGE, start, end); | | 2695 | AMAP_COPY_NOWAIT|AMAP_COPY_NOMERGE, start, end); |
2696 | if (UVM_ET_ISNEEDSCOPY(entry)) { /* failed? */ | | 2696 | if (UVM_ET_ISNEEDSCOPY(entry)) { /* failed? */ |
2697 | error = ENOMEM; | | 2697 | error = ENOMEM; |
2698 | goto bad; | | 2698 | goto bad; |
2699 | } | | 2699 | } |
2700 | | | 2700 | |
2701 | /* amap_copy could clip (during chunk)! update fudge */ | | 2701 | /* amap_copy could clip (during chunk)! update fudge */ |
2702 | if (fudge) { | | 2702 | if (fudge) { |
2703 | fudge = start - entry->start; | | 2703 | fudge = start - entry->start; |
2704 | orig_fudge = fudge; | | 2704 | orig_fudge = fudge; |
2705 | } | | 2705 | } |
2706 | } | | 2706 | } |
2707 | | | 2707 | |
2708 | /* calculate the offset of this from "start" */ | | 2708 | /* calculate the offset of this from "start" */ |
2709 | oldoffset = (entry->start + fudge) - start; | | 2709 | oldoffset = (entry->start + fudge) - start; |
2710 | | | 2710 | |
2711 | /* allocate a new map entry */ | | 2711 | /* allocate a new map entry */ |
2712 | newentry = uvm_mapent_alloc(dstmap, 0); | | 2712 | newentry = uvm_mapent_alloc(dstmap, 0); |
2713 | if (newentry == NULL) { | | 2713 | if (newentry == NULL) { |
2714 | error = ENOMEM; | | 2714 | error = ENOMEM; |
2715 | goto bad; | | 2715 | goto bad; |
2716 | } | | 2716 | } |
2717 | | | 2717 | |
2718 | /* set up new map entry */ | | 2718 | /* set up new map entry */ |
2719 | newentry->next = NULL; | | 2719 | newentry->next = NULL; |
2720 | newentry->prev = endchain; | | 2720 | newentry->prev = endchain; |
2721 | newentry->start = dstaddr + oldoffset; | | 2721 | newentry->start = dstaddr + oldoffset; |
2722 | newentry->end = | | 2722 | newentry->end = |
2723 | newentry->start + (entry->end - (entry->start + fudge)); | | 2723 | newentry->start + (entry->end - (entry->start + fudge)); |
2724 | if (newentry->end > newend || newentry->end < newentry->start) | | 2724 | if (newentry->end > newend || newentry->end < newentry->start) |
2725 | newentry->end = newend; | | 2725 | newentry->end = newend; |
2726 | newentry->object.uvm_obj = entry->object.uvm_obj; | | 2726 | newentry->object.uvm_obj = entry->object.uvm_obj; |
2727 | if (newentry->object.uvm_obj) { | | 2727 | if (newentry->object.uvm_obj) { |
2728 | if (newentry->object.uvm_obj->pgops->pgo_reference) | | 2728 | if (newentry->object.uvm_obj->pgops->pgo_reference) |
2729 | newentry->object.uvm_obj->pgops-> | | 2729 | newentry->object.uvm_obj->pgops-> |
2730 | pgo_reference(newentry->object.uvm_obj); | | 2730 | pgo_reference(newentry->object.uvm_obj); |
2731 | newentry->offset = entry->offset + fudge; | | 2731 | newentry->offset = entry->offset + fudge; |
2732 | } else { | | 2732 | } else { |
2733 | newentry->offset = 0; | | 2733 | newentry->offset = 0; |
2734 | } | | 2734 | } |
2735 | newentry->etype = entry->etype; | | 2735 | newentry->etype = entry->etype; |
2736 | if (flags & UVM_EXTRACT_PROT_ALL) { | | 2736 | if (flags & UVM_EXTRACT_PROT_ALL) { |
2737 | newentry->protection = newentry->max_protection = | | 2737 | newentry->protection = newentry->max_protection = |
2738 | UVM_PROT_ALL; | | 2738 | UVM_PROT_ALL; |
2739 | } else { | | 2739 | } else { |
2740 | newentry->protection = (flags & UVM_EXTRACT_FIXPROT) ? | | 2740 | newentry->protection = (flags & UVM_EXTRACT_FIXPROT) ? |
2741 | entry->max_protection : entry->protection; | | 2741 | entry->max_protection : entry->protection; |
2742 | newentry->max_protection = entry->max_protection; | | 2742 | newentry->max_protection = entry->max_protection; |
2743 | } | | 2743 | } |
2744 | newentry->inheritance = entry->inheritance; | | 2744 | newentry->inheritance = entry->inheritance; |
2745 | newentry->wired_count = 0; | | 2745 | newentry->wired_count = 0; |
2746 | newentry->aref.ar_amap = entry->aref.ar_amap; | | 2746 | newentry->aref.ar_amap = entry->aref.ar_amap; |
2747 | if (newentry->aref.ar_amap) { | | 2747 | if (newentry->aref.ar_amap) { |
2748 | newentry->aref.ar_pageoff = | | 2748 | newentry->aref.ar_pageoff = |
2749 | entry->aref.ar_pageoff + (fudge >> PAGE_SHIFT); | | 2749 | entry->aref.ar_pageoff + (fudge >> PAGE_SHIFT); |
2750 | uvm_map_reference_amap(newentry, AMAP_SHARED | | | 2750 | uvm_map_reference_amap(newentry, AMAP_SHARED | |
2751 | ((flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0)); | | 2751 | ((flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0)); |
2752 | } else { | | 2752 | } else { |
2753 | newentry->aref.ar_pageoff = 0; | | 2753 | newentry->aref.ar_pageoff = 0; |
2754 | } | | 2754 | } |
2755 | newentry->advice = entry->advice; | | 2755 | newentry->advice = entry->advice; |
2756 | if ((flags & UVM_EXTRACT_QREF) != 0) { | | 2756 | if ((flags & UVM_EXTRACT_QREF) != 0) { |
2757 | newentry->flags |= UVM_MAP_NOMERGE; | | 2757 | newentry->flags |= UVM_MAP_NOMERGE; |
2758 | } | | 2758 | } |
2759 | | | 2759 | |
2760 | /* now link it on the chain */ | | 2760 | /* now link it on the chain */ |
2761 | nchain++; | | 2761 | nchain++; |
2762 | nsize += newentry->end - newentry->start; | | 2762 | nsize += newentry->end - newentry->start; |
2763 | if (endchain == NULL) { | | 2763 | if (endchain == NULL) { |
2764 | chain = endchain = newentry; | | 2764 | chain = endchain = newentry; |
2765 | } else { | | 2765 | } else { |
2766 | endchain->next = newentry; | | 2766 | endchain->next = newentry; |
2767 | endchain = newentry; | | 2767 | endchain = newentry; |
2768 | } | | 2768 | } |
2769 | | | 2769 | |
2770 | /* end of 'while' loop! */ | | 2770 | /* end of 'while' loop! */ |
2771 | if ((flags & UVM_EXTRACT_CONTIG) && entry->end < end && | | 2771 | if ((flags & UVM_EXTRACT_CONTIG) && entry->end < end && |
2772 | (entry->next == &srcmap->header || | | 2772 | (entry->next == &srcmap->header || |
2773 | entry->next->start != entry->end)) { | | 2773 | entry->next->start != entry->end)) { |
2774 | error = EINVAL; | | 2774 | error = EINVAL; |
2775 | goto bad; | | 2775 | goto bad; |
2776 | } | | 2776 | } |
2777 | entry = entry->next; | | 2777 | entry = entry->next; |
2778 | fudge = 0; | | 2778 | fudge = 0; |
2779 | } | | 2779 | } |
2780 | | | 2780 | |
2781 | /* | | 2781 | /* |
2782 | * step 4: close off chain (in format expected by uvm_map_replace) | | 2782 | * step 4: close off chain (in format expected by uvm_map_replace) |
2783 | */ | | 2783 | */ |
2784 | | | 2784 | |
2785 | if (chain) | | 2785 | if (chain) |
2786 | chain->prev = endchain; | | 2786 | chain->prev = endchain; |
2787 | | | 2787 | |
2788 | /* | | 2788 | /* |
2789 | * step 5: attempt to lock the dest map so we can pmap_copy. | | 2789 | * step 5: attempt to lock the dest map so we can pmap_copy. |
2790 | * note usage of copy_ok: | | 2790 | * note usage of copy_ok: |
2791 | * 1 => dstmap locked, pmap_copy ok, and we "replace" here (step 5) | | 2791 | * 1 => dstmap locked, pmap_copy ok, and we "replace" here (step 5) |
2792 | * 0 => dstmap unlocked, NO pmap_copy, and we will "replace" in step 7 | | 2792 | * 0 => dstmap unlocked, NO pmap_copy, and we will "replace" in step 7 |
2793 | */ | | 2793 | */ |
2794 | | | 2794 | |
2795 | if (srcmap == dstmap || vm_map_lock_try(dstmap) == true) { | | 2795 | if (srcmap == dstmap || vm_map_lock_try(dstmap) == true) { |
2796 | copy_ok = 1; | | 2796 | copy_ok = 1; |
2797 | if (!uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain, | | 2797 | if (!uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain, |
2798 | nchain, nsize, &resentry)) { | | 2798 | nchain, nsize, &resentry)) { |
2799 | if (srcmap != dstmap) | | 2799 | if (srcmap != dstmap) |
2800 | vm_map_unlock(dstmap); | | 2800 | vm_map_unlock(dstmap); |
2801 | error = EIO; | | 2801 | error = EIO; |
2802 | goto bad; | | 2802 | goto bad; |
2803 | } | | 2803 | } |
2804 | } else { | | 2804 | } else { |
2805 | copy_ok = 0; | | 2805 | copy_ok = 0; |
2806 | /* replace defered until step 7 */ | | 2806 | /* replace defered until step 7 */ |
2807 | } | | 2807 | } |
2808 | | | 2808 | |
2809 | /* | | 2809 | /* |
2810 | * step 6: traverse the srcmap a second time to do the following: | | 2810 | * step 6: traverse the srcmap a second time to do the following: |
2811 | * - if we got a lock on the dstmap do pmap_copy | | 2811 | * - if we got a lock on the dstmap do pmap_copy |
2812 | * - if UVM_EXTRACT_REMOVE remove the entries | | 2812 | * - if UVM_EXTRACT_REMOVE remove the entries |
2813 | * we make use of orig_entry and orig_fudge (saved in step 2) | | 2813 | * we make use of orig_entry and orig_fudge (saved in step 2) |
2814 | */ | | 2814 | */ |
2815 | | | 2815 | |
2816 | if (copy_ok || (flags & UVM_EXTRACT_REMOVE)) { | | 2816 | if (copy_ok || (flags & UVM_EXTRACT_REMOVE)) { |
2817 | | | 2817 | |
2818 | /* purge possible stale hints from srcmap */ | | 2818 | /* purge possible stale hints from srcmap */ |
2819 | if (flags & UVM_EXTRACT_REMOVE) { | | 2819 | if (flags & UVM_EXTRACT_REMOVE) { |
2820 | SAVE_HINT(srcmap, srcmap->hint, orig_entry->prev); | | 2820 | SAVE_HINT(srcmap, srcmap->hint, orig_entry->prev); |
2821 | if (srcmap->first_free != &srcmap->header && | | 2821 | if (srcmap->first_free != &srcmap->header && |
2822 | srcmap->first_free->start >= start) | | 2822 | srcmap->first_free->start >= start) |
2823 | srcmap->first_free = orig_entry->prev; | | 2823 | srcmap->first_free = orig_entry->prev; |
2824 | } | | 2824 | } |
2825 | | | 2825 | |
2826 | entry = orig_entry; | | 2826 | entry = orig_entry; |
2827 | fudge = orig_fudge; | | 2827 | fudge = orig_fudge; |
2828 | deadentry = NULL; /* for UVM_EXTRACT_REMOVE */ | | 2828 | deadentry = NULL; /* for UVM_EXTRACT_REMOVE */ |
2829 | | | 2829 | |
2830 | while (entry->start < end && entry != &srcmap->header) { | | 2830 | while (entry->start < end && entry != &srcmap->header) { |
2831 | if (copy_ok) { | | 2831 | if (copy_ok) { |
2832 | oldoffset = (entry->start + fudge) - start; | | 2832 | oldoffset = (entry->start + fudge) - start; |
2833 | elen = MIN(end, entry->end) - | | 2833 | elen = MIN(end, entry->end) - |
2834 | (entry->start + fudge); | | 2834 | (entry->start + fudge); |
2835 | pmap_copy(dstmap->pmap, srcmap->pmap, | | 2835 | pmap_copy(dstmap->pmap, srcmap->pmap, |
2836 | dstaddr + oldoffset, elen, | | 2836 | dstaddr + oldoffset, elen, |
2837 | entry->start + fudge); | | 2837 | entry->start + fudge); |
2838 | } | | 2838 | } |
2839 | | | 2839 | |
2840 | /* we advance "entry" in the following if statement */ | | 2840 | /* we advance "entry" in the following if statement */ |
2841 | if (flags & UVM_EXTRACT_REMOVE) { | | 2841 | if (flags & UVM_EXTRACT_REMOVE) { |
2842 | #ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */ | | 2842 | #ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */ |
2843 | uvm_map_lock_entry(entry, RW_WRITER); | | 2843 | uvm_map_lock_entry(entry, RW_WRITER); |
2844 | #else | | 2844 | #else |
2845 | uvm_map_lock_entry(entry, RW_READER); | | 2845 | uvm_map_lock_entry(entry, RW_READER); |
2846 | #endif | | 2846 | #endif |
2847 | pmap_remove(srcmap->pmap, entry->start, | | 2847 | pmap_remove(srcmap->pmap, entry->start, |
2848 | entry->end); | | 2848 | entry->end); |
2849 | uvm_map_unlock_entry(entry); | | 2849 | uvm_map_unlock_entry(entry); |
2850 | oldentry = entry; /* save entry */ | | 2850 | oldentry = entry; /* save entry */ |
2851 | entry = entry->next; /* advance */ | | 2851 | entry = entry->next; /* advance */ |
2852 | uvm_map_entry_unlink(srcmap, oldentry); | | 2852 | uvm_map_entry_unlink(srcmap, oldentry); |
2853 | /* add to dead list */ | | 2853 | /* add to dead list */ |
2854 | oldentry->next = deadentry; | | 2854 | oldentry->next = deadentry; |
2855 | deadentry = oldentry; | | 2855 | deadentry = oldentry; |
2856 | } else { | | 2856 | } else { |
2857 | entry = entry->next; /* advance */ | | 2857 | entry = entry->next; /* advance */ |
2858 | } | | 2858 | } |
2859 | | | 2859 | |
2860 | /* end of 'while' loop */ | | 2860 | /* end of 'while' loop */ |
2861 | fudge = 0; | | 2861 | fudge = 0; |
2862 | } | | 2862 | } |
2863 | pmap_update(srcmap->pmap); | | 2863 | pmap_update(srcmap->pmap); |
2864 | | | 2864 | |
2865 | /* | | 2865 | /* |
2866 | * unlock dstmap. we will dispose of deadentry in | | 2866 | * unlock dstmap. we will dispose of deadentry in |
2867 | * step 7 if needed | | 2867 | * step 7 if needed |
2868 | */ | | 2868 | */ |
2869 | | | 2869 | |
2870 | if (copy_ok && srcmap != dstmap) | | 2870 | if (copy_ok && srcmap != dstmap) |
2871 | vm_map_unlock(dstmap); | | 2871 | vm_map_unlock(dstmap); |
2872 | | | 2872 | |
2873 | } else { | | 2873 | } else { |
2874 | deadentry = NULL; | | 2874 | deadentry = NULL; |
2875 | } | | 2875 | } |
2876 | | | 2876 | |
2877 | /* | | 2877 | /* |
2878 | * step 7: we are done with the source map, unlock. if copy_ok | | 2878 | * step 7: we are done with the source map, unlock. if copy_ok |
2879 | * is 0 then we have not replaced the dummy mapping in dstmap yet | | 2879 | * is 0 then we have not replaced the dummy mapping in dstmap yet |
2880 | * and we need to do so now. | | 2880 | * and we need to do so now. |
2881 | */ | | 2881 | */ |
2882 | | | 2882 | |
2883 | vm_map_unlock(srcmap); | | 2883 | vm_map_unlock(srcmap); |
2884 | if ((flags & UVM_EXTRACT_REMOVE) && deadentry) | | 2884 | if ((flags & UVM_EXTRACT_REMOVE) && deadentry) |
2885 | uvm_unmap_detach(deadentry, 0); /* dispose of old entries */ | | 2885 | uvm_unmap_detach(deadentry, 0); /* dispose of old entries */ |
2886 | | | 2886 | |
2887 | /* now do the replacement if we didn't do it in step 5 */ | | 2887 | /* now do the replacement if we didn't do it in step 5 */ |
2888 | if (copy_ok == 0) { | | 2888 | if (copy_ok == 0) { |
2889 | vm_map_lock(dstmap); | | 2889 | vm_map_lock(dstmap); |
2890 | error = uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain, | | 2890 | error = uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain, |
2891 | nchain, nsize, &resentry); | | 2891 | nchain, nsize, &resentry); |
2892 | vm_map_unlock(dstmap); | | 2892 | vm_map_unlock(dstmap); |
2893 | | | 2893 | |
2894 | if (error == false) { | | 2894 | if (error == false) { |
2895 | error = EIO; | | 2895 | error = EIO; |
2896 | goto bad2; | | 2896 | goto bad2; |
2897 | } | | 2897 | } |
2898 | } | | 2898 | } |
2899 | | | 2899 | |
2900 | if (resentry != NULL) | | 2900 | if (resentry != NULL) |
2901 | uvm_mapent_free(resentry); | | 2901 | uvm_mapent_free(resentry); |
2902 | | | 2902 | |
2903 | return (0); | | 2903 | return (0); |
2904 | | | 2904 | |
2905 | /* | | 2905 | /* |
2906 | * bad: failure recovery | | 2906 | * bad: failure recovery |
2907 | */ | | 2907 | */ |
2908 | bad: | | 2908 | bad: |
2909 | vm_map_unlock(srcmap); | | 2909 | vm_map_unlock(srcmap); |
2910 | bad2: /* src already unlocked */ | | 2910 | bad2: /* src already unlocked */ |
2911 | if (chain) | | 2911 | if (chain) |
2912 | uvm_unmap_detach(chain, | | 2912 | uvm_unmap_detach(chain, |
2913 | (flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0); | | 2913 | (flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0); |
2914 | | | 2914 | |
2915 | if (resentry != NULL) | | 2915 | if (resentry != NULL) |
2916 | uvm_mapent_free(resentry); | | 2916 | uvm_mapent_free(resentry); |
2917 | | | 2917 | |
2918 | if ((flags & UVM_EXTRACT_RESERVED) == 0) { | | 2918 | if ((flags & UVM_EXTRACT_RESERVED) == 0) { |
2919 | uvm_unmap(dstmap, dstaddr, dstaddr+len); /* ??? */ | | 2919 | uvm_unmap(dstmap, dstaddr, dstaddr+len); /* ??? */ |
2920 | } | | 2920 | } |
2921 | return (error); | | 2921 | return (error); |
2922 | } | | 2922 | } |
2923 | | | 2923 | |
2924 | /* end of extraction functions */ | | 2924 | /* end of extraction functions */ |
2925 | | | 2925 | |
2926 | /* | | 2926 | /* |
2927 | * uvm_map_submap: punch down part of a map into a submap | | 2927 | * uvm_map_submap: punch down part of a map into a submap |
2928 | * | | 2928 | * |
2929 | * => only the kernel_map is allowed to be submapped | | 2929 | * => only the kernel_map is allowed to be submapped |
2930 | * => the purpose of submapping is to break up the locking granularity | | 2930 | * => the purpose of submapping is to break up the locking granularity |
2931 | * of a larger map | | 2931 | * of a larger map |
2932 | * => the range specified must have been mapped previously with a uvm_map() | | 2932 | * => the range specified must have been mapped previously with a uvm_map() |
2933 | * call [with uobj==NULL] to create a blank map entry in the main map. | | 2933 | * call [with uobj==NULL] to create a blank map entry in the main map. |
2934 | * [And it had better still be blank!] | | 2934 | * [And it had better still be blank!] |
2935 | * => maps which contain submaps should never be copied or forked. | | 2935 | * => maps which contain submaps should never be copied or forked. |
2936 | * => to remove a submap, use uvm_unmap() on the main map | | 2936 | * => to remove a submap, use uvm_unmap() on the main map |
2937 | * and then uvm_map_deallocate() the submap. | | 2937 | * and then uvm_map_deallocate() the submap. |
2938 | * => main map must be unlocked. | | 2938 | * => main map must be unlocked. |
2939 | * => submap must have been init'd and have a zero reference count. | | 2939 | * => submap must have been init'd and have a zero reference count. |
2940 | * [need not be locked as we don't actually reference it] | | 2940 | * [need not be locked as we don't actually reference it] |
2941 | */ | | 2941 | */ |
2942 | | | 2942 | |
2943 | int | | 2943 | int |
2944 | uvm_map_submap(struct vm_map *map, vaddr_t start, vaddr_t end, | | 2944 | uvm_map_submap(struct vm_map *map, vaddr_t start, vaddr_t end, |
2945 | struct vm_map *submap) | | 2945 | struct vm_map *submap) |
2946 | { | | 2946 | { |
2947 | struct vm_map_entry *entry; | | 2947 | struct vm_map_entry *entry; |
2948 | int error; | | 2948 | int error; |
2949 | | | 2949 | |
2950 | vm_map_lock(map); | | 2950 | vm_map_lock(map); |
2951 | VM_MAP_RANGE_CHECK(map, start, end); | | 2951 | VM_MAP_RANGE_CHECK(map, start, end); |
2952 | | | 2952 | |
2953 | if (uvm_map_lookup_entry(map, start, &entry)) { | | 2953 | if (uvm_map_lookup_entry(map, start, &entry)) { |
2954 | UVM_MAP_CLIP_START(map, entry, start); | | 2954 | UVM_MAP_CLIP_START(map, entry, start); |
2955 | UVM_MAP_CLIP_END(map, entry, end); /* to be safe */ | | 2955 | UVM_MAP_CLIP_END(map, entry, end); /* to be safe */ |
2956 | } else { | | 2956 | } else { |
2957 | entry = NULL; | | 2957 | entry = NULL; |
2958 | } | | 2958 | } |
2959 | | | 2959 | |
2960 | if (entry != NULL && | | 2960 | if (entry != NULL && |
2961 | entry->start == start && entry->end == end && | | 2961 | entry->start == start && entry->end == end && |
2962 | entry->object.uvm_obj == NULL && entry->aref.ar_amap == NULL && | | 2962 | entry->object.uvm_obj == NULL && entry->aref.ar_amap == NULL && |
2963 | !UVM_ET_ISCOPYONWRITE(entry) && !UVM_ET_ISNEEDSCOPY(entry)) { | | 2963 | !UVM_ET_ISCOPYONWRITE(entry) && !UVM_ET_ISNEEDSCOPY(entry)) { |
2964 | entry->etype |= UVM_ET_SUBMAP; | | 2964 | entry->etype |= UVM_ET_SUBMAP; |
2965 | entry->object.sub_map = submap; | | 2965 | entry->object.sub_map = submap; |
2966 | entry->offset = 0; | | 2966 | entry->offset = 0; |
2967 | uvm_map_reference(submap); | | 2967 | uvm_map_reference(submap); |
2968 | error = 0; | | 2968 | error = 0; |
2969 | } else { | | 2969 | } else { |
2970 | error = EINVAL; | | 2970 | error = EINVAL; |
2971 | } | | 2971 | } |
2972 | vm_map_unlock(map); | | 2972 | vm_map_unlock(map); |
2973 | | | 2973 | |
2974 | return error; | | 2974 | return error; |
2975 | } | | 2975 | } |
2976 | | | 2976 | |
2977 | /* | | 2977 | /* |
2978 | * uvm_map_protect_user: change map protection on behalf of the user. | | 2978 | * uvm_map_protect_user: change map protection on behalf of the user. |
2979 | * Enforces PAX settings as necessary. | | 2979 | * Enforces PAX settings as necessary. |
2980 | */ | | 2980 | */ |
2981 | int | | 2981 | int |
2982 | uvm_map_protect_user(struct lwp *l, vaddr_t start, vaddr_t end, | | 2982 | uvm_map_protect_user(struct lwp *l, vaddr_t start, vaddr_t end, |
2983 | vm_prot_t new_prot) | | 2983 | vm_prot_t new_prot) |
2984 | { | | 2984 | { |
2985 | int error; | | 2985 | int error; |
2986 | | | 2986 | |
2987 | if ((error = PAX_MPROTECT_VALIDATE(l, new_prot))) | | 2987 | if ((error = PAX_MPROTECT_VALIDATE(l, new_prot))) |
2988 | return error; | | 2988 | return error; |
2989 | | | 2989 | |
2990 | return uvm_map_protect(&l->l_proc->p_vmspace->vm_map, start, end, | | 2990 | return uvm_map_protect(&l->l_proc->p_vmspace->vm_map, start, end, |
2991 | new_prot, false); | | 2991 | new_prot, false); |
2992 | } | | 2992 | } |
2993 | | | 2993 | |
2994 | | | 2994 | |
2995 | /* | | 2995 | /* |
2996 | * uvm_map_protect: change map protection | | 2996 | * uvm_map_protect: change map protection |
2997 | * | | 2997 | * |
2998 | * => set_max means set max_protection. | | 2998 | * => set_max means set max_protection. |
2999 | * => map must be unlocked. | | 2999 | * => map must be unlocked. |
3000 | */ | | 3000 | */ |
3001 | | | 3001 | |
3002 | #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ | | 3002 | #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ |
3003 | ~VM_PROT_WRITE : VM_PROT_ALL) | | 3003 | ~VM_PROT_WRITE : VM_PROT_ALL) |
3004 | | | 3004 | |
3005 | int | | 3005 | int |
3006 | uvm_map_protect(struct vm_map *map, vaddr_t start, vaddr_t end, | | 3006 | uvm_map_protect(struct vm_map *map, vaddr_t start, vaddr_t end, |
3007 | vm_prot_t new_prot, bool set_max) | | 3007 | vm_prot_t new_prot, bool set_max) |
3008 | { | | 3008 | { |
3009 | struct vm_map_entry *current, *entry; | | 3009 | struct vm_map_entry *current, *entry; |
3010 | int error = 0; | | 3010 | int error = 0; |
3011 | UVMHIST_FUNC("uvm_map_protect"); UVMHIST_CALLED(maphist); | | 3011 | UVMHIST_FUNC("uvm_map_protect"); UVMHIST_CALLED(maphist); |
3012 | UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_prot=%#jx)", | | 3012 | UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_prot=%#jx)", |
3013 | (uintptr_t)map, start, end, new_prot); | | 3013 | (uintptr_t)map, start, end, new_prot); |
3014 | | | 3014 | |
3015 | vm_map_lock(map); | | 3015 | vm_map_lock(map); |
3016 | VM_MAP_RANGE_CHECK(map, start, end); | | 3016 | VM_MAP_RANGE_CHECK(map, start, end); |
3017 | if (uvm_map_lookup_entry(map, start, &entry)) { | | 3017 | if (uvm_map_lookup_entry(map, start, &entry)) { |
3018 | UVM_MAP_CLIP_START(map, entry, start); | | 3018 | UVM_MAP_CLIP_START(map, entry, start); |
3019 | } else { | | 3019 | } else { |
3020 | entry = entry->next; | | 3020 | entry = entry->next; |
3021 | } | | 3021 | } |
3022 | | | 3022 | |
3023 | /* | | 3023 | /* |
3024 | * make a first pass to check for protection violations. | | 3024 | * make a first pass to check for protection violations. |
3025 | */ | | 3025 | */ |
3026 | | | 3026 | |
3027 | current = entry; | | 3027 | current = entry; |
3028 | while ((current != &map->header) && (current->start < end)) { | | 3028 | while ((current != &map->header) && (current->start < end)) { |
3029 | if (UVM_ET_ISSUBMAP(current)) { | | 3029 | if (UVM_ET_ISSUBMAP(current)) { |
3030 | error = EINVAL; | | 3030 | error = EINVAL; |
3031 | goto out; | | 3031 | goto out; |
3032 | } | | 3032 | } |
3033 | if ((new_prot & current->max_protection) != new_prot) { | | 3033 | if ((new_prot & current->max_protection) != new_prot) { |
3034 | error = EACCES; | | 3034 | error = EACCES; |
3035 | goto out; | | 3035 | goto out; |
3036 | } | | 3036 | } |
3037 | /* | | 3037 | /* |
3038 | * Don't allow VM_PROT_EXECUTE to be set on entries that | | 3038 | * Don't allow VM_PROT_EXECUTE to be set on entries that |
3039 | * point to vnodes that are associated with a NOEXEC file | | 3039 | * point to vnodes that are associated with a NOEXEC file |
3040 | * system. | | 3040 | * system. |
3041 | */ | | 3041 | */ |
3042 | if (UVM_ET_ISOBJ(current) && | | 3042 | if (UVM_ET_ISOBJ(current) && |
3043 | UVM_OBJ_IS_VNODE(current->object.uvm_obj)) { | | 3043 | UVM_OBJ_IS_VNODE(current->object.uvm_obj)) { |
3044 | struct vnode *vp = | | 3044 | struct vnode *vp = |
3045 | (struct vnode *) current->object.uvm_obj; | | 3045 | (struct vnode *) current->object.uvm_obj; |
3046 | | | 3046 | |
3047 | if ((new_prot & VM_PROT_EXECUTE) != 0 && | | 3047 | if ((new_prot & VM_PROT_EXECUTE) != 0 && |
3048 | (vp->v_mount->mnt_flag & MNT_NOEXEC) != 0) { | | 3048 | (vp->v_mount->mnt_flag & MNT_NOEXEC) != 0) { |
3049 | error = EACCES; | | 3049 | error = EACCES; |
3050 | goto out; | | 3050 | goto out; |
3051 | } | | 3051 | } |
3052 | } | | 3052 | } |
3053 | | | 3053 | |
3054 | current = current->next; | | 3054 | current = current->next; |
3055 | } | | 3055 | } |
3056 | | | 3056 | |
3057 | /* go back and fix up protections (no need to clip this time). */ | | 3057 | /* go back and fix up protections (no need to clip this time). */ |
3058 | | | 3058 | |
3059 | current = entry; | | 3059 | current = entry; |
3060 | while ((current != &map->header) && (current->start < end)) { | | 3060 | while ((current != &map->header) && (current->start < end)) { |
3061 | vm_prot_t old_prot; | | 3061 | vm_prot_t old_prot; |
3062 | | | 3062 | |
3063 | UVM_MAP_CLIP_END(map, current, end); | | 3063 | UVM_MAP_CLIP_END(map, current, end); |
3064 | old_prot = current->protection; | | 3064 | old_prot = current->protection; |
3065 | if (set_max) | | 3065 | if (set_max) |
3066 | current->protection = | | 3066 | current->protection = |
3067 | (current->max_protection = new_prot) & old_prot; | | 3067 | (current->max_protection = new_prot) & old_prot; |
3068 | else | | 3068 | else |
3069 | current->protection = new_prot; | | 3069 | current->protection = new_prot; |
3070 | | | 3070 | |
3071 | /* | | 3071 | /* |
3072 | * update physical map if necessary. worry about copy-on-write | | 3072 | * update physical map if necessary. worry about copy-on-write |
3073 | * here -- CHECK THIS XXX | | 3073 | * here -- CHECK THIS XXX |
3074 | */ | | 3074 | */ |
3075 | | | 3075 | |
3076 | if (current->protection != old_prot) { | | 3076 | if (current->protection != old_prot) { |
3077 | /* update pmap! */ | | 3077 | /* update pmap! */ |
3078 | #ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */ | | 3078 | #ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */ |
3079 | uvm_map_lock_entry(current, RW_WRITER); | | 3079 | uvm_map_lock_entry(current, RW_WRITER); |
3080 | #else | | 3080 | #else |
3081 | uvm_map_lock_entry(current, RW_READER); | | 3081 | uvm_map_lock_entry(current, RW_READER); |
3082 | #endif | | 3082 | #endif |
3083 | pmap_protect(map->pmap, current->start, current->end, | | 3083 | pmap_protect(map->pmap, current->start, current->end, |
3084 | current->protection & MASK(current)); | | 3084 | current->protection & MASK(current)); |
3085 | uvm_map_unlock_entry(current); | | 3085 | uvm_map_unlock_entry(current); |
3086 | | | 3086 | |
3087 | /* | | 3087 | /* |
3088 | * If this entry points at a vnode, and the | | 3088 | * If this entry points at a vnode, and the |
3089 | * protection includes VM_PROT_EXECUTE, mark | | 3089 | * protection includes VM_PROT_EXECUTE, mark |
3090 | * the vnode as VEXECMAP. | | 3090 | * the vnode as VEXECMAP. |
3091 | */ | | 3091 | */ |
3092 | if (UVM_ET_ISOBJ(current)) { | | 3092 | if (UVM_ET_ISOBJ(current)) { |
3093 | struct uvm_object *uobj = | | 3093 | struct uvm_object *uobj = |
3094 | current->object.uvm_obj; | | 3094 | current->object.uvm_obj; |
3095 | | | 3095 | |
3096 | if (UVM_OBJ_IS_VNODE(uobj) && | | 3096 | if (UVM_OBJ_IS_VNODE(uobj) && |
3097 | (current->protection & VM_PROT_EXECUTE)) { | | 3097 | (current->protection & VM_PROT_EXECUTE)) { |
3098 | vn_markexec((struct vnode *) uobj); | | 3098 | vn_markexec((struct vnode *) uobj); |
3099 | } | | 3099 | } |
3100 | } | | 3100 | } |
3101 | } | | 3101 | } |
3102 | | | 3102 | |
3103 | /* | | 3103 | /* |
3104 | * If the map is configured to lock any future mappings, | | 3104 | * If the map is configured to lock any future mappings, |
3105 | * wire this entry now if the old protection was VM_PROT_NONE | | 3105 | * wire this entry now if the old protection was VM_PROT_NONE |
3106 | * and the new protection is not VM_PROT_NONE. | | 3106 | * and the new protection is not VM_PROT_NONE. |
3107 | */ | | 3107 | */ |
3108 | | | 3108 | |
3109 | if ((map->flags & VM_MAP_WIREFUTURE) != 0 && | | 3109 | if ((map->flags & VM_MAP_WIREFUTURE) != 0 && |
3110 | VM_MAPENT_ISWIRED(current) == 0 && | | 3110 | VM_MAPENT_ISWIRED(current) == 0 && |
3111 | old_prot == VM_PROT_NONE && | | 3111 | old_prot == VM_PROT_NONE && |
3112 | new_prot != VM_PROT_NONE) { | | 3112 | new_prot != VM_PROT_NONE) { |
3113 | | | 3113 | |
3114 | /* | | 3114 | /* |
3115 | * We must call pmap_update() here because the | | 3115 | * We must call pmap_update() here because the |
3116 | * pmap_protect() call above might have removed some | | 3116 | * pmap_protect() call above might have removed some |
3117 | * pmap entries and uvm_map_pageable() might create | | 3117 | * pmap entries and uvm_map_pageable() might create |
3118 | * some new pmap entries that rely on the prior | | 3118 | * some new pmap entries that rely on the prior |
3119 | * removals being completely finished. | | 3119 | * removals being completely finished. |
3120 | */ | | 3120 | */ |
3121 | | | 3121 | |
3122 | pmap_update(map->pmap); | | 3122 | pmap_update(map->pmap); |
3123 | | | 3123 | |
3124 | if (uvm_map_pageable(map, current->start, | | 3124 | if (uvm_map_pageable(map, current->start, |
3125 | current->end, false, | | 3125 | current->end, false, |
3126 | UVM_LK_ENTER|UVM_LK_EXIT) != 0) { | | 3126 | UVM_LK_ENTER|UVM_LK_EXIT) != 0) { |
3127 | | | 3127 | |
3128 | /* | | 3128 | /* |
3129 | * If locking the entry fails, remember the | | 3129 | * If locking the entry fails, remember the |
3130 | * error if it's the first one. Note we | | 3130 | * error if it's the first one. Note we |
3131 | * still continue setting the protection in | | 3131 | * still continue setting the protection in |
3132 | * the map, but will return the error | | 3132 | * the map, but will return the error |
3133 | * condition regardless. | | 3133 | * condition regardless. |
3134 | * | | 3134 | * |
3135 | * XXX Ignore what the actual error is, | | 3135 | * XXX Ignore what the actual error is, |
3136 | * XXX just call it a resource shortage | | 3136 | * XXX just call it a resource shortage |
3137 | * XXX so that it doesn't get confused | | 3137 | * XXX so that it doesn't get confused |
3138 | * XXX what uvm_map_protect() itself would | | 3138 | * XXX what uvm_map_protect() itself would |
3139 | * XXX normally return. | | 3139 | * XXX normally return. |
3140 | */ | | 3140 | */ |
3141 | | | 3141 | |
3142 | error = ENOMEM; | | 3142 | error = ENOMEM; |
3143 | } | | 3143 | } |
3144 | } | | 3144 | } |
3145 | current = current->next; | | 3145 | current = current->next; |
3146 | } | | 3146 | } |
3147 | pmap_update(map->pmap); | | 3147 | pmap_update(map->pmap); |
3148 | | | 3148 | |
3149 | out: | | 3149 | out: |
3150 | vm_map_unlock(map); | | 3150 | vm_map_unlock(map); |
3151 | | | 3151 | |
3152 | UVMHIST_LOG(maphist, "<- done, error=%jd",error,0,0,0); | | 3152 | UVMHIST_LOG(maphist, "<- done, error=%jd",error,0,0,0); |
3153 | return error; | | 3153 | return error; |
3154 | } | | 3154 | } |
3155 | | | 3155 | |
3156 | #undef MASK | | 3156 | #undef MASK |
3157 | | | 3157 | |
3158 | /* | | 3158 | /* |
3159 | * uvm_map_inherit: set inheritance code for range of addrs in map. | | 3159 | * uvm_map_inherit: set inheritance code for range of addrs in map. |
3160 | * | | 3160 | * |
3161 | * => map must be unlocked | | 3161 | * => map must be unlocked |
3162 | * => note that the inherit code is used during a "fork". see fork | | 3162 | * => note that the inherit code is used during a "fork". see fork |
3163 | * code for details. | | 3163 | * code for details. |
3164 | */ | | 3164 | */ |
3165 | | | 3165 | |
3166 | int | | 3166 | int |
3167 | uvm_map_inherit(struct vm_map *map, vaddr_t start, vaddr_t end, | | 3167 | uvm_map_inherit(struct vm_map *map, vaddr_t start, vaddr_t end, |
3168 | vm_inherit_t new_inheritance) | | 3168 | vm_inherit_t new_inheritance) |
3169 | { | | 3169 | { |
3170 | struct vm_map_entry *entry, *temp_entry; | | 3170 | struct vm_map_entry *entry, *temp_entry; |
3171 | UVMHIST_FUNC("uvm_map_inherit"); UVMHIST_CALLED(maphist); | | 3171 | UVMHIST_FUNC("uvm_map_inherit"); UVMHIST_CALLED(maphist); |
3172 | UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_inh=%#jx)", | | 3172 | UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_inh=%#jx)", |
3173 | (uintptr_t)map, start, end, new_inheritance); | | 3173 | (uintptr_t)map, start, end, new_inheritance); |
3174 | | | 3174 | |
3175 | switch (new_inheritance) { | | 3175 | switch (new_inheritance) { |
3176 | case MAP_INHERIT_NONE: | | 3176 | case MAP_INHERIT_NONE: |
3177 | case MAP_INHERIT_COPY: | | 3177 | case MAP_INHERIT_COPY: |
3178 | case MAP_INHERIT_SHARE: | | 3178 | case MAP_INHERIT_SHARE: |
3179 | case MAP_INHERIT_ZERO: | | 3179 | case MAP_INHERIT_ZERO: |
3180 | break; | | 3180 | break; |
3181 | default: | | 3181 | default: |
3182 | UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0); | | 3182 | UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0); |
3183 | return EINVAL; | | 3183 | return EINVAL; |
3184 | } | | 3184 | } |
3185 | | | 3185 | |
3186 | vm_map_lock(map); | | 3186 | vm_map_lock(map); |
3187 | VM_MAP_RANGE_CHECK(map, start, end); | | 3187 | VM_MAP_RANGE_CHECK(map, start, end); |
3188 | if (uvm_map_lookup_entry(map, start, &temp_entry)) { | | 3188 | if (uvm_map_lookup_entry(map, start, &temp_entry)) { |
3189 | entry = temp_entry; | | 3189 | entry = temp_entry; |
3190 | UVM_MAP_CLIP_START(map, entry, start); | | 3190 | UVM_MAP_CLIP_START(map, entry, start); |
3191 | } else { | | 3191 | } else { |
3192 | entry = temp_entry->next; | | 3192 | entry = temp_entry->next; |
3193 | } | | 3193 | } |
3194 | while ((entry != &map->header) && (entry->start < end)) { | | 3194 | while ((entry != &map->header) && (entry->start < end)) { |
3195 | UVM_MAP_CLIP_END(map, entry, end); | | 3195 | UVM_MAP_CLIP_END(map, entry, end); |
3196 | entry->inheritance = new_inheritance; | | 3196 | entry->inheritance = new_inheritance; |
3197 | entry = entry->next; | | 3197 | entry = entry->next; |
3198 | } | | 3198 | } |
3199 | vm_map_unlock(map); | | 3199 | vm_map_unlock(map); |
3200 | UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0); | | 3200 | UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0); |
3201 | return 0; | | 3201 | return 0; |
3202 | } | | 3202 | } |
3203 | | | 3203 | |
3204 | /* | | 3204 | /* |
3205 | * uvm_map_advice: set advice code for range of addrs in map. | | 3205 | * uvm_map_advice: set advice code for range of addrs in map. |
3206 | * | | 3206 | * |
3207 | * => map must be unlocked | | 3207 | * => map must be unlocked |
3208 | */ | | 3208 | */ |
3209 | | | 3209 | |
3210 | int | | 3210 | int |
3211 | uvm_map_advice(struct vm_map *map, vaddr_t start, vaddr_t end, int new_advice) | | 3211 | uvm_map_advice(struct vm_map *map, vaddr_t start, vaddr_t end, int new_advice) |
3212 | { | | 3212 | { |
3213 | struct vm_map_entry *entry, *temp_entry; | | 3213 | struct vm_map_entry *entry, *temp_entry; |
3214 | UVMHIST_FUNC("uvm_map_advice"); UVMHIST_CALLED(maphist); | | 3214 | UVMHIST_FUNC("uvm_map_advice"); UVMHIST_CALLED(maphist); |
3215 | UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_adv=%#jx)", | | 3215 | UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_adv=%#jx)", |
3216 | (uintptr_t)map, start, end, new_advice); | | 3216 | (uintptr_t)map, start, end, new_advice); |
3217 | | | 3217 | |
3218 | vm_map_lock(map); | | 3218 | vm_map_lock(map); |
3219 | VM_MAP_RANGE_CHECK(map, start, end); | | 3219 | VM_MAP_RANGE_CHECK(map, start, end); |
3220 | if (uvm_map_lookup_entry(map, start, &temp_entry)) { | | 3220 | if (uvm_map_lookup_entry(map, start, &temp_entry)) { |
3221 | entry = temp_entry; | | 3221 | entry = temp_entry; |
3222 | UVM_MAP_CLIP_START(map, entry, start); | | 3222 | UVM_MAP_CLIP_START(map, entry, start); |
3223 | } else { | | 3223 | } else { |
3224 | entry = temp_entry->next; | | 3224 | entry = temp_entry->next; |
3225 | } | | 3225 | } |
3226 | | | 3226 | |
3227 | /* | | 3227 | /* |
3228 | * XXXJRT: disallow holes? | | 3228 | * XXXJRT: disallow holes? |
3229 | */ | | 3229 | */ |
3230 | | | 3230 | |
3231 | while ((entry != &map->header) && (entry->start < end)) { | | 3231 | while ((entry != &map->header) && (entry->start < end)) { |
3232 | UVM_MAP_CLIP_END(map, entry, end); | | 3232 | UVM_MAP_CLIP_END(map, entry, end); |
3233 | | | 3233 | |
3234 | switch (new_advice) { | | 3234 | switch (new_advice) { |
3235 | case MADV_NORMAL: | | 3235 | case MADV_NORMAL: |
3236 | case MADV_RANDOM: | | 3236 | case MADV_RANDOM: |
3237 | case MADV_SEQUENTIAL: | | 3237 | case MADV_SEQUENTIAL: |
3238 | /* nothing special here */ | | 3238 | /* nothing special here */ |
3239 | break; | | 3239 | break; |
3240 | | | 3240 | |
3241 | default: | | 3241 | default: |
3242 | vm_map_unlock(map); | | 3242 | vm_map_unlock(map); |
3243 | UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0); | | 3243 | UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0); |
3244 | return EINVAL; | | 3244 | return EINVAL; |
3245 | } | | 3245 | } |
3246 | entry->advice = new_advice; | | 3246 | entry->advice = new_advice; |
3247 | entry = entry->next; | | 3247 | entry = entry->next; |
3248 | } | | 3248 | } |
3249 | | | 3249 | |
3250 | vm_map_unlock(map); | | 3250 | vm_map_unlock(map); |
3251 | UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0); | | 3251 | UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0); |
3252 | return 0; | | 3252 | return 0; |
3253 | } | | 3253 | } |
3254 | | | 3254 | |
3255 | /* | | 3255 | /* |
3256 | * uvm_map_willneed: apply MADV_WILLNEED | | 3256 | * uvm_map_willneed: apply MADV_WILLNEED |
3257 | */ | | 3257 | */ |
3258 | | | 3258 | |
3259 | int | | 3259 | int |
3260 | uvm_map_willneed(struct vm_map *map, vaddr_t start, vaddr_t end) | | 3260 | uvm_map_willneed(struct vm_map *map, vaddr_t start, vaddr_t end) |
3261 | { | | 3261 | { |
3262 | struct vm_map_entry *entry; | | 3262 | struct vm_map_entry *entry; |
3263 | UVMHIST_FUNC("uvm_map_willneed"); UVMHIST_CALLED(maphist); | | 3263 | UVMHIST_FUNC("uvm_map_willneed"); UVMHIST_CALLED(maphist); |
3264 | UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx)", | | 3264 | UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx)", |
3265 | (uintptr_t)map, start, end, 0); | | 3265 | (uintptr_t)map, start, end, 0); |
3266 | | | 3266 | |
3267 | vm_map_lock_read(map); | | 3267 | vm_map_lock_read(map); |
3268 | VM_MAP_RANGE_CHECK(map, start, end); | | 3268 | VM_MAP_RANGE_CHECK(map, start, end); |
3269 | if (!uvm_map_lookup_entry(map, start, &entry)) { | | 3269 | if (!uvm_map_lookup_entry(map, start, &entry)) { |
3270 | entry = entry->next; | | 3270 | entry = entry->next; |
3271 | } | | 3271 | } |
3272 | while (entry->start < end) { | | 3272 | while (entry->start < end) { |
3273 | struct vm_amap * const amap = entry->aref.ar_amap; | | 3273 | struct vm_amap * const amap = entry->aref.ar_amap; |
3274 | struct uvm_object * const uobj = entry->object.uvm_obj; | | 3274 | struct uvm_object * const uobj = entry->object.uvm_obj; |
3275 | | | 3275 | |
3276 | KASSERT(entry != &map->header); | | 3276 | KASSERT(entry != &map->header); |
3277 | KASSERT(start < entry->end); | | 3277 | KASSERT(start < entry->end); |
3278 | /* | | 3278 | /* |
3279 | * For now, we handle only the easy but commonly-requested case. | | 3279 | * For now, we handle only the easy but commonly-requested case. |
3280 | * ie. start prefetching of backing uobj pages. | | 3280 | * ie. start prefetching of backing uobj pages. |
3281 | * | | 3281 | * |
3282 | * XXX It might be useful to pmap_enter() the already-in-core | | 3282 | * XXX It might be useful to pmap_enter() the already-in-core |
3283 | * pages by inventing a "weak" mode for uvm_fault() which would | | 3283 | * pages by inventing a "weak" mode for uvm_fault() which would |
3284 | * only do the PGO_LOCKED pgo_get(). | | 3284 | * only do the PGO_LOCKED pgo_get(). |
3285 | */ | | 3285 | */ |
3286 | if (UVM_ET_ISOBJ(entry) && amap == NULL && uobj != NULL) { | | 3286 | if (UVM_ET_ISOBJ(entry) && amap == NULL && uobj != NULL) { |
3287 | off_t offset; | | 3287 | off_t offset; |
3288 | off_t size; | | 3288 | off_t size; |
3289 | | | 3289 | |
3290 | offset = entry->offset; | | 3290 | offset = entry->offset; |
3291 | if (start < entry->start) { | | 3291 | if (start < entry->start) { |
3292 | offset += entry->start - start; | | 3292 | offset += entry->start - start; |
3293 | } | | 3293 | } |
3294 | size = entry->offset + (entry->end - entry->start); | | 3294 | size = entry->offset + (entry->end - entry->start); |
3295 | if (entry->end < end) { | | 3295 | if (entry->end < end) { |
3296 | size -= end - entry->end; | | 3296 | size -= end - entry->end; |
3297 | } | | 3297 | } |
3298 | uvm_readahead(uobj, offset, size); | | 3298 | uvm_readahead(uobj, offset, size); |
3299 | } | | 3299 | } |
3300 | entry = entry->next; | | 3300 | entry = entry->next; |
3301 | } | | 3301 | } |
3302 | vm_map_unlock_read(map); | | 3302 | vm_map_unlock_read(map); |
3303 | UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0); | | 3303 | UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0); |
3304 | return 0; | | 3304 | return 0; |
3305 | } | | 3305 | } |
3306 | | | 3306 | |
3307 | /* | | 3307 | /* |
3308 | * uvm_map_pageable: sets the pageability of a range in a map. | | 3308 | * uvm_map_pageable: sets the pageability of a range in a map. |
3309 | * | | 3309 | * |
3310 | * => wires map entries. should not be used for transient page locking. | | 3310 | * => wires map entries. should not be used for transient page locking. |
3311 | * for that, use uvm_fault_wire()/uvm_fault_unwire() (see uvm_vslock()). | | 3311 | * for that, use uvm_fault_wire()/uvm_fault_unwire() (see uvm_vslock()). |
3312 | * => regions specified as not pageable require lock-down (wired) memory | | 3312 | * => regions specified as not pageable require lock-down (wired) memory |
3313 | * and page tables. | | 3313 | * and page tables. |
3314 | * => map must never be read-locked | | 3314 | * => map must never be read-locked |
3315 | * => if islocked is true, map is already write-locked | | 3315 | * => if islocked is true, map is already write-locked |
3316 | * => we always unlock the map, since we must downgrade to a read-lock | | 3316 | * => we always unlock the map, since we must downgrade to a read-lock |
3317 | * to call uvm_fault_wire() | | 3317 | * to call uvm_fault_wire() |
3318 | * => XXXCDC: check this and try and clean it up. | | 3318 | * => XXXCDC: check this and try and clean it up. |
3319 | */ | | 3319 | */ |
3320 | | | 3320 | |
3321 | int | | 3321 | int |
3322 | uvm_map_pageable(struct vm_map *map, vaddr_t start, vaddr_t end, | | 3322 | uvm_map_pageable(struct vm_map *map, vaddr_t start, vaddr_t end, |
3323 | bool new_pageable, int lockflags) | | 3323 | bool new_pageable, int lockflags) |
3324 | { | | 3324 | { |
3325 | struct vm_map_entry *entry, *start_entry, *failed_entry; | | 3325 | struct vm_map_entry *entry, *start_entry, *failed_entry; |
3326 | int rv; | | 3326 | int rv; |
3327 | #ifdef DIAGNOSTIC | | 3327 | #ifdef DIAGNOSTIC |
3328 | u_int timestamp_save; | | 3328 | u_int timestamp_save; |
3329 | #endif | | 3329 | #endif |
3330 | UVMHIST_FUNC("uvm_map_pageable"); UVMHIST_CALLED(maphist); | | 3330 | UVMHIST_FUNC("uvm_map_pageable"); UVMHIST_CALLED(maphist); |
3331 | UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_pageable=%ju)", | | 3331 | UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_pageable=%ju)", |
3332 | (uintptr_t)map, start, end, new_pageable); | | 3332 | (uintptr_t)map, start, end, new_pageable); |
3333 | KASSERT(map->flags & VM_MAP_PAGEABLE); | | 3333 | KASSERT(map->flags & VM_MAP_PAGEABLE); |
3334 | | | 3334 | |
3335 | if ((lockflags & UVM_LK_ENTER) == 0) | | 3335 | if ((lockflags & UVM_LK_ENTER) == 0) |
3336 | vm_map_lock(map); | | 3336 | vm_map_lock(map); |
3337 | VM_MAP_RANGE_CHECK(map, start, end); | | 3337 | VM_MAP_RANGE_CHECK(map, start, end); |
3338 | | | 3338 | |
3339 | /* | | 3339 | /* |
3340 | * only one pageability change may take place at one time, since | | 3340 | * only one pageability change may take place at one time, since |
3341 | * uvm_fault_wire assumes it will be called only once for each | | 3341 | * uvm_fault_wire assumes it will be called only once for each |
3342 | * wiring/unwiring. therefore, we have to make sure we're actually | | 3342 | * wiring/unwiring. therefore, we have to make sure we're actually |
3343 | * changing the pageability for the entire region. we do so before | | 3343 | * changing the pageability for the entire region. we do so before |
3344 | * making any changes. | | 3344 | * making any changes. |
3345 | */ | | 3345 | */ |
3346 | | | 3346 | |
3347 | if (uvm_map_lookup_entry(map, start, &start_entry) == false) { | | 3347 | if (uvm_map_lookup_entry(map, start, &start_entry) == false) { |
3348 | if ((lockflags & UVM_LK_EXIT) == 0) | | 3348 | if ((lockflags & UVM_LK_EXIT) == 0) |
3349 | vm_map_unlock(map); | | 3349 | vm_map_unlock(map); |
3350 | | | 3350 | |
3351 | UVMHIST_LOG(maphist,"<- done (fault)",0,0,0,0); | | 3351 | UVMHIST_LOG(maphist,"<- done (fault)",0,0,0,0); |
3352 | return EFAULT; | | 3352 | return EFAULT; |
3353 | } | | 3353 | } |
3354 | entry = start_entry; | | 3354 | entry = start_entry; |
3355 | | | 3355 | |
3356 | if (start == end) { /* nothing required */ | | 3356 | if (start == end) { /* nothing required */ |
3357 | if ((lockflags & UVM_LK_EXIT) == 0) | | 3357 | if ((lockflags & UVM_LK_EXIT) == 0) |
3358 | vm_map_unlock(map); | | 3358 | vm_map_unlock(map); |
3359 | | | 3359 | |
3360 | UVMHIST_LOG(maphist,"<- done (nothing)",0,0,0,0); | | 3360 | UVMHIST_LOG(maphist,"<- done (nothing)",0,0,0,0); |
3361 | return 0; | | 3361 | return 0; |
3362 | } | | 3362 | } |
3363 | | | 3363 | |
3364 | /* | | 3364 | /* |
3365 | * handle wiring and unwiring separately. | | 3365 | * handle wiring and unwiring separately. |
3366 | */ | | 3366 | */ |
3367 | | | 3367 | |
3368 | if (new_pageable) { /* unwire */ | | 3368 | if (new_pageable) { /* unwire */ |
3369 | UVM_MAP_CLIP_START(map, entry, start); | | 3369 | UVM_MAP_CLIP_START(map, entry, start); |
3370 | | | 3370 | |
3371 | /* | | 3371 | /* |
3372 | * unwiring. first ensure that the range to be unwired is | | 3372 | * unwiring. first ensure that the range to be unwired is |
3373 | * really wired down and that there are no holes. | | 3373 | * really wired down and that there are no holes. |
3374 | */ | | 3374 | */ |
3375 | | | 3375 | |
3376 | while ((entry != &map->header) && (entry->start < end)) { | | 3376 | while ((entry != &map->header) && (entry->start < end)) { |
3377 | if (entry->wired_count == 0 || | | 3377 | if (entry->wired_count == 0 || |
3378 | (entry->end < end && | | 3378 | (entry->end < end && |
3379 | (entry->next == &map->header || | | 3379 | (entry->next == &map->header || |
3380 | entry->next->start > entry->end))) { | | 3380 | entry->next->start > entry->end))) { |
3381 | if ((lockflags & UVM_LK_EXIT) == 0) | | 3381 | if ((lockflags & UVM_LK_EXIT) == 0) |
3382 | vm_map_unlock(map); | | 3382 | vm_map_unlock(map); |
3383 | UVMHIST_LOG(maphist, "<- done (INVAL)",0,0,0,0); | | 3383 | UVMHIST_LOG(maphist, "<- done (INVAL)",0,0,0,0); |
3384 | return EINVAL; | | 3384 | return EINVAL; |
3385 | } | | 3385 | } |
3386 | entry = entry->next; | | 3386 | entry = entry->next; |
3387 | } | | 3387 | } |
3388 | | | 3388 | |
3389 | /* | | 3389 | /* |
3390 | * POSIX 1003.1b - a single munlock call unlocks a region, | | 3390 | * POSIX 1003.1b - a single munlock call unlocks a region, |
3391 | * regardless of the number of mlock calls made on that | | 3391 | * regardless of the number of mlock calls made on that |
3392 | * region. | | 3392 | * region. |
3393 | */ | | 3393 | */ |
3394 | | | 3394 | |
3395 | entry = start_entry; | | 3395 | entry = start_entry; |
3396 | while ((entry != &map->header) && (entry->start < end)) { | | 3396 | while ((entry != &map->header) && (entry->start < end)) { |
3397 | UVM_MAP_CLIP_END(map, entry, end); | | 3397 | UVM_MAP_CLIP_END(map, entry, end); |
3398 | if (VM_MAPENT_ISWIRED(entry)) | | 3398 | if (VM_MAPENT_ISWIRED(entry)) |
3399 | uvm_map_entry_unwire(map, entry); | | 3399 | uvm_map_entry_unwire(map, entry); |
3400 | entry = entry->next; | | 3400 | entry = entry->next; |
3401 | } | | 3401 | } |
3402 | if ((lockflags & UVM_LK_EXIT) == 0) | | 3402 | if ((lockflags & UVM_LK_EXIT) == 0) |
3403 | vm_map_unlock(map); | | 3403 | vm_map_unlock(map); |
3404 | UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0); | | 3404 | UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0); |
3405 | return 0; | | 3405 | return 0; |
3406 | } | | 3406 | } |
3407 | | | 3407 | |
3408 | /* | | 3408 | /* |
3409 | * wire case: in two passes [XXXCDC: ugly block of code here] | | 3409 | * wire case: in two passes [XXXCDC: ugly block of code here] |
3410 | * | | 3410 | * |
3411 | * 1: holding the write lock, we create any anonymous maps that need | | 3411 | * 1: holding the write lock, we create any anonymous maps that need |
3412 | * to be created. then we clip each map entry to the region to | | 3412 | * to be created. then we clip each map entry to the region to |
3413 | * be wired and increment its wiring count. | | 3413 | * be wired and increment its wiring count. |
3414 | * | | 3414 | * |
3415 | * 2: we downgrade to a read lock, and call uvm_fault_wire to fault | | 3415 | * 2: we downgrade to a read lock, and call uvm_fault_wire to fault |
3416 | * in the pages for any newly wired area (wired_count == 1). | | 3416 | * in the pages for any newly wired area (wired_count == 1). |
3417 | * | | 3417 | * |
3418 | * downgrading to a read lock for uvm_fault_wire avoids a possible | | 3418 | * downgrading to a read lock for uvm_fault_wire avoids a possible |
3419 | * deadlock with another thread that may have faulted on one of | | 3419 | * deadlock with another thread that may have faulted on one of |
3420 | * the pages to be wired (it would mark the page busy, blocking | | 3420 | * the pages to be wired (it would mark the page busy, blocking |
3421 | * us, then in turn block on the map lock that we hold). because | | 3421 | * us, then in turn block on the map lock that we hold). because |
3422 | * of problems in the recursive lock package, we cannot upgrade | | 3422 | * of problems in the recursive lock package, we cannot upgrade |
3423 | * to a write lock in vm_map_lookup. thus, any actions that | | 3423 | * to a write lock in vm_map_lookup. thus, any actions that |
3424 | * require the write lock must be done beforehand. because we | | 3424 | * require the write lock must be done beforehand. because we |
3425 | * keep the read lock on the map, the copy-on-write status of the | | 3425 | * keep the read lock on the map, the copy-on-write status of the |
3426 | * entries we modify here cannot change. | | 3426 | * entries we modify here cannot change. |
3427 | */ | | 3427 | */ |
3428 | | | 3428 | |
3429 | while ((entry != &map->header) && (entry->start < end)) { | | 3429 | while ((entry != &map->header) && (entry->start < end)) { |
3430 | if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */ | | 3430 | if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */ |
3431 | | | 3431 | |
3432 | /* | | 3432 | /* |
3433 | * perform actions of vm_map_lookup that need the | | 3433 | * perform actions of vm_map_lookup that need the |
3434 | * write lock on the map: create an anonymous map | | 3434 | * write lock on the map: create an anonymous map |
3435 | * for a copy-on-write region, or an anonymous map | | 3435 | * for a copy-on-write region, or an anonymous map |
3436 | * for a zero-fill region. (XXXCDC: submap case | | 3436 | * for a zero-fill region. (XXXCDC: submap case |
3437 | * ok?) | | 3437 | * ok?) |
3438 | */ | | 3438 | */ |
3439 | | | 3439 | |
3440 | if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */ | | 3440 | if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */ |
3441 | if (UVM_ET_ISNEEDSCOPY(entry) && | | 3441 | if (UVM_ET_ISNEEDSCOPY(entry) && |
3442 | ((entry->max_protection & VM_PROT_WRITE) || | | 3442 | ((entry->max_protection & VM_PROT_WRITE) || |
3443 | (entry->object.uvm_obj == NULL))) { | | 3443 | (entry->object.uvm_obj == NULL))) { |
3444 | amap_copy(map, entry, 0, start, end); | | 3444 | amap_copy(map, entry, 0, start, end); |
3445 | /* XXXCDC: wait OK? */ | | 3445 | /* XXXCDC: wait OK? */ |
3446 | } | | 3446 | } |
3447 | } | | 3447 | } |
3448 | } | | 3448 | } |
3449 | UVM_MAP_CLIP_START(map, entry, start); | | 3449 | UVM_MAP_CLIP_START(map, entry, start); |
3450 | UVM_MAP_CLIP_END(map, entry, end); | | 3450 | UVM_MAP_CLIP_END(map, entry, end); |
3451 | entry->wired_count++; | | 3451 | entry->wired_count++; |
3452 | | | 3452 | |
3453 | /* | | 3453 | /* |
3454 | * Check for holes | | 3454 | * Check for holes |
3455 | */ | | 3455 | */ |
3456 | | | 3456 | |
3457 | if (entry->protection == VM_PROT_NONE || | | 3457 | if (entry->protection == VM_PROT_NONE || |
3458 | (entry->end < end && | | 3458 | (entry->end < end && |
3459 | (entry->next == &map->header || | | 3459 | (entry->next == &map->header || |
3460 | entry->next->start > entry->end))) { | | 3460 | entry->next->start > entry->end))) { |
3461 | | | 3461 | |
3462 | /* | | 3462 | /* |
3463 | * found one. amap creation actions do not need to | | 3463 | * found one. amap creation actions do not need to |
3464 | * be undone, but the wired counts need to be restored. | | 3464 | * be undone, but the wired counts need to be restored. |
3465 | */ | | 3465 | */ |
3466 | | | 3466 | |
3467 | while (entry != &map->header && entry->end > start) { | | 3467 | while (entry != &map->header && entry->end > start) { |
3468 | entry->wired_count--; | | 3468 | entry->wired_count--; |
3469 | entry = entry->prev; | | 3469 | entry = entry->prev; |
3470 | } | | 3470 | } |
3471 | if ((lockflags & UVM_LK_EXIT) == 0) | | 3471 | if ((lockflags & UVM_LK_EXIT) == 0) |
3472 | vm_map_unlock(map); | | 3472 | vm_map_unlock(map); |
3473 | UVMHIST_LOG(maphist,"<- done (INVALID WIRE)",0,0,0,0); | | 3473 | UVMHIST_LOG(maphist,"<- done (INVALID WIRE)",0,0,0,0); |
3474 | return EINVAL; | | 3474 | return EINVAL; |
3475 | } | | 3475 | } |
3476 | entry = entry->next; | | 3476 | entry = entry->next; |
3477 | } | | 3477 | } |
3478 | | | 3478 | |
3479 | /* | | 3479 | /* |
3480 | * Pass 2. | | 3480 | * Pass 2. |
3481 | */ | | 3481 | */ |
3482 | | | 3482 | |
3483 | #ifdef DIAGNOSTIC | | 3483 | #ifdef DIAGNOSTIC |
3484 | timestamp_save = map->timestamp; | | 3484 | timestamp_save = map->timestamp; |
3485 | #endif | | 3485 | #endif |
3486 | vm_map_busy(map); | | 3486 | vm_map_busy(map); |
3487 | vm_map_unlock(map); | | 3487 | vm_map_unlock(map); |
3488 | | | 3488 | |
3489 | rv = 0; | | 3489 | rv = 0; |
3490 | entry = start_entry; | | 3490 | entry = start_entry; |
3491 | while (entry != &map->header && entry->start < end) { | | 3491 | while (entry != &map->header && entry->start < end) { |
3492 | if (entry->wired_count == 1) { | | 3492 | if (entry->wired_count == 1) { |
3493 | rv = uvm_fault_wire(map, entry->start, entry->end, | | 3493 | rv = uvm_fault_wire(map, entry->start, entry->end, |
3494 | entry->max_protection, 1); | | 3494 | entry->max_protection, 1); |
3495 | if (rv) { | | 3495 | if (rv) { |
3496 | | | 3496 | |
3497 | /* | | 3497 | /* |
3498 | * wiring failed. break out of the loop. | | 3498 | * wiring failed. break out of the loop. |
3499 | * we'll clean up the map below, once we | | 3499 | * we'll clean up the map below, once we |
3500 | * have a write lock again. | | 3500 | * have a write lock again. |
3501 | */ | | 3501 | */ |
3502 | | | 3502 | |
3503 | break; | | 3503 | break; |
3504 | } | | 3504 | } |
3505 | } | | 3505 | } |
3506 | entry = entry->next; | | 3506 | entry = entry->next; |
3507 | } | | 3507 | } |
3508 | | | 3508 | |
3509 | if (rv) { /* failed? */ | | 3509 | if (rv) { /* failed? */ |
3510 | | | 3510 | |
3511 | /* | | 3511 | /* |
3512 | * Get back to an exclusive (write) lock. | | 3512 | * Get back to an exclusive (write) lock. |
3513 | */ | | 3513 | */ |
3514 | | | 3514 | |
3515 | vm_map_lock(map); | | 3515 | vm_map_lock(map); |
3516 | vm_map_unbusy(map); | | 3516 | vm_map_unbusy(map); |
3517 | | | 3517 | |
3518 | #ifdef DIAGNOSTIC | | 3518 | #ifdef DIAGNOSTIC |
3519 | if (timestamp_save + 1 != map->timestamp) | | 3519 | if (timestamp_save + 1 != map->timestamp) |
3520 | panic("uvm_map_pageable: stale map"); | | 3520 | panic("uvm_map_pageable: stale map"); |
3521 | #endif | | 3521 | #endif |
3522 | | | 3522 | |
3523 | /* | | 3523 | /* |
3524 | * first drop the wiring count on all the entries | | 3524 | * first drop the wiring count on all the entries |
3525 | * which haven't actually been wired yet. | | 3525 | * which haven't actually been wired yet. |
3526 | */ | | 3526 | */ |
3527 | | | 3527 | |
3528 | failed_entry = entry; | | 3528 | failed_entry = entry; |
3529 | while (entry != &map->header && entry->start < end) { | | 3529 | while (entry != &map->header && entry->start < end) { |
3530 | entry->wired_count--; | | 3530 | entry->wired_count--; |
3531 | entry = entry->next; | | 3531 | entry = entry->next; |
3532 | } | | 3532 | } |
3533 | | | 3533 | |
3534 | /* | | 3534 | /* |
3535 | * now, unwire all the entries that were successfully | | 3535 | * now, unwire all the entries that were successfully |
3536 | * wired above. | | 3536 | * wired above. |
3537 | */ | | 3537 | */ |
3538 | | | 3538 | |
3539 | entry = start_entry; | | 3539 | entry = start_entry; |
3540 | while (entry != failed_entry) { | | 3540 | while (entry != failed_entry) { |
3541 | entry->wired_count--; | | 3541 | entry->wired_count--; |
3542 | if (VM_MAPENT_ISWIRED(entry) == 0) | | 3542 | if (VM_MAPENT_ISWIRED(entry) == 0) |
3543 | uvm_map_entry_unwire(map, entry); | | 3543 | uvm_map_entry_unwire(map, entry); |
3544 | entry = entry->next; | | 3544 | entry = entry->next; |
3545 | } | | 3545 | } |
3546 | if ((lockflags & UVM_LK_EXIT) == 0) | | 3546 | if ((lockflags & UVM_LK_EXIT) == 0) |
3547 | vm_map_unlock(map); | | 3547 | vm_map_unlock(map); |
3548 | UVMHIST_LOG(maphist, "<- done (RV=%jd)", rv,0,0,0); | | 3548 | UVMHIST_LOG(maphist, "<- done (RV=%jd)", rv,0,0,0); |
3549 | return (rv); | | 3549 | return (rv); |
3550 | } | | 3550 | } |
3551 | | | 3551 | |
3552 | if ((lockflags & UVM_LK_EXIT) == 0) { | | 3552 | if ((lockflags & UVM_LK_EXIT) == 0) { |
3553 | vm_map_unbusy(map); | | 3553 | vm_map_unbusy(map); |
3554 | } else { | | 3554 | } else { |
3555 | | | 3555 | |
3556 | /* | | 3556 | /* |
3557 | * Get back to an exclusive (write) lock. | | 3557 | * Get back to an exclusive (write) lock. |
3558 | */ | | 3558 | */ |
3559 | | | 3559 | |
3560 | vm_map_lock(map); | | 3560 | vm_map_lock(map); |
3561 | vm_map_unbusy(map); | | 3561 | vm_map_unbusy(map); |
3562 | } | | 3562 | } |
3563 | | | 3563 | |
3564 | UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0); | | 3564 | UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0); |
3565 | return 0; | | 3565 | return 0; |
3566 | } | | 3566 | } |
3567 | | | 3567 | |
3568 | /* | | 3568 | /* |
3569 | * uvm_map_pageable_all: special case of uvm_map_pageable - affects | | 3569 | * uvm_map_pageable_all: special case of uvm_map_pageable - affects |
3570 | * all mapped regions. | | 3570 | * all mapped regions. |
3571 | * | | 3571 | * |
3572 | * => map must not be locked. | | 3572 | * => map must not be locked. |
3573 | * => if no flags are specified, all regions are unwired. | | 3573 | * => if no flags are specified, all regions are unwired. |
3574 | * => XXXJRT: has some of the same problems as uvm_map_pageable() above. | | 3574 | * => XXXJRT: has some of the same problems as uvm_map_pageable() above. |
3575 | */ | | 3575 | */ |
3576 | | | 3576 | |
3577 | int | | 3577 | int |
3578 | uvm_map_pageable_all(struct vm_map *map, int flags, vsize_t limit) | | 3578 | uvm_map_pageable_all(struct vm_map *map, int flags, vsize_t limit) |
3579 | { | | 3579 | { |
3580 | struct vm_map_entry *entry, *failed_entry; | | 3580 | struct vm_map_entry *entry, *failed_entry; |
3581 | vsize_t size; | | 3581 | vsize_t size; |
3582 | int rv; | | 3582 | int rv; |
3583 | #ifdef DIAGNOSTIC | | 3583 | #ifdef DIAGNOSTIC |
3584 | u_int timestamp_save; | | 3584 | u_int timestamp_save; |
3585 | #endif | | 3585 | #endif |
3586 | UVMHIST_FUNC("uvm_map_pageable_all"); UVMHIST_CALLED(maphist); | | 3586 | UVMHIST_FUNC("uvm_map_pageable_all"); UVMHIST_CALLED(maphist); |
3587 | UVMHIST_LOG(maphist,"(map=%#jx,flags=%#jx)", (uintptr_t)map, flags, | | 3587 | UVMHIST_LOG(maphist,"(map=%#jx,flags=%#jx)", (uintptr_t)map, flags, |
3588 | 0, 0); | | 3588 | 0, 0); |
3589 | | | 3589 | |
3590 | KASSERT(map->flags & VM_MAP_PAGEABLE); | | 3590 | KASSERT(map->flags & VM_MAP_PAGEABLE); |
3591 | | | 3591 | |
3592 | vm_map_lock(map); | | 3592 | vm_map_lock(map); |
3593 | | | 3593 | |
3594 | /* | | 3594 | /* |
3595 | * handle wiring and unwiring separately. | | 3595 | * handle wiring and unwiring separately. |
3596 | */ | | 3596 | */ |
3597 | | | 3597 | |
3598 | if (flags == 0) { /* unwire */ | | 3598 | if (flags == 0) { /* unwire */ |
3599 | | | 3599 | |
3600 | /* | | 3600 | /* |
3601 | * POSIX 1003.1b -- munlockall unlocks all regions, | | 3601 | * POSIX 1003.1b -- munlockall unlocks all regions, |
3602 | * regardless of how many times mlockall has been called. | | 3602 | * regardless of how many times mlockall has been called. |
3603 | */ | | 3603 | */ |
3604 | | | 3604 | |
3605 | for (entry = map->header.next; entry != &map->header; | | 3605 | for (entry = map->header.next; entry != &map->header; |
3606 | entry = entry->next) { | | 3606 | entry = entry->next) { |
3607 | if (VM_MAPENT_ISWIRED(entry)) | | 3607 | if (VM_MAPENT_ISWIRED(entry)) |
3608 | uvm_map_entry_unwire(map, entry); | | 3608 | uvm_map_entry_unwire(map, entry); |
3609 | } | | 3609 | } |
3610 | map->flags &= ~VM_MAP_WIREFUTURE; | | 3610 | map->flags &= ~VM_MAP_WIREFUTURE; |
3611 | vm_map_unlock(map); | | 3611 | vm_map_unlock(map); |
3612 | UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0); | | 3612 | UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0); |
3613 | return 0; | | 3613 | return 0; |
3614 | } | | 3614 | } |
3615 | | | 3615 | |
3616 | if (flags & MCL_FUTURE) { | | 3616 | if (flags & MCL_FUTURE) { |
3617 | | | 3617 | |
3618 | /* | | 3618 | /* |
3619 | * must wire all future mappings; remember this. | | 3619 | * must wire all future mappings; remember this. |
3620 | */ | | 3620 | */ |
3621 | | | 3621 | |
3622 | map->flags |= VM_MAP_WIREFUTURE; | | 3622 | map->flags |= VM_MAP_WIREFUTURE; |
3623 | } | | 3623 | } |
3624 | | | 3624 | |
3625 | if ((flags & MCL_CURRENT) == 0) { | | 3625 | if ((flags & MCL_CURRENT) == 0) { |
3626 | | | 3626 | |
3627 | /* | | 3627 | /* |
3628 | * no more work to do! | | 3628 | * no more work to do! |
3629 | */ | | 3629 | */ |
3630 | | | 3630 | |
3631 | UVMHIST_LOG(maphist,"<- done (OK no wire)",0,0,0,0); | | 3631 | UVMHIST_LOG(maphist,"<- done (OK no wire)",0,0,0,0); |
3632 | vm_map_unlock(map); | | 3632 | vm_map_unlock(map); |
3633 | return 0; | | 3633 | return 0; |
3634 | } | | 3634 | } |
3635 | | | 3635 | |
3636 | /* | | 3636 | /* |
3637 | * wire case: in three passes [XXXCDC: ugly block of code here] | | 3637 | * wire case: in three passes [XXXCDC: ugly block of code here] |
3638 | * | | 3638 | * |
3639 | * 1: holding the write lock, count all pages mapped by non-wired | | 3639 | * 1: holding the write lock, count all pages mapped by non-wired |
3640 | * entries. if this would cause us to go over our limit, we fail. | | 3640 | * entries. if this would cause us to go over our limit, we fail. |
3641 | * | | 3641 | * |
3642 | * 2: still holding the write lock, we create any anonymous maps that | | 3642 | * 2: still holding the write lock, we create any anonymous maps that |
3643 | * need to be created. then we increment its wiring count. | | 3643 | * need to be created. then we increment its wiring count. |
3644 | * | | 3644 | * |
3645 | * 3: we downgrade to a read lock, and call uvm_fault_wire to fault | | 3645 | * 3: we downgrade to a read lock, and call uvm_fault_wire to fault |
3646 | * in the pages for any newly wired area (wired_count == 1). | | 3646 | * in the pages for any newly wired area (wired_count == 1). |
3647 | * | | 3647 | * |
3648 | * downgrading to a read lock for uvm_fault_wire avoids a possible | | 3648 | * downgrading to a read lock for uvm_fault_wire avoids a possible |
3649 | * deadlock with another thread that may have faulted on one of | | 3649 | * deadlock with another thread that may have faulted on one of |
3650 | * the pages to be wired (it would mark the page busy, blocking | | 3650 | * the pages to be wired (it would mark the page busy, blocking |
3651 | * us, then in turn block on the map lock that we hold). because | | 3651 | * us, then in turn block on the map lock that we hold). because |
3652 | * of problems in the recursive lock package, we cannot upgrade | | 3652 | * of problems in the recursive lock package, we cannot upgrade |
3653 | * to a write lock in vm_map_lookup. thus, any actions that | | 3653 | * to a write lock in vm_map_lookup. thus, any actions that |
3654 | * require the write lock must be done beforehand. because we | | 3654 | * require the write lock must be done beforehand. because we |
3655 | * keep the read lock on the map, the copy-on-write status of the | | 3655 | * keep the read lock on the map, the copy-on-write status of the |
3656 | * entries we modify here cannot change. | | 3656 | * entries we modify here cannot change. |
3657 | */ | | 3657 | */ |
3658 | | | 3658 | |
3659 | for (size = 0, entry = map->header.next; entry != &map->header; | | 3659 | for (size = 0, entry = map->header.next; entry != &map->header; |
3660 | entry = entry->next) { | | 3660 | entry = entry->next) { |
3661 | if (entry->protection != VM_PROT_NONE && | | 3661 | if (entry->protection != VM_PROT_NONE && |
3662 | VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */ | | 3662 | VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */ |
3663 | size += entry->end - entry->start; | | 3663 | size += entry->end - entry->start; |
3664 | } | | 3664 | } |
3665 | } | | 3665 | } |
3666 | | | 3666 | |
3667 | if (atop(size) + uvmexp.wired > uvmexp.wiredmax) { | | 3667 | if (atop(size) + uvmexp.wired > uvmexp.wiredmax) { |
3668 | vm_map_unlock(map); | | 3668 | vm_map_unlock(map); |
3669 | return ENOMEM; | | 3669 | return ENOMEM; |
3670 | } | | 3670 | } |
3671 | | | 3671 | |
3672 | if (limit != 0 && | | 3672 | if (limit != 0 && |
3673 | (size + ptoa(pmap_wired_count(vm_map_pmap(map))) > limit)) { | | 3673 | (size + ptoa(pmap_wired_count(vm_map_pmap(map))) > limit)) { |
3674 | vm_map_unlock(map); | | 3674 | vm_map_unlock(map); |
3675 | return ENOMEM; | | 3675 | return ENOMEM; |
3676 | } | | 3676 | } |
3677 | | | 3677 | |
3678 | /* | | 3678 | /* |
3679 | * Pass 2. | | 3679 | * Pass 2. |
3680 | */ | | 3680 | */ |
3681 | | | 3681 | |
3682 | for (entry = map->header.next; entry != &map->header; | | 3682 | for (entry = map->header.next; entry != &map->header; |
3683 | entry = entry->next) { | | 3683 | entry = entry->next) { |
3684 | if (entry->protection == VM_PROT_NONE) | | 3684 | if (entry->protection == VM_PROT_NONE) |
3685 | continue; | | 3685 | continue; |
3686 | if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */ | | 3686 | if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */ |
3687 | | | 3687 | |
3688 | /* | | 3688 | /* |
3689 | * perform actions of vm_map_lookup that need the | | 3689 | * perform actions of vm_map_lookup that need the |
3690 | * write lock on the map: create an anonymous map | | 3690 | * write lock on the map: create an anonymous map |
3691 | * for a copy-on-write region, or an anonymous map | | 3691 | * for a copy-on-write region, or an anonymous map |
3692 | * for a zero-fill region. (XXXCDC: submap case | | 3692 | * for a zero-fill region. (XXXCDC: submap case |
3693 | * ok?) | | 3693 | * ok?) |
3694 | */ | | 3694 | */ |
3695 | | | 3695 | |
3696 | if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */ | | 3696 | if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */ |
3697 | if (UVM_ET_ISNEEDSCOPY(entry) && | | 3697 | if (UVM_ET_ISNEEDSCOPY(entry) && |
3698 | ((entry->max_protection & VM_PROT_WRITE) || | | 3698 | ((entry->max_protection & VM_PROT_WRITE) || |
3699 | (entry->object.uvm_obj == NULL))) { | | 3699 | (entry->object.uvm_obj == NULL))) { |
3700 | amap_copy(map, entry, 0, entry->start, | | 3700 | amap_copy(map, entry, 0, entry->start, |
3701 | entry->end); | | 3701 | entry->end); |
3702 | /* XXXCDC: wait OK? */ | | 3702 | /* XXXCDC: wait OK? */ |
3703 | } | | 3703 | } |
3704 | } | | 3704 | } |
3705 | } | | 3705 | } |
3706 | entry->wired_count++; | | 3706 | entry->wired_count++; |
3707 | } | | 3707 | } |
3708 | | | 3708 | |
3709 | /* | | 3709 | /* |
3710 | * Pass 3. | | 3710 | * Pass 3. |
3711 | */ | | 3711 | */ |
3712 | | | 3712 | |
3713 | #ifdef DIAGNOSTIC | | 3713 | #ifdef DIAGNOSTIC |
3714 | timestamp_save = map->timestamp; | | 3714 | timestamp_save = map->timestamp; |
3715 | #endif | | 3715 | #endif |
3716 | vm_map_busy(map); | | 3716 | vm_map_busy(map); |
3717 | vm_map_unlock(map); | | 3717 | vm_map_unlock(map); |
3718 | | | 3718 | |
3719 | rv = 0; | | 3719 | rv = 0; |
3720 | for (entry = map->header.next; entry != &map->header; | | 3720 | for (entry = map->header.next; entry != &map->header; |
3721 | entry = entry->next) { | | 3721 | entry = entry->next) { |
3722 | if (entry->wired_count == 1) { | | 3722 | if (entry->wired_count == 1) { |
3723 | rv = uvm_fault_wire(map, entry->start, entry->end, | | 3723 | rv = uvm_fault_wire(map, entry->start, entry->end, |
3724 | entry->max_protection, 1); | | 3724 | entry->max_protection, 1); |
3725 | if (rv) { | | 3725 | if (rv) { |
3726 | | | 3726 | |
3727 | /* | | 3727 | /* |
3728 | * wiring failed. break out of the loop. | | 3728 | * wiring failed. break out of the loop. |
3729 | * we'll clean up the map below, once we | | 3729 | * we'll clean up the map below, once we |
3730 | * have a write lock again. | | 3730 | * have a write lock again. |
3731 | */ | | 3731 | */ |
3732 | | | 3732 | |
3733 | break; | | 3733 | break; |
3734 | } | | 3734 | } |
3735 | } | | 3735 | } |
3736 | } | | 3736 | } |
3737 | | | 3737 | |
3738 | if (rv) { | | 3738 | if (rv) { |
3739 | | | 3739 | |
3740 | /* | | 3740 | /* |
3741 | * Get back an exclusive (write) lock. | | 3741 | * Get back an exclusive (write) lock. |
3742 | */ | | 3742 | */ |
3743 | | | 3743 | |
3744 | vm_map_lock(map); | | 3744 | vm_map_lock(map); |
3745 | vm_map_unbusy(map); | | 3745 | vm_map_unbusy(map); |
3746 | | | 3746 | |
3747 | #ifdef DIAGNOSTIC | | 3747 | #ifdef DIAGNOSTIC |
3748 | if (timestamp_save + 1 != map->timestamp) | | 3748 | if (timestamp_save + 1 != map->timestamp) |
3749 | panic("uvm_map_pageable_all: stale map"); | | 3749 | panic("uvm_map_pageable_all: stale map"); |
3750 | #endif | | 3750 | #endif |
3751 | | | 3751 | |
3752 | /* | | 3752 | /* |
3753 | * first drop the wiring count on all the entries | | 3753 | * first drop the wiring count on all the entries |
3754 | * which haven't actually been wired yet. | | 3754 | * which haven't actually been wired yet. |
3755 | * | | 3755 | * |
3756 | * Skip VM_PROT_NONE entries like we did above. | | 3756 | * Skip VM_PROT_NONE entries like we did above. |
3757 | */ | | 3757 | */ |
3758 | | | 3758 | |
3759 | failed_entry = entry; | | 3759 | failed_entry = entry; |
3760 | for (/* nothing */; entry != &map->header; | | 3760 | for (/* nothing */; entry != &map->header; |
3761 | entry = entry->next) { | | 3761 | entry = entry->next) { |
3762 | if (entry->protection == VM_PROT_NONE) | | 3762 | if (entry->protection == VM_PROT_NONE) |
3763 | continue; | | 3763 | continue; |
3764 | entry->wired_count--; | | 3764 | entry->wired_count--; |
3765 | } | | 3765 | } |
3766 | | | 3766 | |
3767 | /* | | 3767 | /* |
3768 | * now, unwire all the entries that were successfully | | 3768 | * now, unwire all the entries that were successfully |
3769 | * wired above. | | 3769 | * wired above. |
3770 | * | | 3770 | * |
3771 | * Skip VM_PROT_NONE entries like we did above. | | 3771 | * Skip VM_PROT_NONE entries like we did above. |
3772 | */ | | 3772 | */ |
3773 | | | 3773 | |
3774 | for (entry = map->header.next; entry != failed_entry; | | 3774 | for (entry = map->header.next; entry != failed_entry; |
3775 | entry = entry->next) { | | 3775 | entry = entry->next) { |
3776 | if (entry->protection == VM_PROT_NONE) | | 3776 | if (entry->protection == VM_PROT_NONE) |
3777 | continue; | | 3777 | continue; |
3778 | entry->wired_count--; | | 3778 | entry->wired_count--; |
3779 | if (VM_MAPENT_ISWIRED(entry)) | | 3779 | if (VM_MAPENT_ISWIRED(entry)) |
3780 | uvm_map_entry_unwire(map, entry); | | 3780 | uvm_map_entry_unwire(map, entry); |
3781 | } | | 3781 | } |
3782 | vm_map_unlock(map); | | 3782 | vm_map_unlock(map); |
3783 | UVMHIST_LOG(maphist,"<- done (RV=%jd)", rv,0,0,0); | | 3783 | UVMHIST_LOG(maphist,"<- done (RV=%jd)", rv,0,0,0); |
3784 | return (rv); | | 3784 | return (rv); |
3785 | } | | 3785 | } |
3786 | | | 3786 | |
3787 | vm_map_unbusy(map); | | 3787 | vm_map_unbusy(map); |
3788 | | | 3788 | |
3789 | UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0); | | 3789 | UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0); |
3790 | return 0; | | 3790 | return 0; |
3791 | } | | 3791 | } |
3792 | | | 3792 | |
3793 | /* | | 3793 | /* |
3794 | * uvm_map_clean: clean out a map range | | 3794 | * uvm_map_clean: clean out a map range |
3795 | * | | 3795 | * |
3796 | * => valid flags: | | 3796 | * => valid flags: |
3797 | * if (flags & PGO_CLEANIT): dirty pages are cleaned first | | 3797 | * if (flags & PGO_CLEANIT): dirty pages are cleaned first |
3798 | * if (flags & PGO_SYNCIO): dirty pages are written synchronously | | 3798 | * if (flags & PGO_SYNCIO): dirty pages are written synchronously |
3799 | * if (flags & PGO_DEACTIVATE): any cached pages are deactivated after clean | | 3799 | * if (flags & PGO_DEACTIVATE): any cached pages are deactivated after clean |
3800 | * if (flags & PGO_FREE): any cached pages are freed after clean | | 3800 | * if (flags & PGO_FREE): any cached pages are freed after clean |
3801 | * => returns an error if any part of the specified range isn't mapped | | 3801 | * => returns an error if any part of the specified range isn't mapped |
3802 | * => never a need to flush amap layer since the anonymous memory has | | 3802 | * => never a need to flush amap layer since the anonymous memory has |
3803 | * no permanent home, but may deactivate pages there | | 3803 | * no permanent home, but may deactivate pages there |
3804 | * => called from sys_msync() and sys_madvise() | | 3804 | * => called from sys_msync() and sys_madvise() |
3805 | * => caller must not write-lock map (read OK). | | 3805 | * => caller must not write-lock map (read OK). |
3806 | * => we may sleep while cleaning if SYNCIO [with map read-locked] | | 3806 | * => we may sleep while cleaning if SYNCIO [with map read-locked] |
3807 | */ | | 3807 | */ |
3808 | | | 3808 | |
3809 | int | | 3809 | int |
3810 | uvm_map_clean(struct vm_map *map, vaddr_t start, vaddr_t end, int flags) | | 3810 | uvm_map_clean(struct vm_map *map, vaddr_t start, vaddr_t end, int flags) |
3811 | { | | 3811 | { |
3812 | struct vm_map_entry *current, *entry; | | 3812 | struct vm_map_entry *current, *entry; |
3813 | struct uvm_object *uobj; | | 3813 | struct uvm_object *uobj; |
3814 | struct vm_amap *amap; | | 3814 | struct vm_amap *amap; |
3815 | struct vm_anon *anon; | | 3815 | struct vm_anon *anon; |
3816 | struct vm_page *pg; | | 3816 | struct vm_page *pg; |
3817 | vaddr_t offset; | | 3817 | vaddr_t offset; |
3818 | vsize_t size; | | 3818 | vsize_t size; |
3819 | voff_t uoff; | | 3819 | voff_t uoff; |
3820 | int error, refs; | | 3820 | int error, refs; |
3821 | UVMHIST_FUNC("uvm_map_clean"); UVMHIST_CALLED(maphist); | | 3821 | UVMHIST_FUNC("uvm_map_clean"); UVMHIST_CALLED(maphist); |
3822 | | | 3822 | |
3823 | UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,flags=%#jx)", | | 3823 | UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,flags=%#jx)", |
3824 | (uintptr_t)map, start, end, flags); | | 3824 | (uintptr_t)map, start, end, flags); |
3825 | KASSERT((flags & (PGO_FREE|PGO_DEACTIVATE)) != | | 3825 | KASSERT((flags & (PGO_FREE|PGO_DEACTIVATE)) != |
3826 | (PGO_FREE|PGO_DEACTIVATE)); | | 3826 | (PGO_FREE|PGO_DEACTIVATE)); |
3827 | | | 3827 | |
3828 | vm_map_lock_read(map); | | 3828 | vm_map_lock_read(map); |
3829 | VM_MAP_RANGE_CHECK(map, start, end); | | 3829 | VM_MAP_RANGE_CHECK(map, start, end); |
3830 | if (uvm_map_lookup_entry(map, start, &entry) == false) { | | 3830 | if (uvm_map_lookup_entry(map, start, &entry) == false) { |
3831 | vm_map_unlock_read(map); | | 3831 | vm_map_unlock_read(map); |
3832 | return EFAULT; | | 3832 | return EFAULT; |
3833 | } | | 3833 | } |
3834 | | | 3834 | |
3835 | /* | | 3835 | /* |
3836 | * Make a first pass to check for holes and wiring problems. | | 3836 | * Make a first pass to check for holes and wiring problems. |
3837 | */ | | 3837 | */ |
3838 | | | 3838 | |
3839 | for (current = entry; current->start < end; current = current->next) { | | 3839 | for (current = entry; current->start < end; current = current->next) { |
3840 | if (UVM_ET_ISSUBMAP(current)) { | | 3840 | if (UVM_ET_ISSUBMAP(current)) { |
3841 | vm_map_unlock_read(map); | | 3841 | vm_map_unlock_read(map); |
3842 | return EINVAL; | | 3842 | return EINVAL; |
3843 | } | | 3843 | } |
3844 | if ((flags & PGO_FREE) != 0 && VM_MAPENT_ISWIRED(entry)) { | | 3844 | if ((flags & PGO_FREE) != 0 && VM_MAPENT_ISWIRED(entry)) { |
3845 | vm_map_unlock_read(map); | | 3845 | vm_map_unlock_read(map); |
3846 | return EBUSY; | | 3846 | return EBUSY; |
3847 | } | | 3847 | } |
3848 | if (end <= current->end) { | | 3848 | if (end <= current->end) { |
3849 | break; | | 3849 | break; |
3850 | } | | 3850 | } |
3851 | if (current->end != current->next->start) { | | 3851 | if (current->end != current->next->start) { |
3852 | vm_map_unlock_read(map); | | 3852 | vm_map_unlock_read(map); |
3853 | return EFAULT; | | 3853 | return EFAULT; |
3854 | } | | 3854 | } |
3855 | } | | 3855 | } |
3856 | | | 3856 | |
3857 | error = 0; | | 3857 | error = 0; |
3858 | for (current = entry; start < end; current = current->next) { | | 3858 | for (current = entry; start < end; current = current->next) { |
3859 | amap = current->aref.ar_amap; /* upper layer */ | | 3859 | amap = current->aref.ar_amap; /* upper layer */ |
3860 | uobj = current->object.uvm_obj; /* lower layer */ | | 3860 | uobj = current->object.uvm_obj; /* lower layer */ |
3861 | KASSERT(start >= current->start); | | 3861 | KASSERT(start >= current->start); |
3862 | | | 3862 | |
3863 | /* | | 3863 | /* |
3864 | * No amap cleaning necessary if: | | 3864 | * No amap cleaning necessary if: |
3865 | * | | 3865 | * |
3866 | * (1) There's no amap. | | 3866 | * (1) There's no amap. |
3867 | * | | 3867 | * |
3868 | * (2) We're not deactivating or freeing pages. | | 3868 | * (2) We're not deactivating or freeing pages. |
3869 | */ | | 3869 | */ |
3870 | | | 3870 | |
3871 | if (amap == NULL || (flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) | | 3871 | if (amap == NULL || (flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) |
3872 | goto flush_object; | | 3872 | goto flush_object; |
3873 | | | 3873 | |
3874 | offset = start - current->start; | | 3874 | offset = start - current->start; |
3875 | size = MIN(end, current->end) - start; | | 3875 | size = MIN(end, current->end) - start; |
3876 | | | 3876 | |
3877 | amap_lock(amap, RW_WRITER); | | 3877 | amap_lock(amap, RW_WRITER); |
3878 | for ( ; size != 0; size -= PAGE_SIZE, offset += PAGE_SIZE) { | | 3878 | for ( ; size != 0; size -= PAGE_SIZE, offset += PAGE_SIZE) { |
3879 | anon = amap_lookup(¤t->aref, offset); | | 3879 | anon = amap_lookup(¤t->aref, offset); |
3880 | if (anon == NULL) | | 3880 | if (anon == NULL) |
3881 | continue; | | 3881 | continue; |
3882 | | | 3882 | |
3883 | KASSERT(anon->an_lock == amap->am_lock); | | 3883 | KASSERT(anon->an_lock == amap->am_lock); |
3884 | pg = anon->an_page; | | 3884 | pg = anon->an_page; |
3885 | if (pg == NULL) { | | 3885 | if (pg == NULL) { |
3886 | continue; | | 3886 | continue; |
3887 | } | | 3887 | } |
3888 | if (pg->flags & PG_BUSY) { | | 3888 | if (pg->flags & PG_BUSY) { |
3889 | continue; | | 3889 | continue; |
3890 | } | | 3890 | } |
3891 | | | 3891 | |
3892 | switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) { | | 3892 | switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) { |
3893 | | | 3893 | |
3894 | /* | | 3894 | /* |
3895 | * In these first 3 cases, we just deactivate the page. | | 3895 | * In these first 3 cases, we just deactivate the page. |
3896 | */ | | 3896 | */ |
3897 | | | 3897 | |
3898 | case PGO_CLEANIT|PGO_FREE: | | 3898 | case PGO_CLEANIT|PGO_FREE: |
3899 | case PGO_CLEANIT|PGO_DEACTIVATE: | | 3899 | case PGO_CLEANIT|PGO_DEACTIVATE: |
3900 | case PGO_DEACTIVATE: | | 3900 | case PGO_DEACTIVATE: |
3901 | deactivate_it: | | 3901 | deactivate_it: |
3902 | /* | | 3902 | /* |
3903 | * skip the page if it's loaned or wired, | | 3903 | * skip the page if it's loaned or wired, |
3904 | * since it shouldn't be on a paging queue | | 3904 | * since it shouldn't be on a paging queue |
3905 | * at all in these cases. | | 3905 | * at all in these cases. |
3906 | */ | | 3906 | */ |
3907 | | | 3907 | |
3908 | if (pg->loan_count != 0 || | | 3908 | if (pg->loan_count != 0 || |
3909 | pg->wire_count != 0) { | | 3909 | pg->wire_count != 0) { |
3910 | continue; | | 3910 | continue; |
3911 | } | | 3911 | } |
3912 | KASSERT(pg->uanon == anon); | | 3912 | KASSERT(pg->uanon == anon); |
3913 | uvm_pagelock(pg); | | 3913 | uvm_pagelock(pg); |
3914 | uvm_pagedeactivate(pg); | | 3914 | uvm_pagedeactivate(pg); |
3915 | uvm_pageunlock(pg); | | 3915 | uvm_pageunlock(pg); |
3916 | continue; | | 3916 | continue; |
3917 | | | 3917 | |
3918 | case PGO_FREE: | | 3918 | case PGO_FREE: |
3919 | | | 3919 | |
3920 | /* | | 3920 | /* |
3921 | * If there are multiple references to | | 3921 | * If there are multiple references to |
3922 | * the amap, just deactivate the page. | | 3922 | * the amap, just deactivate the page. |
3923 | */ | | 3923 | */ |
3924 | | | 3924 | |
3925 | if (amap_refs(amap) > 1) | | 3925 | if (amap_refs(amap) > 1) |
3926 | goto deactivate_it; | | 3926 | goto deactivate_it; |
3927 | | | 3927 | |
3928 | /* skip the page if it's wired */ | | 3928 | /* skip the page if it's wired */ |
3929 | if (pg->wire_count != 0) { | | 3929 | if (pg->wire_count != 0) { |
3930 | continue; | | 3930 | continue; |
3931 | } | | 3931 | } |
3932 | amap_unadd(¤t->aref, offset); | | 3932 | amap_unadd(¤t->aref, offset); |
3933 | refs = --anon->an_ref; | | 3933 | refs = --anon->an_ref; |
3934 | if (refs == 0) { | | 3934 | if (refs == 0) { |
3935 | uvm_anfree(anon); | | 3935 | uvm_anfree(anon); |
3936 | } | | 3936 | } |
3937 | continue; | | 3937 | continue; |
3938 | } | | 3938 | } |
3939 | } | | 3939 | } |
3940 | amap_unlock(amap); | | 3940 | amap_unlock(amap); |
3941 | | | 3941 | |
3942 | flush_object: | | 3942 | flush_object: |
3943 | /* | | 3943 | /* |
3944 | * flush pages if we've got a valid backing object. | | 3944 | * flush pages if we've got a valid backing object. |
3945 | * note that we must always clean object pages before | | 3945 | * note that we must always clean object pages before |
3946 | * freeing them since otherwise we could reveal stale | | 3946 | * freeing them since otherwise we could reveal stale |
3947 | * data from files. | | 3947 | * data from files. |
3948 | */ | | 3948 | */ |
3949 | | | 3949 | |
3950 | uoff = current->offset + (start - current->start); | | 3950 | uoff = current->offset + (start - current->start); |
3951 | size = MIN(end, current->end) - start; | | 3951 | size = MIN(end, current->end) - start; |
3952 | if (uobj != NULL) { | | 3952 | if (uobj != NULL) { |
3953 | rw_enter(uobj->vmobjlock, RW_WRITER); | | 3953 | rw_enter(uobj->vmobjlock, RW_WRITER); |
3954 | if (uobj->pgops->pgo_put != NULL) | | 3954 | if (uobj->pgops->pgo_put != NULL) |
3955 | error = (uobj->pgops->pgo_put)(uobj, uoff, | | 3955 | error = (uobj->pgops->pgo_put)(uobj, uoff, |
3956 | uoff + size, flags | PGO_CLEANIT); | | 3956 | uoff + size, flags | PGO_CLEANIT); |
3957 | else | | 3957 | else |
3958 | error = 0; | | 3958 | error = 0; |
3959 | } | | 3959 | } |
3960 | start += size; | | 3960 | start += size; |
3961 | } | | 3961 | } |
3962 | vm_map_unlock_read(map); | | 3962 | vm_map_unlock_read(map); |
3963 | return (error); | | 3963 | return (error); |
3964 | } | | 3964 | } |
3965 | | | 3965 | |
3966 | | | 3966 | |
3967 | /* | | 3967 | /* |
3968 | * uvm_map_checkprot: check protection in map | | 3968 | * uvm_map_checkprot: check protection in map |
3969 | * | | 3969 | * |
3970 | * => must allow specified protection in a fully allocated region. | | 3970 | * => must allow specified protection in a fully allocated region. |
3971 | * => map must be read or write locked by caller. | | 3971 | * => map must be read or write locked by caller. |
3972 | */ | | 3972 | */ |
3973 | | | 3973 | |
3974 | bool | | 3974 | bool |
3975 | uvm_map_checkprot(struct vm_map *map, vaddr_t start, vaddr_t end, | | 3975 | uvm_map_checkprot(struct vm_map *map, vaddr_t start, vaddr_t end, |
3976 | vm_prot_t protection) | | 3976 | vm_prot_t protection) |
3977 | { | | 3977 | { |
3978 | struct vm_map_entry *entry; | | 3978 | struct vm_map_entry *entry; |
3979 | struct vm_map_entry *tmp_entry; | | 3979 | struct vm_map_entry *tmp_entry; |
3980 | | | 3980 | |
3981 | if (!uvm_map_lookup_entry(map, start, &tmp_entry)) { | | 3981 | if (!uvm_map_lookup_entry(map, start, &tmp_entry)) { |
3982 | return (false); | | 3982 | return (false); |
3983 | } | | 3983 | } |
3984 | entry = tmp_entry; | | 3984 | entry = tmp_entry; |
3985 | while (start < end) { | | 3985 | while (start < end) { |
3986 | if (entry == &map->header) { | | 3986 | if (entry == &map->header) { |
3987 | return (false); | | 3987 | return (false); |
3988 | } | | 3988 | } |
3989 | | | 3989 | |
3990 | /* | | 3990 | /* |
3991 | * no holes allowed | | 3991 | * no holes allowed |
3992 | */ | | 3992 | */ |
3993 | | | 3993 | |
3994 | if (start < entry->start) { | | 3994 | if (start < entry->start) { |
3995 | return (false); | | 3995 | return (false); |
3996 | } | | 3996 | } |
3997 | | | 3997 | |
3998 | /* | | 3998 | /* |
3999 | * check protection associated with entry | | 3999 | * check protection associated with entry |
4000 | */ | | 4000 | */ |
4001 | | | 4001 | |
4002 | if ((entry->protection & protection) != protection) { | | 4002 | if ((entry->protection & protection) != protection) { |
4003 | return (false); | | 4003 | return (false); |
4004 | } | | 4004 | } |
4005 | start = entry->end; | | 4005 | start = entry->end; |
4006 | entry = entry->next; | | 4006 | entry = entry->next; |
4007 | } | | 4007 | } |
4008 | return (true); | | 4008 | return (true); |
4009 | } | | 4009 | } |
4010 | | | 4010 | |
4011 | /* | | 4011 | /* |
4012 | * uvmspace_alloc: allocate a vmspace structure. | | 4012 | * uvmspace_alloc: allocate a vmspace structure. |
4013 | * | | 4013 | * |
4014 | * - structure includes vm_map and pmap | | 4014 | * - structure includes vm_map and pmap |
4015 | * - XXX: no locking on this structure | | 4015 | * - XXX: no locking on this structure |
4016 | * - refcnt set to 1, rest must be init'd by caller | | 4016 | * - refcnt set to 1, rest must be init'd by caller |
4017 | */ | | 4017 | */ |
4018 | struct vmspace * | | 4018 | struct vmspace * |
4019 | uvmspace_alloc(vaddr_t vmin, vaddr_t vmax, bool topdown) | | 4019 | uvmspace_alloc(vaddr_t vmin, vaddr_t vmax, bool topdown) |
4020 | { | | 4020 | { |
4021 | struct vmspace *vm; | | 4021 | struct vmspace *vm; |
4022 | UVMHIST_FUNC("uvmspace_alloc"); UVMHIST_CALLED(maphist); | | 4022 | UVMHIST_FUNC("uvmspace_alloc"); UVMHIST_CALLED(maphist); |
4023 | | | 4023 | |
4024 | vm = pool_cache_get(&uvm_vmspace_cache, PR_WAITOK); | | 4024 | vm = pool_cache_get(&uvm_vmspace_cache, PR_WAITOK); |
4025 | uvmspace_init(vm, NULL, vmin, vmax, topdown); | | 4025 | uvmspace_init(vm, NULL, vmin, vmax, topdown); |
4026 | UVMHIST_LOG(maphist,"<- done (vm=%#jx)", (uintptr_t)vm, 0, 0, 0); | | 4026 | UVMHIST_LOG(maphist,"<- done (vm=%#jx)", (uintptr_t)vm, 0, 0, 0); |
4027 | return (vm); | | 4027 | return (vm); |
4028 | } | | 4028 | } |
4029 | | | 4029 | |
4030 | /* | | 4030 | /* |
4031 | * uvmspace_init: initialize a vmspace structure. | | 4031 | * uvmspace_init: initialize a vmspace structure. |
4032 | * | | 4032 | * |
4033 | * - XXX: no locking on this structure | | 4033 | * - XXX: no locking on this structure |
4034 | * - refcnt set to 1, rest must be init'd by caller | | 4034 | * - refcnt set to 1, rest must be init'd by caller |
4035 | */ | | 4035 | */ |
4036 | void | | 4036 | void |
4037 | uvmspace_init(struct vmspace *vm, struct pmap *pmap, vaddr_t vmin, | | 4037 | uvmspace_init(struct vmspace *vm, struct pmap *pmap, vaddr_t vmin, |
4038 | vaddr_t vmax, bool topdown) | | 4038 | vaddr_t vmax, bool topdown) |
4039 | { | | 4039 | { |
4040 | UVMHIST_FUNC("uvmspace_init"); UVMHIST_CALLED(maphist); | | 4040 | UVMHIST_FUNC("uvmspace_init"); UVMHIST_CALLED(maphist); |
4041 | | | 4041 | |
4042 | UVMHIST_LOG(maphist, "(vm=%#jx, pmap=%#jx, vmin=%#jx, vmax=%#jx", | | 4042 | UVMHIST_LOG(maphist, "(vm=%#jx, pmap=%#jx, vmin=%#jx, vmax=%#jx", |
4043 | (uintptr_t)vm, (uintptr_t)pmap, vmin, vmax); | | 4043 | (uintptr_t)vm, (uintptr_t)pmap, vmin, vmax); |
4044 | UVMHIST_LOG(maphist, " topdown=%ju)", topdown, 0, 0, 0); | | 4044 | UVMHIST_LOG(maphist, " topdown=%ju)", topdown, 0, 0, 0); |
4045 | | | 4045 | |
4046 | memset(vm, 0, sizeof(*vm)); | | 4046 | memset(vm, 0, sizeof(*vm)); |
4047 | uvm_map_setup(&vm->vm_map, vmin, vmax, VM_MAP_PAGEABLE | | 4047 | uvm_map_setup(&vm->vm_map, vmin, vmax, VM_MAP_PAGEABLE |
4048 | | (topdown ? VM_MAP_TOPDOWN : 0) | | 4048 | | (topdown ? VM_MAP_TOPDOWN : 0) |
4049 | ); | | 4049 | ); |
4050 | if (pmap) | | 4050 | if (pmap) |
4051 | pmap_reference(pmap); | | 4051 | pmap_reference(pmap); |
4052 | else | | 4052 | else |
4053 | pmap = pmap_create(); | | 4053 | pmap = pmap_create(); |
4054 | vm->vm_map.pmap = pmap; | | 4054 | vm->vm_map.pmap = pmap; |
4055 | vm->vm_refcnt = 1; | | 4055 | vm->vm_refcnt = 1; |
4056 | UVMHIST_LOG(maphist,"<- done",0,0,0,0); | | 4056 | UVMHIST_LOG(maphist,"<- done",0,0,0,0); |
4057 | } | | 4057 | } |
4058 | | | 4058 | |
4059 | /* | | 4059 | /* |
4060 | * uvmspace_share: share a vmspace between two processes | | 4060 | * uvmspace_share: share a vmspace between two processes |
4061 | * | | 4061 | * |
4062 | * - used for vfork, threads(?) | | 4062 | * - used for vfork, threads(?) |
4063 | */ | | 4063 | */ |
4064 | | | 4064 | |
4065 | void | | 4065 | void |
4066 | uvmspace_share(struct proc *p1, struct proc *p2) | | 4066 | uvmspace_share(struct proc *p1, struct proc *p2) |
4067 | { | | 4067 | { |
4068 | | | 4068 | |
4069 | uvmspace_addref(p1->p_vmspace); | | 4069 | uvmspace_addref(p1->p_vmspace); |
4070 | p2->p_vmspace = p1->p_vmspace; | | 4070 | p2->p_vmspace = p1->p_vmspace; |
4071 | } | | 4071 | } |
4072 | | | 4072 | |
4073 | #if 0 | | 4073 | #if 0 |
4074 | | | 4074 | |
4075 | /* | | 4075 | /* |
4076 | * uvmspace_unshare: ensure that process "p" has its own, unshared, vmspace | | 4076 | * uvmspace_unshare: ensure that process "p" has its own, unshared, vmspace |
4077 | * | | 4077 | * |
4078 | * - XXX: no locking on vmspace | | 4078 | * - XXX: no locking on vmspace |
4079 | */ | | 4079 | */ |
4080 | | | 4080 | |
4081 | void | | 4081 | void |
4082 | uvmspace_unshare(struct lwp *l) | | 4082 | uvmspace_unshare(struct lwp *l) |
4083 | { | | 4083 | { |
4084 | struct proc *p = l->l_proc; | | 4084 | struct proc *p = l->l_proc; |
4085 | struct vmspace *nvm, *ovm = p->p_vmspace; | | 4085 | struct vmspace *nvm, *ovm = p->p_vmspace; |
4086 | | | 4086 | |
4087 | if (ovm->vm_refcnt == 1) | | 4087 | if (ovm->vm_refcnt == 1) |
4088 | /* nothing to do: vmspace isn't shared in the first place */ | | 4088 | /* nothing to do: vmspace isn't shared in the first place */ |
4089 | return; | | 4089 | return; |
4090 | | | 4090 | |
4091 | /* make a new vmspace, still holding old one */ | | 4091 | /* make a new vmspace, still holding old one */ |
4092 | nvm = uvmspace_fork(ovm); | | 4092 | nvm = uvmspace_fork(ovm); |
4093 | | | 4093 | |
4094 | kpreempt_disable(); | | 4094 | kpreempt_disable(); |
4095 | pmap_deactivate(l); /* unbind old vmspace */ | | 4095 | pmap_deactivate(l); /* unbind old vmspace */ |
4096 | p->p_vmspace = nvm; | | 4096 | p->p_vmspace = nvm; |
4097 | pmap_activate(l); /* switch to new vmspace */ | | 4097 | pmap_activate(l); /* switch to new vmspace */ |
4098 | kpreempt_enable(); | | 4098 | kpreempt_enable(); |
4099 | | | 4099 | |
4100 | uvmspace_free(ovm); /* drop reference to old vmspace */ | | 4100 | uvmspace_free(ovm); /* drop reference to old vmspace */ |
4101 | } | | 4101 | } |
4102 | | | 4102 | |
4103 | #endif | | 4103 | #endif |
4104 | | | 4104 | |
4105 | | | 4105 | |
4106 | /* | | 4106 | /* |
4107 | * uvmspace_spawn: a new process has been spawned and needs a vmspace | | 4107 | * uvmspace_spawn: a new process has been spawned and needs a vmspace |
4108 | */ | | 4108 | */ |
4109 | | | 4109 | |
4110 | void | | 4110 | void |
4111 | uvmspace_spawn(struct lwp *l, vaddr_t start, vaddr_t end, bool topdown) | | 4111 | uvmspace_spawn(struct lwp *l, vaddr_t start, vaddr_t end, bool topdown) |
4112 | { | | 4112 | { |
4113 | struct proc *p = l->l_proc; | | 4113 | struct proc *p = l->l_proc; |
4114 | struct vmspace *nvm; | | 4114 | struct vmspace *nvm; |
4115 | | | 4115 | |
4116 | #ifdef __HAVE_CPU_VMSPACE_EXEC | | 4116 | #ifdef __HAVE_CPU_VMSPACE_EXEC |
4117 | cpu_vmspace_exec(l, start, end); | | 4117 | cpu_vmspace_exec(l, start, end); |
4118 | #endif | | 4118 | #endif |
4119 | | | 4119 | |
4120 | nvm = uvmspace_alloc(start, end, topdown); | | 4120 | nvm = uvmspace_alloc(start, end, topdown); |
4121 | kpreempt_disable(); | | 4121 | kpreempt_disable(); |
4122 | p->p_vmspace = nvm; | | 4122 | p->p_vmspace = nvm; |
4123 | pmap_activate(l); | | 4123 | pmap_activate(l); |
4124 | kpreempt_enable(); | | 4124 | kpreempt_enable(); |
4125 | } | | 4125 | } |
4126 | | | 4126 | |
4127 | /* | | 4127 | /* |
4128 | * uvmspace_exec: the process wants to exec a new program | | 4128 | * uvmspace_exec: the process wants to exec a new program |
4129 | */ | | 4129 | */ |
4130 | | | 4130 | |
4131 | void | | 4131 | void |
4132 | uvmspace_exec(struct lwp *l, vaddr_t start, vaddr_t end, bool topdown) | | 4132 | uvmspace_exec(struct lwp *l, vaddr_t start, vaddr_t end, bool topdown) |
4133 | { | | 4133 | { |
4134 | struct proc *p = l->l_proc; | | 4134 | struct proc *p = l->l_proc; |
4135 | struct vmspace *nvm, *ovm = p->p_vmspace; | | 4135 | struct vmspace *nvm, *ovm = p->p_vmspace; |
4136 | struct vm_map *map; | | 4136 | struct vm_map *map; |
4137 | int flags; | | 4137 | int flags; |
4138 | | | 4138 | |
4139 | KASSERT(ovm != NULL); | | 4139 | KASSERT(ovm != NULL); |
4140 | #ifdef __HAVE_CPU_VMSPACE_EXEC | | 4140 | #ifdef __HAVE_CPU_VMSPACE_EXEC |
4141 | cpu_vmspace_exec(l, start, end); | | 4141 | cpu_vmspace_exec(l, start, end); |
4142 | #endif | | 4142 | #endif |
4143 | | | 4143 | |
4144 | map = &ovm->vm_map; | | 4144 | map = &ovm->vm_map; |
4145 | /* | | 4145 | /* |
4146 | * see if more than one process is using this vmspace... | | 4146 | * see if more than one process is using this vmspace... |
4147 | */ | | 4147 | */ |
4148 | | | 4148 | |
4149 | if (ovm->vm_refcnt == 1 | | 4149 | if (ovm->vm_refcnt == 1 |
4150 | && topdown == ((ovm->vm_map.flags & VM_MAP_TOPDOWN) != 0)) { | | 4150 | && topdown == ((ovm->vm_map.flags & VM_MAP_TOPDOWN) != 0)) { |
4151 | | | 4151 | |
4152 | /* | | 4152 | /* |
4153 | * if p is the only process using its vmspace then we can safely | | 4153 | * if p is the only process using its vmspace then we can safely |
4154 | * recycle that vmspace for the program that is being exec'd. | | 4154 | * recycle that vmspace for the program that is being exec'd. |
4155 | * But only if TOPDOWN matches the requested value for the new | | 4155 | * But only if TOPDOWN matches the requested value for the new |
4156 | * vm space! | | 4156 | * vm space! |
4157 | */ | | 4157 | */ |
4158 | | | 4158 | |
4159 | /* | | 4159 | /* |
4160 | * SYSV SHM semantics require us to kill all segments on an exec | | 4160 | * SYSV SHM semantics require us to kill all segments on an exec |
4161 | */ | | 4161 | */ |
4162 | if (uvm_shmexit && ovm->vm_shm) | | 4162 | if (uvm_shmexit && ovm->vm_shm) |
4163 | (*uvm_shmexit)(ovm); | | 4163 | (*uvm_shmexit)(ovm); |
4164 | | | 4164 | |
4165 | /* | | 4165 | /* |
4166 | * POSIX 1003.1b -- "lock future mappings" is revoked | | 4166 | * POSIX 1003.1b -- "lock future mappings" is revoked |
4167 | * when a process execs another program image. | | 4167 | * when a process execs another program image. |
4168 | */ | | 4168 | */ |
4169 | | | 4169 | |
4170 | map->flags &= ~VM_MAP_WIREFUTURE; | | 4170 | map->flags &= ~VM_MAP_WIREFUTURE; |
4171 | | | 4171 | |
4172 | /* | | 4172 | /* |
4173 | * now unmap the old program. | | 4173 | * now unmap the old program. |
4174 | * | | 4174 | * |
4175 | * XXX set VM_MAP_DYING for the duration, so pmap_update() | | 4175 | * XXX set VM_MAP_DYING for the duration, so pmap_update() |
4176 | * is not called until the pmap has been totally cleared out | | 4176 | * is not called until the pmap has been totally cleared out |
4177 | * after pmap_remove_all(), or it can confuse some pmap | | 4177 | * after pmap_remove_all(), or it can confuse some pmap |
4178 | * implementations. it would be nice to handle this by | | 4178 | * implementations. it would be nice to handle this by |
4179 | * deferring the pmap_update() while it is known the address | | 4179 | * deferring the pmap_update() while it is known the address |
4180 | * space is not visible to any user LWP other than curlwp, | | 4180 | * space is not visible to any user LWP other than curlwp, |
4181 | * but there isn't an elegant way of inferring that right | | 4181 | * but there isn't an elegant way of inferring that right |
4182 | * now. | | 4182 | * now. |
4183 | */ | | 4183 | */ |
4184 | | | 4184 | |
4185 | flags = pmap_remove_all(map->pmap) ? UVM_FLAG_VAONLY : 0; | | 4185 | flags = pmap_remove_all(map->pmap) ? UVM_FLAG_VAONLY : 0; |
4186 | map->flags |= VM_MAP_DYING; | | 4186 | map->flags |= VM_MAP_DYING; |
4187 | uvm_unmap1(map, vm_map_min(map), vm_map_max(map), flags); | | 4187 | uvm_unmap1(map, vm_map_min(map), vm_map_max(map), flags); |
4188 | map->flags &= ~VM_MAP_DYING; | | 4188 | map->flags &= ~VM_MAP_DYING; |
4189 | pmap_update(map->pmap); | | 4189 | pmap_update(map->pmap); |
4190 | KASSERT(map->header.prev == &map->header); | | 4190 | KASSERT(map->header.prev == &map->header); |
4191 | KASSERT(map->nentries == 0); | | 4191 | KASSERT(map->nentries == 0); |
4192 | | | 4192 | |
4193 | /* | | 4193 | /* |
4194 | * resize the map | | 4194 | * resize the map |
4195 | */ | | 4195 | */ |
4196 | | | 4196 | |
4197 | vm_map_setmin(map, start); | | 4197 | vm_map_setmin(map, start); |
4198 | vm_map_setmax(map, end); | | 4198 | vm_map_setmax(map, end); |
4199 | } else { | | 4199 | } else { |
4200 | | | 4200 | |
4201 | /* | | 4201 | /* |
4202 | * p's vmspace is being shared, so we can't reuse it for p since | | 4202 | * p's vmspace is being shared, so we can't reuse it for p since |
4203 | * it is still being used for others. allocate a new vmspace | | 4203 | * it is still being used for others. allocate a new vmspace |
4204 | * for p | | 4204 | * for p |
4205 | */ | | 4205 | */ |
4206 | | | 4206 | |
4207 | nvm = uvmspace_alloc(start, end, topdown); | | 4207 | nvm = uvmspace_alloc(start, end, topdown); |
4208 | | | 4208 | |
4209 | /* | | 4209 | /* |
4210 | * install new vmspace and drop our ref to the old one. | | 4210 | * install new vmspace and drop our ref to the old one. |
4211 | */ | | 4211 | */ |
4212 | | | 4212 | |
4213 | kpreempt_disable(); | | 4213 | kpreempt_disable(); |
4214 | pmap_deactivate(l); | | 4214 | pmap_deactivate(l); |
4215 | p->p_vmspace = nvm; | | 4215 | p->p_vmspace = nvm; |
4216 | pmap_activate(l); | | 4216 | pmap_activate(l); |
4217 | kpreempt_enable(); | | 4217 | kpreempt_enable(); |
4218 | | | 4218 | |
4219 | uvmspace_free(ovm); | | 4219 | uvmspace_free(ovm); |
4220 | } | | 4220 | } |
4221 | } | | 4221 | } |
4222 | | | 4222 | |
4223 | /* | | 4223 | /* |
4224 | * uvmspace_addref: add a reference to a vmspace. | | 4224 | * uvmspace_addref: add a reference to a vmspace. |
4225 | */ | | 4225 | */ |
4226 | | | 4226 | |
4227 | void | | 4227 | void |
4228 | uvmspace_addref(struct vmspace *vm) | | 4228 | uvmspace_addref(struct vmspace *vm) |
4229 | { | | 4229 | { |
4230 | | | 4230 | |
4231 | KASSERT((vm->vm_map.flags & VM_MAP_DYING) == 0); | | 4231 | KASSERT((vm->vm_map.flags & VM_MAP_DYING) == 0); |
4232 | KASSERT(vm->vm_refcnt > 0); | | 4232 | KASSERT(vm->vm_refcnt > 0); |
4233 | atomic_inc_uint(&vm->vm_refcnt); | | 4233 | atomic_inc_uint(&vm->vm_refcnt); |
4234 | } | | 4234 | } |
4235 | | | 4235 | |
4236 | /* | | 4236 | /* |
4237 | * uvmspace_free: free a vmspace data structure | | 4237 | * uvmspace_free: free a vmspace data structure |
4238 | */ | | 4238 | */ |
4239 | | | 4239 | |
4240 | void | | 4240 | void |
4241 | uvmspace_free(struct vmspace *vm) | | 4241 | uvmspace_free(struct vmspace *vm) |
4242 | { | | 4242 | { |
4243 | struct vm_map_entry *dead_entries; | | 4243 | struct vm_map_entry *dead_entries; |
4244 | struct vm_map *map = &vm->vm_map; | | 4244 | struct vm_map *map = &vm->vm_map; |
4245 | int flags; | | 4245 | int flags; |
4246 | | | 4246 | |
4247 | UVMHIST_FUNC("uvmspace_free"); UVMHIST_CALLED(maphist); | | 4247 | UVMHIST_FUNC("uvmspace_free"); UVMHIST_CALLED(maphist); |
4248 | | | 4248 | |
4249 | UVMHIST_LOG(maphist,"(vm=%#jx) ref=%jd", (uintptr_t)vm, vm->vm_refcnt, | | 4249 | UVMHIST_LOG(maphist,"(vm=%#jx) ref=%jd", (uintptr_t)vm, vm->vm_refcnt, |
4250 | 0, 0); | | 4250 | 0, 0); |
4251 | if (atomic_dec_uint_nv(&vm->vm_refcnt) > 0) | | 4251 | if (atomic_dec_uint_nv(&vm->vm_refcnt) > 0) |
4252 | return; | | 4252 | return; |
4253 | | | 4253 | |
4254 | /* | | 4254 | /* |
4255 | * at this point, there should be no other references to the map. | | 4255 | * at this point, there should be no other references to the map. |
4256 | * delete all of the mappings, then destroy the pmap. | | 4256 | * delete all of the mappings, then destroy the pmap. |
4257 | */ | | 4257 | */ |
4258 | | | 4258 | |
4259 | map->flags |= VM_MAP_DYING; | | 4259 | map->flags |= VM_MAP_DYING; |
4260 | flags = pmap_remove_all(map->pmap) ? UVM_FLAG_VAONLY : 0; | | 4260 | flags = pmap_remove_all(map->pmap) ? UVM_FLAG_VAONLY : 0; |
4261 | | | 4261 | |
4262 | /* Get rid of any SYSV shared memory segments. */ | | 4262 | /* Get rid of any SYSV shared memory segments. */ |
4263 | if (uvm_shmexit && vm->vm_shm != NULL) | | 4263 | if (uvm_shmexit && vm->vm_shm != NULL) |
4264 | (*uvm_shmexit)(vm); | | 4264 | (*uvm_shmexit)(vm); |
4265 | | | 4265 | |
4266 | if (map->nentries) { | | 4266 | if (map->nentries) { |
4267 | uvm_unmap_remove(map, vm_map_min(map), vm_map_max(map), | | 4267 | uvm_unmap_remove(map, vm_map_min(map), vm_map_max(map), |
4268 | &dead_entries, flags); | | 4268 | &dead_entries, flags); |
4269 | if (dead_entries != NULL) | | 4269 | if (dead_entries != NULL) |
4270 | uvm_unmap_detach(dead_entries, 0); | | 4270 | uvm_unmap_detach(dead_entries, 0); |
4271 | } | | 4271 | } |
4272 | KASSERT(map->nentries == 0); | | 4272 | KASSERT(map->nentries == 0); |
4273 | KASSERT(map->size == 0); | | 4273 | KASSERT(map->size == 0); |
4274 | | | 4274 | |
4275 | mutex_destroy(&map->misc_lock); | | 4275 | mutex_destroy(&map->misc_lock); |
4276 | rw_destroy(&map->lock); | | 4276 | rw_destroy(&map->lock); |
4277 | cv_destroy(&map->cv); | | 4277 | cv_destroy(&map->cv); |
4278 | pmap_destroy(map->pmap); | | 4278 | pmap_destroy(map->pmap); |
4279 | pool_cache_put(&uvm_vmspace_cache, vm); | | 4279 | pool_cache_put(&uvm_vmspace_cache, vm); |
4280 | } | | 4280 | } |
4281 | | | 4281 | |
4282 | static struct vm_map_entry * | | 4282 | static struct vm_map_entry * |
4283 | uvm_mapent_clone(struct vm_map *new_map, struct vm_map_entry *old_entry, | | 4283 | uvm_mapent_clone(struct vm_map *new_map, struct vm_map_entry *old_entry, |
4284 | int flags) | | 4284 | int flags) |
4285 | { | | 4285 | { |
4286 | struct vm_map_entry *new_entry; | | 4286 | struct vm_map_entry *new_entry; |
4287 | | | 4287 | |
4288 | new_entry = uvm_mapent_alloc(new_map, 0); | | 4288 | new_entry = uvm_mapent_alloc(new_map, 0); |
4289 | /* old_entry -> new_entry */ | | 4289 | /* old_entry -> new_entry */ |
4290 | uvm_mapent_copy(old_entry, new_entry); | | 4290 | uvm_mapent_copy(old_entry, new_entry); |
4291 | | | 4291 | |
4292 | /* new pmap has nothing wired in it */ | | 4292 | /* new pmap has nothing wired in it */ |
4293 | new_entry->wired_count = 0; | | 4293 | new_entry->wired_count = 0; |
4294 | | | 4294 | |
4295 | /* | | 4295 | /* |
4296 | * gain reference to object backing the map (can't | | 4296 | * gain reference to object backing the map (can't |
4297 | * be a submap, already checked this case). | | 4297 | * be a submap, already checked this case). |
4298 | */ | | 4298 | */ |
4299 | | | 4299 | |
4300 | if (new_entry->aref.ar_amap) | | 4300 | if (new_entry->aref.ar_amap) |
4301 | uvm_map_reference_amap(new_entry, flags); | | 4301 | uvm_map_reference_amap(new_entry, flags); |
4302 | | | 4302 | |
4303 | if (new_entry->object.uvm_obj && | | 4303 | if (new_entry->object.uvm_obj && |
4304 | new_entry->object.uvm_obj->pgops->pgo_reference) | | 4304 | new_entry->object.uvm_obj->pgops->pgo_reference) |
4305 | new_entry->object.uvm_obj->pgops->pgo_reference( | | 4305 | new_entry->object.uvm_obj->pgops->pgo_reference( |
4306 | new_entry->object.uvm_obj); | | 4306 | new_entry->object.uvm_obj); |
4307 | | | 4307 | |
4308 | /* insert entry at end of new_map's entry list */ | | 4308 | /* insert entry at end of new_map's entry list */ |
4309 | uvm_map_entry_link(new_map, new_map->header.prev, | | 4309 | uvm_map_entry_link(new_map, new_map->header.prev, |
4310 | new_entry); | | 4310 | new_entry); |
4311 | | | 4311 | |
4312 | return new_entry; | | 4312 | return new_entry; |
4313 | } | | 4313 | } |
4314 | | | 4314 | |
4315 | /* | | 4315 | /* |
4316 | * share the mapping: this means we want the old and | | 4316 | * share the mapping: this means we want the old and |
4317 | * new entries to share amaps and backing objects. | | 4317 | * new entries to share amaps and backing objects. |
4318 | */ | | 4318 | */ |
4319 | static void | | 4319 | static void |
4320 | uvm_mapent_forkshared(struct vm_map *new_map, struct vm_map *old_map, | | 4320 | uvm_mapent_forkshared(struct vm_map *new_map, struct vm_map *old_map, |
4321 | struct vm_map_entry *old_entry) | | 4321 | struct vm_map_entry *old_entry) |
4322 | { | | 4322 | { |
4323 | /* | | 4323 | /* |
4324 | * if the old_entry needs a new amap (due to prev fork) | | 4324 | * if the old_entry needs a new amap (due to prev fork) |
4325 | * then we need to allocate it now so that we have | | 4325 | * then we need to allocate it now so that we have |
4326 | * something we own to share with the new_entry. [in | | 4326 | * something we own to share with the new_entry. [in |
4327 | * other words, we need to clear needs_copy] | | 4327 | * other words, we need to clear needs_copy] |
4328 | */ | | 4328 | */ |
4329 | | | 4329 | |
4330 | if (UVM_ET_ISNEEDSCOPY(old_entry)) { | | 4330 | if (UVM_ET_ISNEEDSCOPY(old_entry)) { |
4331 | /* get our own amap, clears needs_copy */ | | 4331 | /* get our own amap, clears needs_copy */ |
4332 | amap_copy(old_map, old_entry, AMAP_COPY_NOCHUNK, | | 4332 | amap_copy(old_map, old_entry, AMAP_COPY_NOCHUNK, |
4333 | 0, 0); | | 4333 | 0, 0); |
4334 | /* XXXCDC: WAITOK??? */ | | 4334 | /* XXXCDC: WAITOK??? */ |
4335 | } | | 4335 | } |
4336 | | | 4336 | |
4337 | uvm_mapent_clone(new_map, old_entry, AMAP_SHARED); | | 4337 | uvm_mapent_clone(new_map, old_entry, AMAP_SHARED); |
4338 | } | | 4338 | } |
4339 | | | 4339 | |
4340 | | | 4340 | |
4341 | static void | | 4341 | static void |
4342 | uvm_mapent_forkcopy(struct vm_map *new_map, struct vm_map *old_map, | | 4342 | uvm_mapent_forkcopy(struct vm_map *new_map, struct vm_map *old_map, |
4343 | struct vm_map_entry *old_entry) | | 4343 | struct vm_map_entry *old_entry) |
4344 | { | | 4344 | { |
4345 | struct vm_map_entry *new_entry; | | 4345 | struct vm_map_entry *new_entry; |
4346 | | | 4346 | |
4347 | /* | | 4347 | /* |
4348 | * copy-on-write the mapping (using mmap's | | 4348 | * copy-on-write the mapping (using mmap's |
4349 | * MAP_PRIVATE semantics) | | 4349 | * MAP_PRIVATE semantics) |
4350 | * | | 4350 | * |
4351 | * allocate new_entry, adjust reference counts. | | 4351 | * allocate new_entry, adjust reference counts. |
4352 | * (note that new references are read-only). | | 4352 | * (note that new references are read-only). |
4353 | */ | | 4353 | */ |
4354 | | | 4354 | |
4355 | new_entry = uvm_mapent_clone(new_map, old_entry, 0); | | 4355 | new_entry = uvm_mapent_clone(new_map, old_entry, 0); |
4356 | | | 4356 | |
4357 | new_entry->etype |= | | 4357 | new_entry->etype |= |
4358 | (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY); | | 4358 | (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY); |
4359 | | | 4359 | |
4360 | /* | | 4360 | /* |
4361 | * the new entry will need an amap. it will either | | 4361 | * the new entry will need an amap. it will either |
4362 | * need to be copied from the old entry or created | | 4362 | * need to be copied from the old entry or created |
4363 | * from scratch (if the old entry does not have an | | 4363 | * from scratch (if the old entry does not have an |
4364 | * amap). can we defer this process until later | | 4364 | * amap). can we defer this process until later |
4365 | * (by setting "needs_copy") or do we need to copy | | 4365 | * (by setting "needs_copy") or do we need to copy |
4366 | * the amap now? | | 4366 | * the amap now? |
4367 | * | | 4367 | * |
4368 | * we must copy the amap now if any of the following | | 4368 | * we must copy the amap now if any of the following |
4369 | * conditions hold: | | 4369 | * conditions hold: |
4370 | * 1. the old entry has an amap and that amap is | | 4370 | * 1. the old entry has an amap and that amap is |
4371 | * being shared. this means that the old (parent) | | 4371 | * being shared. this means that the old (parent) |
4372 | * process is sharing the amap with another | | 4372 | * process is sharing the amap with another |
4373 | * process. if we do not clear needs_copy here | | 4373 | * process. if we do not clear needs_copy here |
4374 | * we will end up in a situation where both the | | 4374 | * we will end up in a situation where both the |
4375 | * parent and child process are refering to the | | 4375 | * parent and child process are refering to the |
4376 | * same amap with "needs_copy" set. if the | | 4376 | * same amap with "needs_copy" set. if the |
4377 | * parent write-faults, the fault routine will | | 4377 | * parent write-faults, the fault routine will |
4378 | * clear "needs_copy" in the parent by allocating | | 4378 | * clear "needs_copy" in the parent by allocating |
4379 | * a new amap. this is wrong because the | | 4379 | * a new amap. this is wrong because the |
4380 | * parent is supposed to be sharing the old amap | | 4380 | * parent is supposed to be sharing the old amap |
4381 | * and the new amap will break that. | | 4381 | * and the new amap will break that. |
4382 | * | | 4382 | * |
4383 | * 2. if the old entry has an amap and a non-zero | | 4383 | * 2. if the old entry has an amap and a non-zero |
4384 | * wire count then we are going to have to call | | 4384 | * wire count then we are going to have to call |
4385 | * amap_cow_now to avoid page faults in the | | 4385 | * amap_cow_now to avoid page faults in the |
4386 | * parent process. since amap_cow_now requires | | 4386 | * parent process. since amap_cow_now requires |
4387 | * "needs_copy" to be clear we might as well | | 4387 | * "needs_copy" to be clear we might as well |
4388 | * clear it here as well. | | 4388 | * clear it here as well. |
4389 | * | | 4389 | * |
4390 | */ | | 4390 | */ |
4391 | | | 4391 | |
4392 | if (old_entry->aref.ar_amap != NULL) { | | 4392 | if (old_entry->aref.ar_amap != NULL) { |
4393 | if ((amap_flags(old_entry->aref.ar_amap) & AMAP_SHARED) != 0 || | | 4393 | if ((amap_flags(old_entry->aref.ar_amap) & AMAP_SHARED) != 0 || |
4394 | VM_MAPENT_ISWIRED(old_entry)) { | | 4394 | VM_MAPENT_ISWIRED(old_entry)) { |
4395 | | | 4395 | |
4396 | amap_copy(new_map, new_entry, | | 4396 | amap_copy(new_map, new_entry, |
4397 | AMAP_COPY_NOCHUNK, 0, 0); | | 4397 | AMAP_COPY_NOCHUNK, 0, 0); |
4398 | /* XXXCDC: M_WAITOK ... ok? */ | | 4398 | /* XXXCDC: M_WAITOK ... ok? */ |
4399 | } | | 4399 | } |
4400 | } | | 4400 | } |
4401 | | | 4401 | |
4402 | /* | | 4402 | /* |
4403 | * if the parent's entry is wired down, then the | | 4403 | * if the parent's entry is wired down, then the |
4404 | * parent process does not want page faults on | | 4404 | * parent process does not want page faults on |
4405 | * access to that memory. this means that we | | 4405 | * access to that memory. this means that we |
4406 | * cannot do copy-on-write because we can't write | | 4406 | * cannot do copy-on-write because we can't write |
4407 | * protect the old entry. in this case we | | 4407 | * protect the old entry. in this case we |
4408 | * resolve all copy-on-write faults now, using | | 4408 | * resolve all copy-on-write faults now, using |
4409 | * amap_cow_now. note that we have already | | 4409 | * amap_cow_now. note that we have already |
4410 | * allocated any needed amap (above). | | 4410 | * allocated any needed amap (above). |
4411 | */ | | 4411 | */ |
4412 | | | 4412 | |
4413 | if (VM_MAPENT_ISWIRED(old_entry)) { | | 4413 | if (VM_MAPENT_ISWIRED(old_entry)) { |
4414 | | | 4414 | |
4415 | /* | | 4415 | /* |
4416 | * resolve all copy-on-write faults now | | 4416 | * resolve all copy-on-write faults now |
4417 | * (note that there is nothing to do if | | 4417 | * (note that there is nothing to do if |
4418 | * the old mapping does not have an amap). | | 4418 | * the old mapping does not have an amap). |
4419 | */ | | 4419 | */ |
4420 | if (old_entry->aref.ar_amap) | | 4420 | if (old_entry->aref.ar_amap) |
4421 | amap_cow_now(new_map, new_entry); | | 4421 | amap_cow_now(new_map, new_entry); |
4422 | | | 4422 | |
4423 | } else { | | 4423 | } else { |
4424 | /* | | 4424 | /* |
4425 | * setup mappings to trigger copy-on-write faults | | 4425 | * setup mappings to trigger copy-on-write faults |
4426 | * we must write-protect the parent if it has | | 4426 | * we must write-protect the parent if it has |
4427 | * an amap and it is not already "needs_copy"... | | 4427 | * an amap and it is not already "needs_copy"... |
4428 | * if it is already "needs_copy" then the parent | | 4428 | * if it is already "needs_copy" then the parent |
4429 | * has already been write-protected by a previous | | 4429 | * has already been write-protected by a previous |
4430 | * fork operation. | | 4430 | * fork operation. |
4431 | */ | | 4431 | */ |
4432 | if (old_entry->aref.ar_amap && | | 4432 | if (old_entry->aref.ar_amap && |
4433 | !UVM_ET_ISNEEDSCOPY(old_entry)) { | | 4433 | !UVM_ET_ISNEEDSCOPY(old_entry)) { |
4434 | if (old_entry->max_protection & VM_PROT_WRITE) { | | 4434 | if (old_entry->max_protection & VM_PROT_WRITE) { |
4435 | #ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */ | | 4435 | #ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */ |
4436 | uvm_map_lock_entry(old_entry, RW_WRITER); | | 4436 | uvm_map_lock_entry(old_entry, RW_WRITER); |
4437 | #else | | 4437 | #else |
4438 | uvm_map_lock_entry(old_entry, RW_READER); | | 4438 | uvm_map_lock_entry(old_entry, RW_READER); |
4439 | #endif | | 4439 | #endif |
4440 | pmap_protect(old_map->pmap, | | 4440 | pmap_protect(old_map->pmap, |
4441 | old_entry->start, old_entry->end, | | 4441 | old_entry->start, old_entry->end, |
4442 | old_entry->protection & ~VM_PROT_WRITE); | | 4442 | old_entry->protection & ~VM_PROT_WRITE); |
4443 | uvm_map_unlock_entry(old_entry); | | 4443 | uvm_map_unlock_entry(old_entry); |
4444 | } | | 4444 | } |
4445 | old_entry->etype |= UVM_ET_NEEDSCOPY; | | 4445 | old_entry->etype |= UVM_ET_NEEDSCOPY; |
4446 | } | | 4446 | } |
4447 | } | | 4447 | } |
4448 | } | | 4448 | } |
4449 | | | 4449 | |
4450 | /* | | 4450 | /* |
4451 | * zero the mapping: the new entry will be zero initialized | | 4451 | * zero the mapping: the new entry will be zero initialized |
4452 | */ | | 4452 | */ |
4453 | static void | | 4453 | static void |
4454 | uvm_mapent_forkzero(struct vm_map *new_map, struct vm_map *old_map, | | 4454 | uvm_mapent_forkzero(struct vm_map *new_map, struct vm_map *old_map, |
4455 | struct vm_map_entry *old_entry) | | 4455 | struct vm_map_entry *old_entry) |
4456 | { | | 4456 | { |
4457 | struct vm_map_entry *new_entry; | | 4457 | struct vm_map_entry *new_entry; |
4458 | | | 4458 | |
4459 | new_entry = uvm_mapent_clone(new_map, old_entry, 0); | | 4459 | new_entry = uvm_mapent_clone(new_map, old_entry, 0); |
4460 | | | 4460 | |
4461 | new_entry->etype |= | | 4461 | new_entry->etype |= |
4462 | (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY); | | 4462 | (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY); |
4463 | | | 4463 | |
4464 | if (new_entry->aref.ar_amap) { | | 4464 | if (new_entry->aref.ar_amap) { |
4465 | uvm_map_unreference_amap(new_entry, 0); | | 4465 | uvm_map_unreference_amap(new_entry, 0); |
4466 | new_entry->aref.ar_pageoff = 0; | | 4466 | new_entry->aref.ar_pageoff = 0; |
4467 | new_entry->aref.ar_amap = NULL; | | 4467 | new_entry->aref.ar_amap = NULL; |
4468 | } | | 4468 | } |
4469 | | | 4469 | |
4470 | if (UVM_ET_ISOBJ(new_entry)) { | | 4470 | if (UVM_ET_ISOBJ(new_entry)) { |
4471 | if (new_entry->object.uvm_obj->pgops->pgo_detach) | | 4471 | if (new_entry->object.uvm_obj->pgops->pgo_detach) |
4472 | new_entry->object.uvm_obj->pgops->pgo_detach( | | 4472 | new_entry->object.uvm_obj->pgops->pgo_detach( |
4473 | new_entry->object.uvm_obj); | | 4473 | new_entry->object.uvm_obj); |
4474 | new_entry->object.uvm_obj = NULL; | | 4474 | new_entry->object.uvm_obj = NULL; |
4475 | new_entry->etype &= ~UVM_ET_OBJ; | | 4475 | new_entry->etype &= ~UVM_ET_OBJ; |
4476 | } | | 4476 | } |
4477 | } | | 4477 | } |
4478 | | | 4478 | |
4479 | /* | | 4479 | /* |
4480 | * F O R K - m a i n e n t r y p o i n t | | 4480 | * F O R K - m a i n e n t r y p o i n t |
4481 | */ | | 4481 | */ |
4482 | /* | | 4482 | /* |
4483 | * uvmspace_fork: fork a process' main map | | 4483 | * uvmspace_fork: fork a process' main map |
4484 | * | | 4484 | * |
4485 | * => create a new vmspace for child process from parent. | | 4485 | * => create a new vmspace for child process from parent. |
4486 | * => parent's map must not be locked. | | 4486 | * => parent's map must not be locked. |
4487 | */ | | 4487 | */ |
4488 | | | 4488 | |
4489 | struct vmspace * | | 4489 | struct vmspace * |
4490 | uvmspace_fork(struct vmspace *vm1) | | 4490 | uvmspace_fork(struct vmspace *vm1) |
4491 | { | | 4491 | { |
4492 | struct vmspace *vm2; | | 4492 | struct vmspace *vm2; |
4493 | struct vm_map *old_map = &vm1->vm_map; | | 4493 | struct vm_map *old_map = &vm1->vm_map; |
4494 | struct vm_map *new_map; | | 4494 | struct vm_map *new_map; |
4495 | struct vm_map_entry *old_entry; | | 4495 | struct vm_map_entry *old_entry; |
4496 | UVMHIST_FUNC("uvmspace_fork"); UVMHIST_CALLED(maphist); | | 4496 | UVMHIST_FUNC("uvmspace_fork"); UVMHIST_CALLED(maphist); |
4497 | | | 4497 | |
4498 | vm_map_lock(old_map); | | 4498 | vm_map_lock(old_map); |
4499 | | | 4499 | |
4500 | vm2 = uvmspace_alloc(vm_map_min(old_map), vm_map_max(old_map), | | 4500 | vm2 = uvmspace_alloc(vm_map_min(old_map), vm_map_max(old_map), |
4501 | vm1->vm_map.flags & VM_MAP_TOPDOWN); | | 4501 | vm1->vm_map.flags & VM_MAP_TOPDOWN); |
4502 | memcpy(&vm2->vm_startcopy, &vm1->vm_startcopy, | | 4502 | memcpy(&vm2->vm_startcopy, &vm1->vm_startcopy, |
4503 | (char *) (vm1 + 1) - (char *) &vm1->vm_startcopy); | | 4503 | (char *) (vm1 + 1) - (char *) &vm1->vm_startcopy); |
4504 | new_map = &vm2->vm_map; /* XXX */ | | 4504 | new_map = &vm2->vm_map; /* XXX */ |
4505 | | | 4505 | |
4506 | old_entry = old_map->header.next; | | 4506 | old_entry = old_map->header.next; |
4507 | new_map->size = old_map->size; | | 4507 | new_map->size = old_map->size; |
4508 | | | 4508 | |
4509 | /* | | 4509 | /* |
4510 | * go entry-by-entry | | 4510 | * go entry-by-entry |
4511 | */ | | 4511 | */ |
4512 | | | 4512 | |
4513 | while (old_entry != &old_map->header) { | | 4513 | while (old_entry != &old_map->header) { |
4514 | | | 4514 | |
4515 | /* | | 4515 | /* |
4516 | * first, some sanity checks on the old entry | | 4516 | * first, some sanity checks on the old entry |
4517 | */ | | 4517 | */ |
4518 | | | 4518 | |
4519 | KASSERT(!UVM_ET_ISSUBMAP(old_entry)); | | 4519 | KASSERT(!UVM_ET_ISSUBMAP(old_entry)); |
4520 | KASSERT(UVM_ET_ISCOPYONWRITE(old_entry) || | | 4520 | KASSERT(UVM_ET_ISCOPYONWRITE(old_entry) || |
4521 | !UVM_ET_ISNEEDSCOPY(old_entry)); | | 4521 | !UVM_ET_ISNEEDSCOPY(old_entry)); |
4522 | | | 4522 | |
4523 | switch (old_entry->inheritance) { | | 4523 | switch (old_entry->inheritance) { |
4524 | case MAP_INHERIT_NONE: | | 4524 | case MAP_INHERIT_NONE: |
4525 | /* | | 4525 | /* |
4526 | * drop the mapping, modify size | | 4526 | * drop the mapping, modify size |
4527 | */ | | 4527 | */ |
4528 | new_map->size -= old_entry->end - old_entry->start; | | 4528 | new_map->size -= old_entry->end - old_entry->start; |
4529 | break; | | 4529 | break; |
4530 | | | 4530 | |
4531 | case MAP_INHERIT_SHARE: | | 4531 | case MAP_INHERIT_SHARE: |
4532 | uvm_mapent_forkshared(new_map, old_map, old_entry); | | 4532 | uvm_mapent_forkshared(new_map, old_map, old_entry); |
4533 | break; | | 4533 | break; |
4534 | | | 4534 | |
4535 | case MAP_INHERIT_COPY: | | 4535 | case MAP_INHERIT_COPY: |
4536 | uvm_mapent_forkcopy(new_map, old_map, old_entry); | | 4536 | uvm_mapent_forkcopy(new_map, old_map, old_entry); |
4537 | break; | | 4537 | break; |
4538 | | | 4538 | |
4539 | case MAP_INHERIT_ZERO: | | 4539 | case MAP_INHERIT_ZERO: |
4540 | uvm_mapent_forkzero(new_map, old_map, old_entry); | | 4540 | uvm_mapent_forkzero(new_map, old_map, old_entry); |
4541 | break; | | 4541 | break; |
4542 | default: | | 4542 | default: |
4543 | KASSERT(0); | | 4543 | KASSERT(0); |
4544 | break; | | 4544 | break; |
4545 | } | | 4545 | } |
4546 | old_entry = old_entry->next; | | 4546 | old_entry = old_entry->next; |
4547 | } | | 4547 | } |
4548 | | | 4548 | |
4549 | pmap_update(old_map->pmap); | | 4549 | pmap_update(old_map->pmap); |
4550 | vm_map_unlock(old_map); | | 4550 | vm_map_unlock(old_map); |
4551 | | | 4551 | |
4552 | if (uvm_shmfork && vm1->vm_shm) | | 4552 | if (uvm_shmfork && vm1->vm_shm) |
4553 | (*uvm_shmfork)(vm1, vm2); | | 4553 | (*uvm_shmfork)(vm1, vm2); |
4554 | | | 4554 | |
4555 | #ifdef PMAP_FORK | | 4555 | #ifdef PMAP_FORK |
4556 | pmap_fork(vm1->vm_map.pmap, vm2->vm_map.pmap); | | 4556 | pmap_fork(vm1->vm_map.pmap, vm2->vm_map.pmap); |
4557 | #endif | | 4557 | #endif |
4558 | | | 4558 | |
4559 | UVMHIST_LOG(maphist,"<- done",0,0,0,0); | | 4559 | UVMHIST_LOG(maphist,"<- done",0,0,0,0); |
4560 | return (vm2); | | 4560 | return (vm2); |
4561 | } | | 4561 | } |
4562 | | | 4562 | |
4563 | | | 4563 | |
4564 | /* | | 4564 | /* |
4565 | * uvm_mapent_trymerge: try to merge an entry with its neighbors. | | 4565 | * uvm_mapent_trymerge: try to merge an entry with its neighbors. |
4566 | * | | 4566 | * |
4567 | * => called with map locked. | | 4567 | * => called with map locked. |
4568 | * => return non zero if successfully merged. | | 4568 | * => return non zero if successfully merged. |
4569 | */ | | 4569 | */ |
4570 | | | 4570 | |
4571 | int | | 4571 | int |
4572 | uvm_mapent_trymerge(struct vm_map *map, struct vm_map_entry *entry, int flags) | | 4572 | uvm_mapent_trymerge(struct vm_map *map, struct vm_map_entry *entry, int flags) |
4573 | { | | 4573 | { |
4574 | struct uvm_object *uobj; | | 4574 | struct uvm_object *uobj; |
4575 | struct vm_map_entry *next; | | 4575 | struct vm_map_entry *next; |
4576 | struct vm_map_entry *prev; | | 4576 | struct vm_map_entry *prev; |
4577 | vsize_t size; | | 4577 | vsize_t size; |
4578 | int merged = 0; | | 4578 | int merged = 0; |
4579 | bool copying; | | 4579 | bool copying; |
4580 | int newetype; | | 4580 | int newetype; |
4581 | | | 4581 | |
4582 | if (entry->aref.ar_amap != NULL) { | | 4582 | if (entry->aref.ar_amap != NULL) { |
4583 | return 0; | | 4583 | return 0; |
4584 | } | | 4584 | } |
4585 | if ((entry->flags & UVM_MAP_NOMERGE) != 0) { | | 4585 | if ((entry->flags & UVM_MAP_NOMERGE) != 0) { |
4586 | return 0; | | 4586 | return 0; |
4587 | } | | 4587 | } |
4588 | | | 4588 | |
4589 | uobj = entry->object.uvm_obj; | | 4589 | uobj = entry->object.uvm_obj; |
4590 | size = entry->end - entry->start; | | 4590 | size = entry->end - entry->start; |
4591 | copying = (flags & UVM_MERGE_COPYING) != 0; | | 4591 | copying = (flags & UVM_MERGE_COPYING) != 0; |
4592 | newetype = copying ? (entry->etype & ~UVM_ET_NEEDSCOPY) : entry->etype; | | 4592 | newetype = copying ? (entry->etype & ~UVM_ET_NEEDSCOPY) : entry->etype; |
4593 | | | 4593 | |
4594 | next = entry->next; | | 4594 | next = entry->next; |
4595 | if (next != &map->header && | | 4595 | if (next != &map->header && |
4596 | next->start == entry->end && | | 4596 | next->start == entry->end && |
4597 | ((copying && next->aref.ar_amap != NULL && | | 4597 | ((copying && next->aref.ar_amap != NULL && |
4598 | amap_refs(next->aref.ar_amap) == 1) || | | 4598 | amap_refs(next->aref.ar_amap) == 1) || |
4599 | (!copying && next->aref.ar_amap == NULL)) && | | 4599 | (!copying && next->aref.ar_amap == NULL)) && |
4600 | UVM_ET_ISCOMPATIBLE(next, newetype, | | 4600 | UVM_ET_ISCOMPATIBLE(next, newetype, |
4601 | uobj, entry->flags, entry->protection, | | 4601 | uobj, entry->flags, entry->protection, |
4602 | entry->max_protection, entry->inheritance, entry->advice, | | 4602 | entry->max_protection, entry->inheritance, entry->advice, |
4603 | entry->wired_count) && | | 4603 | entry->wired_count) && |
4604 | (uobj == NULL || entry->offset + size == next->offset)) { | | 4604 | (uobj == NULL || entry->offset + size == next->offset)) { |
4605 | int error; | | 4605 | int error; |
4606 | | | 4606 | |
4607 | if (copying) { | | 4607 | if (copying) { |
4608 | error = amap_extend(next, size, | | 4608 | error = amap_extend(next, size, |
4609 | AMAP_EXTEND_NOWAIT|AMAP_EXTEND_BACKWARDS); | | 4609 | AMAP_EXTEND_NOWAIT|AMAP_EXTEND_BACKWARDS); |
4610 | } else { | | 4610 | } else { |
4611 | error = 0; | | 4611 | error = 0; |
4612 | } | | 4612 | } |
4613 | if (error == 0) { | | 4613 | if (error == 0) { |
4614 | if (uobj) { | | 4614 | if (uobj) { |
4615 | if (uobj->pgops->pgo_detach) { | | 4615 | if (uobj->pgops->pgo_detach) { |
4616 | uobj->pgops->pgo_detach(uobj); | | 4616 | uobj->pgops->pgo_detach(uobj); |
4617 | } | | 4617 | } |
4618 | } | | 4618 | } |
4619 | | | 4619 | |
4620 | entry->end = next->end; | | 4620 | entry->end = next->end; |
4621 | clear_hints(map, next); | | 4621 | clear_hints(map, next); |
4622 | uvm_map_entry_unlink(map, next); | | 4622 | uvm_map_entry_unlink(map, next); |
4623 | if (copying) { | | 4623 | if (copying) { |
4624 | entry->aref = next->aref; | | 4624 | entry->aref = next->aref; |
4625 | entry->etype &= ~UVM_ET_NEEDSCOPY; | | 4625 | entry->etype &= ~UVM_ET_NEEDSCOPY; |
4626 | } | | 4626 | } |
4627 | uvm_map_check(map, "trymerge forwardmerge"); | | 4627 | uvm_map_check(map, "trymerge forwardmerge"); |
4628 | uvm_mapent_free(next); | | 4628 | uvm_mapent_free(next); |
4629 | merged++; | | 4629 | merged++; |
4630 | } | | 4630 | } |
4631 | } | | 4631 | } |
4632 | | | 4632 | |
4633 | prev = entry->prev; | | 4633 | prev = entry->prev; |
4634 | if (prev != &map->header && | | 4634 | if (prev != &map->header && |
4635 | prev->end == entry->start && | | 4635 | prev->end == entry->start && |
4636 | ((copying && !merged && prev->aref.ar_amap != NULL && | | 4636 | ((copying && !merged && prev->aref.ar_amap != NULL && |
4637 | amap_refs(prev->aref.ar_amap) == 1) || | | 4637 | amap_refs(prev->aref.ar_amap) == 1) || |
4638 | (!copying && prev->aref.ar_amap == NULL)) && | | 4638 | (!copying && prev->aref.ar_amap == NULL)) && |
4639 | UVM_ET_ISCOMPATIBLE(prev, newetype, | | 4639 | UVM_ET_ISCOMPATIBLE(prev, newetype, |
4640 | uobj, entry->flags, entry->protection, | | 4640 | uobj, entry->flags, entry->protection, |
4641 | entry->max_protection, entry->inheritance, entry->advice, | | 4641 | entry->max_protection, entry->inheritance, entry->advice, |
4642 | entry->wired_count) && | | 4642 | entry->wired_count) && |
4643 | (uobj == NULL || | | 4643 | (uobj == NULL || |
4644 | prev->offset + prev->end - prev->start == entry->offset)) { | | 4644 | prev->offset + prev->end - prev->start == entry->offset)) { |
4645 | int error; | | 4645 | int error; |
4646 | | | 4646 | |
4647 | if (copying) { | | 4647 | if (copying) { |
4648 | error = amap_extend(prev, size, | | 4648 | error = amap_extend(prev, size, |
4649 | AMAP_EXTEND_NOWAIT|AMAP_EXTEND_FORWARDS); | | 4649 | AMAP_EXTEND_NOWAIT|AMAP_EXTEND_FORWARDS); |
4650 | } else { | | 4650 | } else { |
4651 | error = 0; | | 4651 | error = 0; |
4652 | } | | 4652 | } |
4653 | if (error == 0) { | | 4653 | if (error == 0) { |
4654 | if (uobj) { | | 4654 | if (uobj) { |
4655 | if (uobj->pgops->pgo_detach) { | | 4655 | if (uobj->pgops->pgo_detach) { |
4656 | uobj->pgops->pgo_detach(uobj); | | 4656 | uobj->pgops->pgo_detach(uobj); |
4657 | } | | 4657 | } |
4658 | entry->offset = prev->offset; | | 4658 | entry->offset = prev->offset; |
4659 | } | | 4659 | } |
4660 | | | 4660 | |
4661 | entry->start = prev->start; | | 4661 | entry->start = prev->start; |
4662 | clear_hints(map, prev); | | 4662 | clear_hints(map, prev); |
4663 | uvm_map_entry_unlink(map, prev); | | 4663 | uvm_map_entry_unlink(map, prev); |
4664 | if (copying) { | | 4664 | if (copying) { |
4665 | entry->aref = prev->aref; | | 4665 | entry->aref = prev->aref; |
4666 | entry->etype &= ~UVM_ET_NEEDSCOPY; | | 4666 | entry->etype &= ~UVM_ET_NEEDSCOPY; |
4667 | } | | 4667 | } |
4668 | uvm_map_check(map, "trymerge backmerge"); | | 4668 | uvm_map_check(map, "trymerge backmerge"); |
4669 | uvm_mapent_free(prev); | | 4669 | uvm_mapent_free(prev); |
4670 | merged++; | | 4670 | merged++; |
4671 | } | | 4671 | } |
4672 | } | | 4672 | } |
4673 | | | 4673 | |
4674 | return merged; | | 4674 | return merged; |
4675 | } | | 4675 | } |
4676 | | | 4676 | |
4677 | /* | | 4677 | /* |
4678 | * uvm_map_setup: init map | | 4678 | * uvm_map_setup: init map |
4679 | * | | 4679 | * |
4680 | * => map must not be in service yet. | | 4680 | * => map must not be in service yet. |
4681 | */ | | 4681 | */ |
4682 | | | 4682 | |
4683 | void | | 4683 | void |
4684 | uvm_map_setup(struct vm_map *map, vaddr_t vmin, vaddr_t vmax, int flags) | | 4684 | uvm_map_setup(struct vm_map *map, vaddr_t vmin, vaddr_t vmax, int flags) |
4685 | { | | 4685 | { |
4686 | | | 4686 | |
4687 | rb_tree_init(&map->rb_tree, &uvm_map_tree_ops); | | 4687 | rb_tree_init(&map->rb_tree, &uvm_map_tree_ops); |
4688 | map->header.next = map->header.prev = &map->header; | | 4688 | map->header.next = map->header.prev = &map->header; |
4689 | map->nentries = 0; | | 4689 | map->nentries = 0; |
4690 | map->size = 0; | | 4690 | map->size = 0; |
4691 | map->ref_count = 1; | | 4691 | map->ref_count = 1; |
4692 | vm_map_setmin(map, vmin); | | 4692 | vm_map_setmin(map, vmin); |
4693 | vm_map_setmax(map, vmax); | | 4693 | vm_map_setmax(map, vmax); |
4694 | map->flags = flags; | | 4694 | map->flags = flags; |
4695 | map->first_free = &map->header; | | 4695 | map->first_free = &map->header; |
4696 | map->hint = &map->header; | | 4696 | map->hint = &map->header; |
4697 | map->timestamp = 0; | | 4697 | map->timestamp = 0; |
4698 | map->busy = NULL; | | 4698 | map->busy = NULL; |
4699 | | | 4699 | |
4700 | rw_init(&map->lock); | | 4700 | rw_init(&map->lock); |
4701 | cv_init(&map->cv, "vm_map"); | | 4701 | cv_init(&map->cv, "vm_map"); |
4702 | mutex_init(&map->misc_lock, MUTEX_DRIVER, IPL_NONE); | | 4702 | mutex_init(&map->misc_lock, MUTEX_DRIVER, IPL_NONE); |
4703 | } | | 4703 | } |
4704 | | | 4704 | |
4705 | /* | | 4705 | /* |
4706 | * U N M A P - m a i n e n t r y p o i n t | | 4706 | * U N M A P - m a i n e n t r y p o i n t |
4707 | */ | | 4707 | */ |
4708 | | | 4708 | |
4709 | /* | | 4709 | /* |
4710 | * uvm_unmap1: remove mappings from a vm_map (from "start" up to "stop") | | 4710 | * uvm_unmap1: remove mappings from a vm_map (from "start" up to "stop") |
4711 | * | | 4711 | * |
4712 | * => caller must check alignment and size | | 4712 | * => caller must check alignment and size |
4713 | * => map must be unlocked (we will lock it) | | 4713 | * => map must be unlocked (we will lock it) |
4714 | * => flags is UVM_FLAG_QUANTUM or 0. | | 4714 | * => flags is UVM_FLAG_QUANTUM or 0. |
4715 | */ | | 4715 | */ |
4716 | | | 4716 | |
4717 | void | | 4717 | void |
4718 | uvm_unmap1(struct vm_map *map, vaddr_t start, vaddr_t end, int flags) | | 4718 | uvm_unmap1(struct vm_map *map, vaddr_t start, vaddr_t end, int flags) |
4719 | { | | 4719 | { |
4720 | struct vm_map_entry *dead_entries; | | 4720 | struct vm_map_entry *dead_entries; |
4721 | UVMHIST_FUNC("uvm_unmap"); UVMHIST_CALLED(maphist); | | 4721 | UVMHIST_FUNC("uvm_unmap"); UVMHIST_CALLED(maphist); |
4722 | | | 4722 | |
4723 | KASSERTMSG(start < end, | | 4723 | KASSERTMSG(start < end, |
4724 | "%s: map %p: start %#jx < end %#jx", __func__, map, | | 4724 | "%s: map %p: start %#jx < end %#jx", __func__, map, |
4725 | (uintmax_t)start, (uintmax_t)end); | | 4725 | (uintmax_t)start, (uintmax_t)end); |
4726 | UVMHIST_LOG(maphist, " (map=%#jx, start=%#jx, end=%#jx)", | | 4726 | UVMHIST_LOG(maphist, " (map=%#jx, start=%#jx, end=%#jx)", |
4727 | (uintptr_t)map, start, end, 0); | | 4727 | (uintptr_t)map, start, end, 0); |
4728 | if (map == kernel_map) { | | 4728 | if (map == kernel_map) { |
4729 | LOCKDEBUG_MEM_CHECK((void *)start, end - start); | | 4729 | LOCKDEBUG_MEM_CHECK((void *)start, end - start); |
4730 | } | | 4730 | } |
4731 | | | 4731 | |
4732 | /* | | 4732 | /* |
4733 | * work now done by helper functions. wipe the pmap's and then | | 4733 | * work now done by helper functions. wipe the pmap's and then |
4734 | * detach from the dead entries... | | 4734 | * detach from the dead entries... |
4735 | */ | | 4735 | */ |
4736 | vm_map_lock(map); | | 4736 | vm_map_lock(map); |
4737 | uvm_unmap_remove(map, start, end, &dead_entries, flags); | | 4737 | uvm_unmap_remove(map, start, end, &dead_entries, flags); |
4738 | vm_map_unlock(map); | | 4738 | vm_map_unlock(map); |
4739 | | | 4739 | |
4740 | if (dead_entries != NULL) | | 4740 | if (dead_entries != NULL) |
4741 | uvm_unmap_detach(dead_entries, 0); | | 4741 | uvm_unmap_detach(dead_entries, 0); |
4742 | | | 4742 | |
4743 | UVMHIST_LOG(maphist, "<- done", 0,0,0,0); | | 4743 | UVMHIST_LOG(maphist, "<- done", 0,0,0,0); |
4744 | } | | 4744 | } |
4745 | | | 4745 | |
4746 | | | 4746 | |
4747 | /* | | 4747 | /* |
4748 | * uvm_map_reference: add reference to a map | | 4748 | * uvm_map_reference: add reference to a map |
4749 | * | | 4749 | * |
4750 | * => map need not be locked | | 4750 | * => map need not be locked |
4751 | */ | | 4751 | */ |
4752 | | | 4752 | |
4753 | void | | 4753 | void |
4754 | uvm_map_reference(struct vm_map *map) | | 4754 | uvm_map_reference(struct vm_map *map) |
4755 | { | | 4755 | { |
4756 | | | 4756 | |
4757 | atomic_inc_uint(&map->ref_count); | | 4757 | atomic_inc_uint(&map->ref_count); |
4758 | } | | 4758 | } |
4759 | | | 4759 | |
4760 | void | | 4760 | void |
4761 | uvm_map_lock_entry(struct vm_map_entry *entry, krw_t op) | | 4761 | uvm_map_lock_entry(struct vm_map_entry *entry, krw_t op) |
4762 | { | | 4762 | { |
4763 | | | 4763 | |
4764 | if (entry->aref.ar_amap != NULL) { | | 4764 | if (entry->aref.ar_amap != NULL) { |
4765 | amap_lock(entry->aref.ar_amap, op); | | 4765 | amap_lock(entry->aref.ar_amap, op); |
4766 | } | | 4766 | } |
4767 | if (UVM_ET_ISOBJ(entry)) { | | 4767 | if (UVM_ET_ISOBJ(entry)) { |
4768 | rw_enter(entry->object.uvm_obj->vmobjlock, op); | | 4768 | rw_enter(entry->object.uvm_obj->vmobjlock, op); |
4769 | } | | 4769 | } |
4770 | } | | 4770 | } |
4771 | | | 4771 | |
4772 | void | | 4772 | void |
4773 | uvm_map_unlock_entry(struct vm_map_entry *entry) | | 4773 | uvm_map_unlock_entry(struct vm_map_entry *entry) |
4774 | { | | 4774 | { |
4775 | | | 4775 | |
4776 | if (UVM_ET_ISOBJ(entry)) { | | 4776 | if (UVM_ET_ISOBJ(entry)) { |
4777 | rw_exit(entry->object.uvm_obj->vmobjlock); | | 4777 | rw_exit(entry->object.uvm_obj->vmobjlock); |
4778 | } | | 4778 | } |
4779 | if (entry->aref.ar_amap != NULL) { | | 4779 | if (entry->aref.ar_amap != NULL) { |
4780 | amap_unlock(entry->aref.ar_amap); | | 4780 | amap_unlock(entry->aref.ar_amap); |
4781 | } | | 4781 | } |
4782 | } | | 4782 | } |
4783 | | | 4783 | |
4784 | /* | | 4784 | /* |
4785 | * uvm_voaddr_acquire: returns the virtual object address corresponding | | 4785 | * uvm_voaddr_acquire: returns the virtual object address corresponding |
4786 | * to the specified virtual address. | | 4786 | * to the specified virtual address. |
4787 | * | | 4787 | * |
4788 | * => resolves COW so the true page identity is tracked. | | 4788 | * => resolves COW so the true page identity is tracked. |
4789 | * | | 4789 | * |
4790 | * => acquires a reference on the page's owner (uvm_object or vm_anon) | | 4790 | * => acquires a reference on the page's owner (uvm_object or vm_anon) |
4791 | */ | | 4791 | */ |
4792 | bool | | 4792 | bool |
4793 | uvm_voaddr_acquire(struct vm_map * const map, vaddr_t const va, | | 4793 | uvm_voaddr_acquire(struct vm_map * const map, vaddr_t const va, |
4794 | struct uvm_voaddr * const voaddr) | | 4794 | struct uvm_voaddr * const voaddr) |
4795 | { | | 4795 | { |
4796 | struct vm_map_entry *entry; | | 4796 | struct vm_map_entry *entry; |
4797 | struct vm_anon *anon = NULL; | | 4797 | struct vm_anon *anon = NULL; |
4798 | bool result = false; | | 4798 | bool result = false; |
4799 | bool exclusive = false; | | 4799 | bool exclusive = false; |
4800 | void (*unlock_fn)(struct vm_map *); | | 4800 | void (*unlock_fn)(struct vm_map *); |
4801 | | | 4801 | |
4802 | UVMHIST_FUNC("uvm_voaddr_acquire"); UVMHIST_CALLED(maphist); | | 4802 | UVMHIST_FUNC("uvm_voaddr_acquire"); UVMHIST_CALLED(maphist); |
4803 | UVMHIST_LOG(maphist,"(map=%#jx,va=%jx)", (uintptr_t)map, va, 0, 0); | | 4803 | UVMHIST_LOG(maphist,"(map=%#jx,va=%jx)", (uintptr_t)map, va, 0, 0); |
4804 | | | 4804 | |
4805 | const vaddr_t start = trunc_page(va); | | 4805 | const vaddr_t start = trunc_page(va); |
4806 | const vaddr_t end = round_page(va+1); | | 4806 | const vaddr_t end = round_page(va+1); |
4807 | | | 4807 | |
4808 | lookup_again: | | 4808 | lookup_again: |
4809 | if (__predict_false(exclusive)) { | | 4809 | if (__predict_false(exclusive)) { |
4810 | vm_map_lock(map); | | 4810 | vm_map_lock(map); |
4811 | unlock_fn = vm_map_unlock; | | 4811 | unlock_fn = vm_map_unlock; |
4812 | } else { | | 4812 | } else { |
4813 | vm_map_lock_read(map); | | 4813 | vm_map_lock_read(map); |
4814 | unlock_fn = vm_map_unlock_read; | | 4814 | unlock_fn = vm_map_unlock_read; |
4815 | } | | 4815 | } |
4816 | | | 4816 | |
4817 | if (__predict_false(!uvm_map_lookup_entry(map, start, &entry))) { | | 4817 | if (__predict_false(!uvm_map_lookup_entry(map, start, &entry))) { |
4818 | unlock_fn(map); | | 4818 | unlock_fn(map); |
4819 | UVMHIST_LOG(maphist,"<- done (no entry)",0,0,0,0); | | 4819 | UVMHIST_LOG(maphist,"<- done (no entry)",0,0,0,0); |
4820 | return false; | | 4820 | return false; |
4821 | } | | 4821 | } |
4822 | | | 4822 | |
4823 | if (__predict_false(entry->protection == VM_PROT_NONE)) { | | 4823 | if (__predict_false(entry->protection == VM_PROT_NONE)) { |
4824 | unlock_fn(map); | | 4824 | unlock_fn(map); |
4825 | UVMHIST_LOG(maphist,"<- done (PROT_NONE)",0,0,0,0); | | 4825 | UVMHIST_LOG(maphist,"<- done (PROT_NONE)",0,0,0,0); |
4826 | return false; | | 4826 | return false; |
4827 | } | | 4827 | } |
4828 | | | 4828 | |
4829 | /* | | 4829 | /* |
4830 | * We have a fast path for the common case of "no COW resolution | | 4830 | * We have a fast path for the common case of "no COW resolution |
4831 | * needed" whereby we have taken a read lock on the map and if | | 4831 | * needed" whereby we have taken a read lock on the map and if |
4832 | * we don't encounter any need to create a vm_anon then great! | | 4832 | * we don't encounter any need to create a vm_anon then great! |
4833 | * But if we do, we loop around again, instead taking an exclusive | | 4833 | * But if we do, we loop around again, instead taking an exclusive |
4834 | * lock so that we can perform the fault. | | 4834 | * lock so that we can perform the fault. |
4835 | * | | 4835 | * |
4836 | * In the event that we have to resolve the fault, we do nearly the | | 4836 | * In the event that we have to resolve the fault, we do nearly the |
4837 | * same work as uvm_map_pageable() does: | | 4837 | * same work as uvm_map_pageable() does: |
4838 | * | | 4838 | * |
4839 | * 1: holding the write lock, we create any anonymous maps that need | | 4839 | * 1: holding the write lock, we create any anonymous maps that need |
4840 | * to be created. however, we do NOT need to clip the map entries | | 4840 | * to be created. however, we do NOT need to clip the map entries |
4841 | * in this case. | | 4841 | * in this case. |
4842 | * | | 4842 | * |
4843 | * 2: we downgrade to a read lock, and call uvm_fault_wire to fault | | 4843 | * 2: we downgrade to a read lock, and call uvm_fault_wire to fault |
4844 | * in the page (assuming the entry is not already wired). this | | 4844 | * in the page (assuming the entry is not already wired). this |
4845 | * is done because we need the vm_anon to be present. | | 4845 | * is done because we need the vm_anon to be present. |
4846 | */ | | 4846 | */ |
4847 | if (__predict_true(!VM_MAPENT_ISWIRED(entry))) { | | 4847 | if (__predict_true(!VM_MAPENT_ISWIRED(entry))) { |
4848 | | | 4848 | |
4849 | bool need_fault = false; | | 4849 | bool need_fault = false; |
4850 | | | 4850 | |
4851 | /* | | 4851 | /* |
4852 | * perform the action of vm_map_lookup that need the | | 4852 | * perform the action of vm_map_lookup that need the |
4853 | * write lock on the map: create an anonymous map for | | 4853 | * write lock on the map: create an anonymous map for |
4854 | * a copy-on-write region, or an anonymous map for | | 4854 | * a copy-on-write region, or an anonymous map for |
4855 | * a zero-fill region. | | 4855 | * a zero-fill region. |
4856 | */ | | 4856 | */ |
4857 | if (__predict_false(UVM_ET_ISSUBMAP(entry))) { | | 4857 | if (__predict_false(UVM_ET_ISSUBMAP(entry))) { |
4858 | unlock_fn(map); | | 4858 | unlock_fn(map); |
4859 | UVMHIST_LOG(maphist,"<- done (submap)",0,0,0,0); | | 4859 | UVMHIST_LOG(maphist,"<- done (submap)",0,0,0,0); |
4860 | return false; | | 4860 | return false; |
4861 | } | | 4861 | } |
4862 | if (__predict_false(UVM_ET_ISNEEDSCOPY(entry) && | | 4862 | if (__predict_false(UVM_ET_ISNEEDSCOPY(entry) && |
4863 | ((entry->max_protection & VM_PROT_WRITE) || | | 4863 | ((entry->max_protection & VM_PROT_WRITE) || |
4864 | (entry->object.uvm_obj == NULL)))) { | | 4864 | (entry->object.uvm_obj == NULL)))) { |
4865 | if (!exclusive) { | | 4865 | if (!exclusive) { |
4866 | /* need to take the slow path */ | | 4866 | /* need to take the slow path */ |
4867 | KASSERT(unlock_fn == vm_map_unlock_read); | | 4867 | KASSERT(unlock_fn == vm_map_unlock_read); |
4868 | vm_map_unlock_read(map); | | 4868 | vm_map_unlock_read(map); |
4869 | exclusive = true; | | 4869 | exclusive = true; |
4870 | goto lookup_again; | | 4870 | goto lookup_again; |
4871 | } | | 4871 | } |
4872 | need_fault = true; | | 4872 | need_fault = true; |
4873 | amap_copy(map, entry, 0, start, end); | | 4873 | amap_copy(map, entry, 0, start, end); |
4874 | /* XXXCDC: wait OK? */ | | 4874 | /* XXXCDC: wait OK? */ |
4875 | } | | 4875 | } |
4876 | | | 4876 | |
4877 | /* | | 4877 | /* |
4878 | * do a quick check to see if the fault has already | | 4878 | * do a quick check to see if the fault has already |
4879 | * been resolved to the upper layer. | | 4879 | * been resolved to the upper layer. |
4880 | */ | | 4880 | */ |
4881 | if (__predict_true(entry->aref.ar_amap != NULL && | | 4881 | if (__predict_true(entry->aref.ar_amap != NULL && |
4882 | need_fault == false)) { | | 4882 | need_fault == false)) { |
4883 | amap_lock(entry->aref.ar_amap, RW_WRITER); | | 4883 | amap_lock(entry->aref.ar_amap, RW_WRITER); |
4884 | anon = amap_lookup(&entry->aref, start - entry->start); | | 4884 | anon = amap_lookup(&entry->aref, start - entry->start); |
4885 | if (__predict_true(anon != NULL)) { | | 4885 | if (__predict_true(anon != NULL)) { |
4886 | /* amap unlocked below */ | | 4886 | /* amap unlocked below */ |
4887 | goto found_anon; | | 4887 | goto found_anon; |
4888 | } | | 4888 | } |
4889 | amap_unlock(entry->aref.ar_amap); | | 4889 | amap_unlock(entry->aref.ar_amap); |
4890 | need_fault = true; | | 4890 | need_fault = true; |
4891 | } | | 4891 | } |
4892 | | | 4892 | |
4893 | /* | | 4893 | /* |
4894 | * we predict this test as false because if we reach | | 4894 | * we predict this test as false because if we reach |
4895 | * this point, then we are likely dealing with a | | 4895 | * this point, then we are likely dealing with a |
4896 | * shared memory region backed by a uvm_object, in | | 4896 | * shared memory region backed by a uvm_object, in |
4897 | * which case a fault to create the vm_anon is not | | 4897 | * which case a fault to create the vm_anon is not |
4898 | * necessary. | | 4898 | * necessary. |
4899 | */ | | 4899 | */ |
4900 | if (__predict_false(need_fault)) { | | 4900 | if (__predict_false(need_fault)) { |
4901 | if (exclusive) { | | 4901 | if (exclusive) { |
4902 | vm_map_busy(map); | | 4902 | vm_map_busy(map); |
4903 | vm_map_unlock(map); | | 4903 | vm_map_unlock(map); |
4904 | unlock_fn = vm_map_unbusy; | | 4904 | unlock_fn = vm_map_unbusy; |
4905 | } | | 4905 | } |
4906 | | | 4906 | |
4907 | if (uvm_fault_wire(map, start, end, | | 4907 | if (uvm_fault_wire(map, start, end, |
4908 | entry->max_protection, 1)) { | | 4908 | entry->max_protection, 1)) { |
4909 | /* wiring failed */ | | 4909 | /* wiring failed */ |
4910 | unlock_fn(map); | | 4910 | unlock_fn(map); |
4911 | UVMHIST_LOG(maphist,"<- done (wire failed)", | | 4911 | UVMHIST_LOG(maphist,"<- done (wire failed)", |
4912 | 0,0,0,0); | | 4912 | 0,0,0,0); |
4913 | return false; | | 4913 | return false; |
4914 | } | | 4914 | } |
4915 | | | 4915 | |
4916 | /* | | 4916 | /* |
4917 | * now that we have resolved the fault, we can unwire | | 4917 | * now that we have resolved the fault, we can unwire |
4918 | * the page. | | 4918 | * the page. |
4919 | */ | | 4919 | */ |
4920 | if (exclusive) { | | 4920 | if (exclusive) { |
4921 | vm_map_lock(map); | | 4921 | vm_map_lock(map); |
4922 | vm_map_unbusy(map); | | 4922 | vm_map_unbusy(map); |
4923 | unlock_fn = vm_map_unlock; | | 4923 | unlock_fn = vm_map_unlock; |
4924 | } | | 4924 | } |
4925 | | | 4925 | |
4926 | uvm_fault_unwire_locked(map, start, end); | | 4926 | uvm_fault_unwire_locked(map, start, end); |
4927 | } | | 4927 | } |
4928 | } | | 4928 | } |
4929 | | | 4929 | |
4930 | /* check the upper layer */ | | 4930 | /* check the upper layer */ |
4931 | if (entry->aref.ar_amap) { | | 4931 | if (entry->aref.ar_amap) { |
4932 | amap_lock(entry->aref.ar_amap, RW_WRITER); | | 4932 | amap_lock(entry->aref.ar_amap, RW_WRITER); |
4933 | anon = amap_lookup(&entry->aref, start - entry->start); | | 4933 | anon = amap_lookup(&entry->aref, start - entry->start); |
4934 | if (anon) { | | 4934 | if (anon) { |
4935 | found_anon: KASSERT(anon->an_lock == entry->aref.ar_amap->am_lock); | | 4935 | found_anon: KASSERT(anon->an_lock == entry->aref.ar_amap->am_lock); |
4936 | anon->an_ref++; | | 4936 | anon->an_ref++; |
4937 | KASSERT(anon->an_ref != 0); | | 4937 | KASSERT(anon->an_ref != 0); |
4938 | voaddr->type = UVM_VOADDR_TYPE_ANON; | | 4938 | voaddr->type = UVM_VOADDR_TYPE_ANON; |
4939 | voaddr->anon = anon; | | 4939 | voaddr->anon = anon; |
4940 | voaddr->offset = va & PAGE_MASK; | | 4940 | voaddr->offset = va & PAGE_MASK; |
4941 | result = true; | | 4941 | result = true; |
4942 | } | | 4942 | } |
4943 | amap_unlock(entry->aref.ar_amap); | | 4943 | amap_unlock(entry->aref.ar_amap); |
4944 | } | | 4944 | } |
4945 | | | 4945 | |
4946 | /* check the lower layer */ | | 4946 | /* check the lower layer */ |
4947 | if (!result && UVM_ET_ISOBJ(entry)) { | | 4947 | if (!result && UVM_ET_ISOBJ(entry)) { |
4948 | struct uvm_object *uobj = entry->object.uvm_obj; | | 4948 | struct uvm_object *uobj = entry->object.uvm_obj; |
4949 | | | 4949 | |
4950 | KASSERT(uobj != NULL); | | 4950 | KASSERT(uobj != NULL); |
4951 | (*uobj->pgops->pgo_reference)(uobj); | | 4951 | (*uobj->pgops->pgo_reference)(uobj); |
4952 | voaddr->type = UVM_VOADDR_TYPE_OBJECT; | | 4952 | voaddr->type = UVM_VOADDR_TYPE_OBJECT; |
4953 | voaddr->uobj = uobj; | | 4953 | voaddr->uobj = uobj; |
4954 | voaddr->offset = entry->offset + (va - entry->start); | | 4954 | voaddr->offset = entry->offset + (va - entry->start); |
4955 | result = true; | | 4955 | result = true; |
4956 | } | | 4956 | } |
4957 | | | 4957 | |
4958 | unlock_fn(map); | | 4958 | unlock_fn(map); |
4959 | | | 4959 | |
4960 | if (result) { | | 4960 | if (result) { |
4961 | UVMHIST_LOG(maphist, | | 4961 | UVMHIST_LOG(maphist, |
4962 | "<- done OK (type=%jd,owner=#%jx,offset=%jx)", | | 4962 | "<- done OK (type=%jd,owner=#%jx,offset=%jx)", |
4963 | voaddr->type, voaddr->uobj, voaddr->offset, 0); | | 4963 | voaddr->type, voaddr->uobj, voaddr->offset, 0); |
4964 | } else { | | 4964 | } else { |
4965 | UVMHIST_LOG(maphist,"<- done (failed)",0,0,0,0); | | 4965 | UVMHIST_LOG(maphist,"<- done (failed)",0,0,0,0); |
4966 | } | | 4966 | } |
4967 | | | 4967 | |
4968 | return result; | | 4968 | return result; |
4969 | } | | 4969 | } |
4970 | | | 4970 | |
4971 | /* | | 4971 | /* |
4972 | * uvm_voaddr_release: release the references held by the | | 4972 | * uvm_voaddr_release: release the references held by the |
4973 | * vitual object address. | | 4973 | * vitual object address. |
4974 | */ | | 4974 | */ |
4975 | void | | 4975 | void |
4976 | uvm_voaddr_release(struct uvm_voaddr * const voaddr) | | 4976 | uvm_voaddr_release(struct uvm_voaddr * const voaddr) |
4977 | { | | 4977 | { |
4978 | | | 4978 | |
4979 | switch (voaddr->type) { | | 4979 | switch (voaddr->type) { |
4980 | case UVM_VOADDR_TYPE_OBJECT: { | | 4980 | case UVM_VOADDR_TYPE_OBJECT: { |
4981 | struct uvm_object * const uobj = voaddr->uobj; | | 4981 | struct uvm_object * const uobj = voaddr->uobj; |
4982 | | | 4982 | |
4983 | KASSERT(uobj != NULL); | | 4983 | KASSERT(uobj != NULL); |
4984 | KASSERT(uobj->pgops->pgo_detach != NULL); | | 4984 | KASSERT(uobj->pgops->pgo_detach != NULL); |
4985 | (*uobj->pgops->pgo_detach)(uobj); | | 4985 | (*uobj->pgops->pgo_detach)(uobj); |
4986 | break; | | 4986 | break; |
4987 | } | | 4987 | } |
4988 | case UVM_VOADDR_TYPE_ANON: { | | 4988 | case UVM_VOADDR_TYPE_ANON: { |
4989 | struct vm_anon * const anon = voaddr->anon; | | 4989 | struct vm_anon * const anon = voaddr->anon; |
4990 | | | 4990 | |
4991 | KASSERT(anon != NULL); | | 4991 | KASSERT(anon != NULL); |
4992 | rw_enter(anon->an_lock, RW_WRITER); | | 4992 | rw_enter(anon->an_lock, RW_WRITER); |
4993 | KASSERT(anon->an_ref > 0); | | 4993 | KASSERT(anon->an_ref > 0); |
4994 | anon->an_ref--; | | 4994 | anon->an_ref--; |
4995 | if (anon->an_ref == 0) { | | 4995 | if (anon->an_ref == 0) { |
4996 | uvm_anon_release(anon); | | 4996 | uvm_anon_release(anon); |
4997 | } else { | | 4997 | } else { |
4998 | rw_exit(anon->an_lock); | | 4998 | rw_exit(anon->an_lock); |
4999 | } | | 4999 | } |
5000 | break; | | 5000 | break; |
5001 | } | | 5001 | } |
5002 | default: | | 5002 | default: |
5003 | panic("uvm_voaddr_release: bad type"); | | 5003 | panic("uvm_voaddr_release: bad type"); |
5004 | } | | 5004 | } |
5005 | memset(voaddr, 0, sizeof(*voaddr)); | | 5005 | memset(voaddr, 0, sizeof(*voaddr)); |
5006 | } | | 5006 | } |
5007 | | | 5007 | |
5008 | /* | | 5008 | /* |
5009 | * uvm_voaddr_compare: compare two uvm_voaddr objects. | | 5009 | * uvm_voaddr_compare: compare two uvm_voaddr objects. |
5010 | * | | 5010 | * |
5011 | * => memcmp() semantics | | 5011 | * => memcmp() semantics |
5012 | */ | | 5012 | */ |
5013 | int | | 5013 | int |
5014 | uvm_voaddr_compare(const struct uvm_voaddr * const voaddr1, | | 5014 | uvm_voaddr_compare(const struct uvm_voaddr * const voaddr1, |
5015 | const struct uvm_voaddr * const voaddr2) | | 5015 | const struct uvm_voaddr * const voaddr2) |
5016 | { | | 5016 | { |
5017 | | | 5017 | |
5018 | KASSERT(voaddr1->type == UVM_VOADDR_TYPE_OBJECT || | | 5018 | KASSERT(voaddr1->type == UVM_VOADDR_TYPE_OBJECT || |
5019 | voaddr1->type == UVM_VOADDR_TYPE_ANON); | | 5019 | voaddr1->type == UVM_VOADDR_TYPE_ANON); |
5020 | | | 5020 | |
5021 | KASSERT(voaddr2->type == UVM_VOADDR_TYPE_OBJECT || | | 5021 | KASSERT(voaddr2->type == UVM_VOADDR_TYPE_OBJECT || |
5022 | voaddr2->type == UVM_VOADDR_TYPE_ANON); | | 5022 | voaddr2->type == UVM_VOADDR_TYPE_ANON); |
5023 | | | 5023 | |
5024 | if (voaddr1->type < voaddr2->type) | | 5024 | if (voaddr1->type < voaddr2->type) |
5025 | return -1; | | 5025 | return -1; |
5026 | if (voaddr1->type > voaddr2->type) | | 5026 | if (voaddr1->type > voaddr2->type) |
5027 | return 1; | | 5027 | return 1; |
5028 | | | 5028 | |
5029 | /* These fields are unioned together. */ | | 5029 | /* These fields are unioned together. */ |
5030 | CTASSERT(offsetof(struct uvm_voaddr, uobj) == | | 5030 | CTASSERT(offsetof(struct uvm_voaddr, uobj) == |
5031 | offsetof(struct uvm_voaddr, anon)); | | 5031 | offsetof(struct uvm_voaddr, anon)); |
5032 | const uintptr_t addr1 = (uintptr_t)voaddr1->uobj; | | 5032 | const uintptr_t addr1 = (uintptr_t)voaddr1->uobj; |
5033 | const uintptr_t addr2 = (uintptr_t)voaddr2->uobj; | | 5033 | const uintptr_t addr2 = (uintptr_t)voaddr2->uobj; |
5034 | | | 5034 | |
5035 | if (addr1 < addr2) | | 5035 | if (addr1 < addr2) |
5036 | return -1; | | 5036 | return -1; |
5037 | if (addr1 > addr2) | | 5037 | if (addr1 > addr2) |
5038 | return 1; | | 5038 | return 1; |
5039 | | | 5039 | |
5040 | if (voaddr1->offset < voaddr2->offset) | | 5040 | if (voaddr1->offset < voaddr2->offset) |
5041 | return -1; | | 5041 | return -1; |
5042 | if (voaddr1->offset > voaddr2->offset) | | 5042 | if (voaddr1->offset > voaddr2->offset) |
5043 | return 1; | | 5043 | return 1; |
5044 | | | 5044 | |
5045 | return 0; | | 5045 | return 0; |
5046 | } | | 5046 | } |
5047 | | | 5047 | |
5048 | #if defined(DDB) || defined(DEBUGPRINT) | | 5048 | #if defined(DDB) || defined(DEBUGPRINT) |
5049 | | | 5049 | |
5050 | /* | | 5050 | /* |
5051 | * uvm_map_printit: actually prints the map | | 5051 | * uvm_map_printit: actually prints the map |
5052 | */ | | 5052 | */ |
5053 | | | 5053 | |
5054 | void | | 5054 | void |
5055 | uvm_map_printit(struct vm_map *map, bool full, | | 5055 | uvm_map_printit(struct vm_map *map, bool full, |
5056 | void (*pr)(const char *, ...)) | | 5056 | void (*pr)(const char *, ...)) |
5057 | { | | 5057 | { |
5058 | struct vm_map_entry *entry; | | 5058 | struct vm_map_entry *entry; |
5059 | | | 5059 | |
5060 | (*pr)("MAP %p: [%#lx->%#lx]\n", map, vm_map_min(map), | | 5060 | (*pr)("MAP %p: [%#lx->%#lx]\n", map, vm_map_min(map), |
5061 | vm_map_max(map)); | | 5061 | vm_map_max(map)); |
5062 | (*pr)("\t#ent=%d, sz=%d, ref=%d, version=%d, flags=%#x\n", | | 5062 | (*pr)("\t#ent=%d, sz=%d, ref=%d, version=%d, flags=%#x\n", |
5063 | map->nentries, map->size, map->ref_count, map->timestamp, | | 5063 | map->nentries, map->size, map->ref_count, map->timestamp, |
5064 | map->flags); | | 5064 | map->flags); |
5065 | (*pr)("\tpmap=%p(resident=%ld, wired=%ld)\n", map->pmap, | | 5065 | (*pr)("\tpmap=%p(resident=%ld, wired=%ld)\n", map->pmap, |
5066 | pmap_resident_count(map->pmap), pmap_wired_count(map->pmap)); | | 5066 | pmap_resident_count(map->pmap), pmap_wired_count(map->pmap)); |
5067 | if (!full) | | 5067 | if (!full) |
5068 | return; | | 5068 | return; |
5069 | for (entry = map->header.next; entry != &map->header; | | 5069 | for (entry = map->header.next; entry != &map->header; |
5070 | entry = entry->next) { | | 5070 | entry = entry->next) { |
5071 | (*pr)(" - %p: %#lx->%#lx: obj=%p/%#llx, amap=%p/%d\n", | | 5071 | (*pr)(" - %p: %#lx->%#lx: obj=%p/%#llx, amap=%p/%d\n", |
5072 | entry, entry->start, entry->end, entry->object.uvm_obj, | | 5072 | entry, entry->start, entry->end, entry->object.uvm_obj, |
5073 | (long long)entry->offset, entry->aref.ar_amap, | | 5073 | (long long)entry->offset, entry->aref.ar_amap, |
5074 | entry->aref.ar_pageoff); | | 5074 | entry->aref.ar_pageoff); |
5075 | (*pr)( | | 5075 | (*pr)( |
5076 | "\tsubmap=%c, cow=%c, nc=%c, prot(max)=%d/%d, inh=%d, " | | 5076 | "\tsubmap=%c, cow=%c, nc=%c, prot(max)=%d/%d, inh=%d, " |
5077 | "wc=%d, adv=%d\n", | | 5077 | "wc=%d, adv=%d\n", |
5078 | (entry->etype & UVM_ET_SUBMAP) ? 'T' : 'F', | | 5078 | (entry->etype & UVM_ET_SUBMAP) ? 'T' : 'F', |
5079 | (entry->etype & UVM_ET_COPYONWRITE) ? 'T' : 'F', | | 5079 | (entry->etype & UVM_ET_COPYONWRITE) ? 'T' : 'F', |
5080 | (entry->etype & UVM_ET_NEEDSCOPY) ? 'T' : 'F', | | 5080 | (entry->etype & UVM_ET_NEEDSCOPY) ? 'T' : 'F', |
5081 | entry->protection, entry->max_protection, | | 5081 | entry->protection, entry->max_protection, |
5082 | entry->inheritance, entry->wired_count, entry->advice); | | 5082 | entry->inheritance, entry->wired_count, entry->advice); |
5083 | } | | 5083 | } |
5084 | } | | 5084 | } |
5085 | | | 5085 | |
5086 | void | | 5086 | void |
5087 | uvm_whatis(uintptr_t addr, void (*pr)(const char *, ...)) | | 5087 | uvm_whatis(uintptr_t addr, void (*pr)(const char *, ...)) |
5088 | { | | 5088 | { |
5089 | struct vm_map *map; | | 5089 | struct vm_map *map; |
5090 | | | 5090 | |
5091 | for (map = kernel_map;;) { | | 5091 | for (map = kernel_map;;) { |
5092 | struct vm_map_entry *entry; | | 5092 | struct vm_map_entry *entry; |
5093 | | | 5093 | |
5094 | if (!uvm_map_lookup_entry_bytree(map, (vaddr_t)addr, &entry)) { | | 5094 | if (!uvm_map_lookup_entry_bytree(map, (vaddr_t)addr, &entry)) { |
5095 | break; | | 5095 | break; |
5096 | } | | 5096 | } |
5097 | (*pr)("%p is %p+%zu from VMMAP %p\n", | | 5097 | (*pr)("%p is %p+%zu from VMMAP %p\n", |
5098 | (void *)addr, (void *)entry->start, | | 5098 | (void *)addr, (void *)entry->start, |
5099 | (size_t)(addr - (uintptr_t)entry->start), map); | | 5099 | (size_t)(addr - (uintptr_t)entry->start), map); |
5100 | if (!UVM_ET_ISSUBMAP(entry)) { | | 5100 | if (!UVM_ET_ISSUBMAP(entry)) { |
5101 | break; | | 5101 | break; |
5102 | } | | 5102 | } |
5103 | map = entry->object.sub_map; | | 5103 | map = entry->object.sub_map; |
5104 | } | | 5104 | } |
5105 | } | | 5105 | } |
5106 | | | 5106 | |
5107 | #endif /* DDB || DEBUGPRINT */ | | 5107 | #endif /* DDB || DEBUGPRINT */ |
5108 | | | 5108 | |
5109 | #ifndef __USER_VA0_IS_SAFE | | 5109 | #ifndef __USER_VA0_IS_SAFE |
5110 | static int | | 5110 | static int |
5111 | sysctl_user_va0_disable(SYSCTLFN_ARGS) | | 5111 | sysctl_user_va0_disable(SYSCTLFN_ARGS) |
5112 | { | | 5112 | { |
5113 | struct sysctlnode node; | | 5113 | struct sysctlnode node; |
5114 | int t, error; | | 5114 | int t, error; |
5115 | | | 5115 | |
5116 | node = *rnode; | | 5116 | node = *rnode; |
5117 | node.sysctl_data = &t; | | 5117 | node.sysctl_data = &t; |
5118 | t = user_va0_disable; | | 5118 | t = user_va0_disable; |
5119 | error = sysctl_lookup(SYSCTLFN_CALL(&node)); | | 5119 | error = sysctl_lookup(SYSCTLFN_CALL(&node)); |
5120 | if (error || newp == NULL) | | 5120 | if (error || newp == NULL) |
5121 | return (error); | | 5121 | return (error); |
5122 | | | 5122 | |
5123 | if (!t && user_va0_disable && | | 5123 | if (!t && user_va0_disable && |
5124 | kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_MAP_VA_ZERO, 0, | | 5124 | kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_MAP_VA_ZERO, 0, |
5125 | NULL, NULL, NULL)) | | 5125 | NULL, NULL, NULL)) |
5126 | return EPERM; | | 5126 | return EPERM; |
5127 | | | 5127 | |
5128 | user_va0_disable = !!t; | | 5128 | user_va0_disable = !!t; |
5129 | return 0; | | 5129 | return 0; |
5130 | } | | 5130 | } |
5131 | #endif | | 5131 | #endif |
5132 | | | 5132 | |
5133 | static int | | 5133 | static int |
5134 | fill_vmentry(struct lwp *l, struct proc *p, struct kinfo_vmentry *kve, | | 5134 | fill_vmentry(struct lwp *l, struct proc *p, struct kinfo_vmentry *kve, |
5135 | struct vm_map *m, struct vm_map_entry *e) | | 5135 | struct vm_map *m, struct vm_map_entry *e) |
5136 | { | | 5136 | { |
5137 | #ifndef _RUMPKERNEL | | 5137 | #ifndef _RUMPKERNEL |
5138 | int error; | | 5138 | int error; |
5139 | | | 5139 | |
5140 | memset(kve, 0, sizeof(*kve)); | | 5140 | memset(kve, 0, sizeof(*kve)); |
5141 | KASSERT(e != NULL); | | 5141 | KASSERT(e != NULL); |
5142 | if (UVM_ET_ISOBJ(e)) { | | 5142 | if (UVM_ET_ISOBJ(e)) { |
5143 | struct uvm_object *uobj = e->object.uvm_obj; | | 5143 | struct uvm_object *uobj = e->object.uvm_obj; |
5144 | KASSERT(uobj != NULL); | | 5144 | KASSERT(uobj != NULL); |
5145 | kve->kve_ref_count = uobj->uo_refs; | | 5145 | kve->kve_ref_count = uobj->uo_refs; |
5146 | kve->kve_count = uobj->uo_npages; | | 5146 | kve->kve_count = uobj->uo_npages; |
5147 | if (UVM_OBJ_IS_VNODE(uobj)) { | | 5147 | if (UVM_OBJ_IS_VNODE(uobj)) { |
5148 | struct vattr va; | | 5148 | struct vattr va; |
5149 | struct vnode *vp = (struct vnode *)uobj; | | 5149 | struct vnode *vp = (struct vnode *)uobj; |
5150 | vn_lock(vp, LK_SHARED | LK_RETRY); | | 5150 | vn_lock(vp, LK_SHARED | LK_RETRY); |
5151 | error = VOP_GETATTR(vp, &va, l->l_cred); | | 5151 | error = VOP_GETATTR(vp, &va, l->l_cred); |
5152 | VOP_UNLOCK(vp); | | 5152 | VOP_UNLOCK(vp); |
5153 | kve->kve_type = KVME_TYPE_VNODE; | | 5153 | kve->kve_type = KVME_TYPE_VNODE; |
5154 | if (error == 0) { | | 5154 | if (error == 0) { |
5155 | kve->kve_vn_size = vp->v_size; | | 5155 | kve->kve_vn_size = vp->v_size; |
5156 | kve->kve_vn_type = (int)vp->v_type; | | 5156 | kve->kve_vn_type = (int)vp->v_type; |
5157 | kve->kve_vn_mode = va.va_mode; | | 5157 | kve->kve_vn_mode = va.va_mode; |
5158 | kve->kve_vn_rdev = va.va_rdev; | | 5158 | kve->kve_vn_rdev = va.va_rdev; |
5159 | kve->kve_vn_fileid = va.va_fileid; | | 5159 | kve->kve_vn_fileid = va.va_fileid; |
5160 | kve->kve_vn_fsid = va.va_fsid; | | 5160 | kve->kve_vn_fsid = va.va_fsid; |
5161 | error = vnode_to_path(kve->kve_path, | | 5161 | error = vnode_to_path(kve->kve_path, |
5162 | sizeof(kve->kve_path) / 2, vp, l, p); | | 5162 | sizeof(kve->kve_path) / 2, vp, l, p); |
5163 | #ifdef DIAGNOSTIC | | 5163 | #ifdef DIAGNOSTIC |
5164 | if (error) | | 5164 | if (error) |
5165 | printf("%s: vp %p error %d\n", __func__, | | 5165 | printf("%s: vp %p error %d\n", __func__, |
5166 | vp, error); | | 5166 | vp, error); |
5167 | #endif | | 5167 | #endif |
5168 | } | | 5168 | } |
5169 | } else if (UVM_OBJ_IS_KERN_OBJECT(uobj)) { | | 5169 | } else if (UVM_OBJ_IS_KERN_OBJECT(uobj)) { |
5170 | kve->kve_type = KVME_TYPE_KERN; | | 5170 | kve->kve_type = KVME_TYPE_KERN; |
5171 | } else if (UVM_OBJ_IS_DEVICE(uobj)) { | | 5171 | } else if (UVM_OBJ_IS_DEVICE(uobj)) { |
5172 | kve->kve_type = KVME_TYPE_DEVICE; | | 5172 | kve->kve_type = KVME_TYPE_DEVICE; |
5173 | } else if (UVM_OBJ_IS_AOBJ(uobj)) { | | 5173 | } else if (UVM_OBJ_IS_AOBJ(uobj)) { |
5174 | kve->kve_type = KVME_TYPE_ANON; | | 5174 | kve->kve_type = KVME_TYPE_ANON; |
5175 | } else { | | 5175 | } else { |
5176 | kve->kve_type = KVME_TYPE_OBJECT; | | 5176 | kve->kve_type = KVME_TYPE_OBJECT; |
5177 | } | | 5177 | } |
5178 | } else if (UVM_ET_ISSUBMAP(e)) { | | 5178 | } else if (UVM_ET_ISSUBMAP(e)) { |
5179 | struct vm_map *map = e->object.sub_map; | | 5179 | struct vm_map *map = e->object.sub_map; |
5180 | KASSERT(map != NULL); | | 5180 | KASSERT(map != NULL); |
5181 | kve->kve_ref_count = map->ref_count; | | 5181 | kve->kve_ref_count = map->ref_count; |
5182 | kve->kve_count = map->nentries; | | 5182 | kve->kve_count = map->nentries; |
5183 | kve->kve_type = KVME_TYPE_SUBMAP; | | 5183 | kve->kve_type = KVME_TYPE_SUBMAP; |
5184 | } else | | 5184 | } else |
5185 | kve->kve_type = KVME_TYPE_UNKNOWN; | | 5185 | kve->kve_type = KVME_TYPE_UNKNOWN; |
5186 | | | 5186 | |
5187 | kve->kve_start = e->start; | | 5187 | kve->kve_start = e->start; |
5188 | kve->kve_end = e->end; | | 5188 | kve->kve_end = e->end; |
5189 | kve->kve_offset = e->offset; | | 5189 | kve->kve_offset = e->offset; |
5190 | kve->kve_wired_count = e->wired_count; | | 5190 | kve->kve_wired_count = e->wired_count; |
5191 | kve->kve_inheritance = e->inheritance; | | 5191 | kve->kve_inheritance = e->inheritance; |
5192 | kve->kve_attributes = 0; /* unused */ | | 5192 | kve->kve_attributes = 0; /* unused */ |
5193 | kve->kve_advice = e->advice; | | 5193 | kve->kve_advice = e->advice; |
5194 | #define PROT(p) (((p) & VM_PROT_READ) ? KVME_PROT_READ : 0) | \ | | 5194 | #define PROT(p) (((p) & VM_PROT_READ) ? KVME_PROT_READ : 0) | \ |
5195 | (((p) & VM_PROT_WRITE) ? KVME_PROT_WRITE : 0) | \ | | 5195 | (((p) & VM_PROT_WRITE) ? KVME_PROT_WRITE : 0) | \ |
5196 | (((p) & VM_PROT_EXECUTE) ? KVME_PROT_EXEC : 0) | | 5196 | (((p) & VM_PROT_EXECUTE) ? KVME_PROT_EXEC : 0) |
5197 | kve->kve_protection = PROT(e->protection); | | 5197 | kve->kve_protection = PROT(e->protection); |
5198 | kve->kve_max_protection = PROT(e->max_protection); | | 5198 | kve->kve_max_protection = PROT(e->max_protection); |
5199 | kve->kve_flags |= (e->etype & UVM_ET_COPYONWRITE) | | 5199 | kve->kve_flags |= (e->etype & UVM_ET_COPYONWRITE) |
5200 | ? KVME_FLAG_COW : 0; | | 5200 | ? KVME_FLAG_COW : 0; |
5201 | kve->kve_flags |= (e->etype & UVM_ET_NEEDSCOPY) | | 5201 | kve->kve_flags |= (e->etype & UVM_ET_NEEDSCOPY) |
5202 | ? KVME_FLAG_NEEDS_COPY : 0; | | 5202 | ? KVME_FLAG_NEEDS_COPY : 0; |
5203 | kve->kve_flags |= (m->flags & VM_MAP_TOPDOWN) | | 5203 | kve->kve_flags |= (m->flags & VM_MAP_TOPDOWN) |
5204 | ? KVME_FLAG_GROWS_DOWN : KVME_FLAG_GROWS_UP; | | 5204 | ? KVME_FLAG_GROWS_DOWN : KVME_FLAG_GROWS_UP; |
5205 | kve->kve_flags |= (m->flags & VM_MAP_PAGEABLE) | | 5205 | kve->kve_flags |= (m->flags & VM_MAP_PAGEABLE) |
5206 | ? KVME_FLAG_PAGEABLE : 0; | | 5206 | ? KVME_FLAG_PAGEABLE : 0; |
5207 | #endif | | 5207 | #endif |
5208 | return 0; | | 5208 | return 0; |
5209 | } | | 5209 | } |
5210 | | | 5210 | |
5211 | static int | | 5211 | static int |
5212 | fill_vmentries(struct lwp *l, pid_t pid, u_int elem_size, void *oldp, | | 5212 | fill_vmentries(struct lwp *l, pid_t pid, u_int elem_size, void *oldp, |
5213 | size_t *oldlenp) | | 5213 | size_t *oldlenp) |
5214 | { | | 5214 | { |
5215 | int error; | | 5215 | int error; |
5216 | struct proc *p; | | 5216 | struct proc *p; |
5217 | struct kinfo_vmentry *vme; | | 5217 | struct kinfo_vmentry *vme; |
5218 | struct vmspace *vm; | | 5218 | struct vmspace *vm; |
5219 | struct vm_map *map; | | 5219 | struct vm_map *map; |
5220 | struct vm_map_entry *entry; | | 5220 | struct vm_map_entry *entry; |
5221 | char *dp; | | 5221 | char *dp; |
5222 | size_t count, vmesize; | | 5222 | size_t count, vmesize; |
5223 | | | 5223 | |
5224 | if (elem_size == 0 || elem_size > 2 * sizeof(*vme)) | | 5224 | if (elem_size == 0 || elem_size > 2 * sizeof(*vme)) |
5225 | return EINVAL; | | 5225 | return EINVAL; |
5226 | | | 5226 | |
5227 | if (oldp) { | | 5227 | if (oldp) { |
5228 | if (*oldlenp > 10UL * 1024UL * 1024UL) | | 5228 | if (*oldlenp > 10UL * 1024UL * 1024UL) |
5229 | return E2BIG; | | 5229 | return E2BIG; |
5230 | count = *oldlenp / elem_size; | | 5230 | count = *oldlenp / elem_size; |
5231 | if (count == 0) | | 5231 | if (count == 0) |
5232 | return ENOMEM; | | 5232 | return ENOMEM; |
5233 | vmesize = count * sizeof(*vme); | | 5233 | vmesize = count * sizeof(*vme); |
5234 | } else | | 5234 | } else |
5235 | vmesize = 0; | | 5235 | vmesize = 0; |
5236 | | | 5236 | |
5237 | if ((error = proc_find_locked(l, &p, pid)) != 0) | | 5237 | if ((error = proc_find_locked(l, &p, pid)) != 0) |
5238 | return error; | | 5238 | return error; |
5239 | | | 5239 | |
5240 | vme = NULL; | | 5240 | vme = NULL; |
5241 | count = 0; | | 5241 | count = 0; |
5242 | | | 5242 | |
5243 | if ((error = proc_vmspace_getref(p, &vm)) != 0) | | 5243 | if ((error = proc_vmspace_getref(p, &vm)) != 0) |
5244 | goto out; | | 5244 | goto out; |
5245 | | | 5245 | |
5246 | map = &vm->vm_map; | | 5246 | map = &vm->vm_map; |
5247 | vm_map_lock_read(map); | | 5247 | vm_map_lock_read(map); |
5248 | | | 5248 | |
5249 | dp = oldp; | | 5249 | dp = oldp; |
5250 | if (oldp) | | 5250 | if (oldp) |
5251 | vme = kmem_alloc(vmesize, KM_SLEEP); | | 5251 | vme = kmem_alloc(vmesize, KM_SLEEP); |
5252 | for (entry = map->header.next; entry != &map->header; | | 5252 | for (entry = map->header.next; entry != &map->header; |
5253 | entry = entry->next) { | | 5253 | entry = entry->next) { |
5254 | if (oldp && (dp - (char *)oldp) < vmesize) { | | 5254 | if (oldp && (dp - (char *)oldp) < vmesize) { |
5255 | error = fill_vmentry(l, p, &vme[count], map, entry); | | 5255 | error = fill_vmentry(l, p, &vme[count], map, entry); |
5256 | if (error) | | 5256 | if (error) |
5257 | goto out; | | 5257 | goto out; |
5258 | dp += elem_size; | | 5258 | dp += elem_size; |
5259 | } | | 5259 | } |
5260 | count++; | | 5260 | count++; |
5261 | } | | 5261 | } |
5262 | vm_map_unlock_read(map); | | 5262 | vm_map_unlock_read(map); |
5263 | uvmspace_free(vm); | | 5263 | uvmspace_free(vm); |
5264 | | | 5264 | |
5265 | out: | | 5265 | out: |
5266 | if (pid != -1) | | 5266 | if (pid != -1) |
5267 | mutex_exit(p->p_lock); | | 5267 | mutex_exit(p->p_lock); |
5268 | if (error == 0) { | | 5268 | if (error == 0) { |
5269 | const u_int esize = uimin(sizeof(*vme), elem_size); | | 5269 | const u_int esize = uimin(sizeof(*vme), elem_size); |
5270 | dp = oldp; | | 5270 | dp = oldp; |
5271 | for (size_t i = 0; i < count; i++) { | | 5271 | for (size_t i = 0; i < count; i++) { |
5272 | if (oldp && (dp - (char *)oldp) < vmesize) { | | 5272 | if (oldp && (dp - (char *)oldp) < vmesize) { |
5273 | error = sysctl_copyout(l, &vme[i], dp, esize); | | 5273 | error = sysctl_copyout(l, &vme[i], dp, esize); |
5274 | if (error) | | 5274 | if (error) |
5275 | break; | | 5275 | break; |
5276 | dp += elem_size; | | 5276 | dp += elem_size; |
5277 | } else | | 5277 | } else |
5278 | break; | | 5278 | break; |
5279 | } | | 5279 | } |
5280 | count *= elem_size; | | 5280 | count *= elem_size; |
5281 | if (oldp != NULL && *oldlenp < count) | | 5281 | if (oldp != NULL && *oldlenp < count) |
5282 | error = ENOSPC; | | 5282 | error = ENOSPC; |
5283 | *oldlenp = count; | | 5283 | *oldlenp = count; |
5284 | } | | 5284 | } |
5285 | if (vme) | | 5285 | if (vme) |
5286 | kmem_free(vme, vmesize); | | 5286 | kmem_free(vme, vmesize); |
5287 | return error; | | 5287 | return error; |
5288 | } | | 5288 | } |
5289 | | | 5289 | |
5290 | static int | | 5290 | static int |
5291 | sysctl_vmproc(SYSCTLFN_ARGS) | | 5291 | sysctl_vmproc(SYSCTLFN_ARGS) |
5292 | { | | 5292 | { |
5293 | int error; | | 5293 | int error; |
5294 | | | 5294 | |
5295 | if (namelen == 1 && name[0] == CTL_QUERY) | | 5295 | if (namelen == 1 && name[0] == CTL_QUERY) |
5296 | return (sysctl_query(SYSCTLFN_CALL(rnode))); | | 5296 | return (sysctl_query(SYSCTLFN_CALL(rnode))); |
5297 | | | 5297 | |
5298 | if (namelen == 0) | | 5298 | if (namelen == 0) |
5299 | return EINVAL; | | 5299 | return EINVAL; |
5300 | | | 5300 | |
5301 | switch (name[0]) { | | 5301 | switch (name[0]) { |
5302 | case VM_PROC_MAP: | | 5302 | case VM_PROC_MAP: |
5303 | if (namelen != 3) | | 5303 | if (namelen != 3) |
5304 | return EINVAL; | | 5304 | return EINVAL; |
5305 | sysctl_unlock(); | | 5305 | sysctl_unlock(); |
5306 | error = fill_vmentries(l, name[1], name[2], oldp, oldlenp); | | 5306 | error = fill_vmentries(l, name[1], name[2], oldp, oldlenp); |
5307 | sysctl_relock(); | | 5307 | sysctl_relock(); |
5308 | return error; | | 5308 | return error; |
5309 | default: | | 5309 | default: |
5310 | return EINVAL; | | 5310 | return EINVAL; |
5311 | } | | 5311 | } |
5312 | } | | 5312 | } |
5313 | | | 5313 | |
5314 | SYSCTL_SETUP(sysctl_uvmmap_setup, "sysctl uvmmap setup") | | 5314 | SYSCTL_SETUP(sysctl_uvmmap_setup, "sysctl uvmmap setup") |
5315 | { | | 5315 | { |
5316 | | | 5316 | |
5317 | sysctl_createv(clog, 0, NULL, NULL, | | 5317 | sysctl_createv(clog, 0, NULL, NULL, |
5318 | CTLFLAG_PERMANENT, | | 5318 | CTLFLAG_PERMANENT, |
5319 | CTLTYPE_STRUCT, "proc", | | 5319 | CTLTYPE_STRUCT, "proc", |
5320 | SYSCTL_DESCR("Process vm information"), | | 5320 | SYSCTL_DESCR("Process vm information"), |
5321 | sysctl_vmproc, 0, NULL, 0, | | 5321 | sysctl_vmproc, 0, NULL, 0, |
5322 | CTL_VM, VM_PROC, CTL_EOL); | | 5322 | CTL_VM, VM_PROC, CTL_EOL); |
5323 | #ifndef __USER_VA0_IS_SAFE | | 5323 | #ifndef __USER_VA0_IS_SAFE |
5324 | sysctl_createv(clog, 0, NULL, NULL, | | 5324 | sysctl_createv(clog, 0, NULL, NULL, |
5325 | CTLFLAG_PERMANENT|CTLFLAG_READWRITE, | | 5325 | CTLFLAG_PERMANENT|CTLFLAG_READWRITE, |
5326 | CTLTYPE_INT, "user_va0_disable", | | 5326 | CTLTYPE_INT, "user_va0_disable", |
5327 | SYSCTL_DESCR("Disable VA 0"), | | 5327 | SYSCTL_DESCR("Disable VA 0"), |
5328 | sysctl_user_va0_disable, 0, &user_va0_disable, 0, | | 5328 | sysctl_user_va0_disable, 0, &user_va0_disable, 0, |
5329 | CTL_VM, CTL_CREATE, CTL_EOL); | | 5329 | CTL_VM, CTL_CREATE, CTL_EOL); |
5330 | #endif | | 5330 | #endif |
5331 | } | | 5331 | } |