Sat Apr 18 17:22:26 2020 UTC ()
Fix trailing whitespace.


(riastradh)
diff -r1.379 -r1.380 src/sys/uvm/uvm_map.c

cvs diff -r1.379 -r1.380 src/sys/uvm/uvm_map.c (switch to unified diff)

--- src/sys/uvm/uvm_map.c 2020/04/18 03:27:13 1.379
+++ src/sys/uvm/uvm_map.c 2020/04/18 17:22:26 1.380
@@ -1,1068 +1,1068 @@ @@ -1,1068 +1,1068 @@
1/* $NetBSD: uvm_map.c,v 1.379 2020/04/18 03:27:13 thorpej Exp $ */ 1/* $NetBSD: uvm_map.c,v 1.380 2020/04/18 17:22:26 riastradh Exp $ */
2 2
3/* 3/*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * Copyright (c) 1991, 1993, The Regents of the University of California. 5 * Copyright (c) 1991, 1993, The Regents of the University of California.
6 * 6 *
7 * All rights reserved. 7 * All rights reserved.
8 * 8 *
9 * This code is derived from software contributed to Berkeley by 9 * This code is derived from software contributed to Berkeley by
10 * The Mach Operating System project at Carnegie-Mellon University. 10 * The Mach Operating System project at Carnegie-Mellon University.
11 * 11 *
12 * Redistribution and use in source and binary forms, with or without 12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions 13 * modification, are permitted provided that the following conditions
14 * are met: 14 * are met:
15 * 1. Redistributions of source code must retain the above copyright 15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer. 16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright 17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the 18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution. 19 * documentation and/or other materials provided with the distribution.
20 * 3. Neither the name of the University nor the names of its contributors 20 * 3. Neither the name of the University nor the names of its contributors
21 * may be used to endorse or promote products derived from this software 21 * may be used to endorse or promote products derived from this software
22 * without specific prior written permission. 22 * without specific prior written permission.
23 * 23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE. 34 * SUCH DAMAGE.
35 * 35 *
36 * @(#)vm_map.c 8.3 (Berkeley) 1/12/94 36 * @(#)vm_map.c 8.3 (Berkeley) 1/12/94
37 * from: Id: uvm_map.c,v 1.1.2.27 1998/02/07 01:16:54 chs Exp 37 * from: Id: uvm_map.c,v 1.1.2.27 1998/02/07 01:16:54 chs Exp
38 * 38 *
39 * 39 *
40 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 40 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
41 * All rights reserved. 41 * All rights reserved.
42 * 42 *
43 * Permission to use, copy, modify and distribute this software and 43 * Permission to use, copy, modify and distribute this software and
44 * its documentation is hereby granted, provided that both the copyright 44 * its documentation is hereby granted, provided that both the copyright
45 * notice and this permission notice appear in all copies of the 45 * notice and this permission notice appear in all copies of the
46 * software, derivative works or modified versions, and any portions 46 * software, derivative works or modified versions, and any portions
47 * thereof, and that both notices appear in supporting documentation. 47 * thereof, and that both notices appear in supporting documentation.
48 * 48 *
49 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 49 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
50 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 50 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
51 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 51 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
52 * 52 *
53 * Carnegie Mellon requests users of this software to return to 53 * Carnegie Mellon requests users of this software to return to
54 * 54 *
55 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 55 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
56 * School of Computer Science 56 * School of Computer Science
57 * Carnegie Mellon University 57 * Carnegie Mellon University
58 * Pittsburgh PA 15213-3890 58 * Pittsburgh PA 15213-3890
59 * 59 *
60 * any improvements or extensions that they make and grant Carnegie the 60 * any improvements or extensions that they make and grant Carnegie the
61 * rights to redistribute these changes. 61 * rights to redistribute these changes.
62 */ 62 */
63 63
64/* 64/*
65 * uvm_map.c: uvm map operations 65 * uvm_map.c: uvm map operations
66 */ 66 */
67 67
68#include <sys/cdefs.h> 68#include <sys/cdefs.h>
69__KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.379 2020/04/18 03:27:13 thorpej Exp $"); 69__KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.380 2020/04/18 17:22:26 riastradh Exp $");
70 70
71#include "opt_ddb.h" 71#include "opt_ddb.h"
72#include "opt_pax.h" 72#include "opt_pax.h"
73#include "opt_uvmhist.h" 73#include "opt_uvmhist.h"
74#include "opt_uvm.h" 74#include "opt_uvm.h"
75#include "opt_sysv.h" 75#include "opt_sysv.h"
76 76
77#include <sys/param.h> 77#include <sys/param.h>
78#include <sys/systm.h> 78#include <sys/systm.h>
79#include <sys/mman.h> 79#include <sys/mman.h>
80#include <sys/proc.h> 80#include <sys/proc.h>
81#include <sys/pool.h> 81#include <sys/pool.h>
82#include <sys/kernel.h> 82#include <sys/kernel.h>
83#include <sys/mount.h> 83#include <sys/mount.h>
84#include <sys/pax.h> 84#include <sys/pax.h>
85#include <sys/vnode.h> 85#include <sys/vnode.h>
86#include <sys/filedesc.h> 86#include <sys/filedesc.h>
87#include <sys/lockdebug.h> 87#include <sys/lockdebug.h>
88#include <sys/atomic.h> 88#include <sys/atomic.h>
89#include <sys/sysctl.h> 89#include <sys/sysctl.h>
90#ifndef __USER_VA0_IS_SAFE 90#ifndef __USER_VA0_IS_SAFE
91#include <sys/kauth.h> 91#include <sys/kauth.h>
92#include "opt_user_va0_disable_default.h" 92#include "opt_user_va0_disable_default.h"
93#endif 93#endif
94 94
95#include <sys/shm.h> 95#include <sys/shm.h>
96 96
97#include <uvm/uvm.h> 97#include <uvm/uvm.h>
98#include <uvm/uvm_readahead.h> 98#include <uvm/uvm_readahead.h>
99 99
100#if defined(DDB) || defined(DEBUGPRINT) 100#if defined(DDB) || defined(DEBUGPRINT)
101#include <uvm/uvm_ddb.h> 101#include <uvm/uvm_ddb.h>
102#endif 102#endif
103 103
104#ifdef UVMHIST 104#ifdef UVMHIST
105#ifndef UVMHIST_MAPHIST_SIZE 105#ifndef UVMHIST_MAPHIST_SIZE
106#define UVMHIST_MAPHIST_SIZE 100 106#define UVMHIST_MAPHIST_SIZE 100
107#endif 107#endif
108#ifndef UVMHIST_PDHIST_SIZE 108#ifndef UVMHIST_PDHIST_SIZE
109#define UVMHIST_PDHIST_SIZE 100 109#define UVMHIST_PDHIST_SIZE 100
110#endif 110#endif
111static struct kern_history_ent maphistbuf[UVMHIST_MAPHIST_SIZE]; 111static struct kern_history_ent maphistbuf[UVMHIST_MAPHIST_SIZE];
112UVMHIST_DEFINE(maphist) = UVMHIST_INITIALIZER(maphist, maphistbuf); 112UVMHIST_DEFINE(maphist) = UVMHIST_INITIALIZER(maphist, maphistbuf);
113#endif 113#endif
114 114
115#if !defined(UVMMAP_COUNTERS) 115#if !defined(UVMMAP_COUNTERS)
116 116
117#define UVMMAP_EVCNT_DEFINE(name) /* nothing */ 117#define UVMMAP_EVCNT_DEFINE(name) /* nothing */
118#define UVMMAP_EVCNT_INCR(ev) /* nothing */ 118#define UVMMAP_EVCNT_INCR(ev) /* nothing */
119#define UVMMAP_EVCNT_DECR(ev) /* nothing */ 119#define UVMMAP_EVCNT_DECR(ev) /* nothing */
120 120
121#else /* defined(UVMMAP_NOCOUNTERS) */ 121#else /* defined(UVMMAP_NOCOUNTERS) */
122 122
123#include <sys/evcnt.h> 123#include <sys/evcnt.h>
124#define UVMMAP_EVCNT_DEFINE(name) \ 124#define UVMMAP_EVCNT_DEFINE(name) \
125struct evcnt uvmmap_evcnt_##name = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, \ 125struct evcnt uvmmap_evcnt_##name = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, \
126 "uvmmap", #name); \ 126 "uvmmap", #name); \
127EVCNT_ATTACH_STATIC(uvmmap_evcnt_##name); 127EVCNT_ATTACH_STATIC(uvmmap_evcnt_##name);
128#define UVMMAP_EVCNT_INCR(ev) uvmmap_evcnt_##ev.ev_count++ 128#define UVMMAP_EVCNT_INCR(ev) uvmmap_evcnt_##ev.ev_count++
129#define UVMMAP_EVCNT_DECR(ev) uvmmap_evcnt_##ev.ev_count-- 129#define UVMMAP_EVCNT_DECR(ev) uvmmap_evcnt_##ev.ev_count--
130 130
131#endif /* defined(UVMMAP_NOCOUNTERS) */ 131#endif /* defined(UVMMAP_NOCOUNTERS) */
132 132
133UVMMAP_EVCNT_DEFINE(ubackmerge) 133UVMMAP_EVCNT_DEFINE(ubackmerge)
134UVMMAP_EVCNT_DEFINE(uforwmerge) 134UVMMAP_EVCNT_DEFINE(uforwmerge)
135UVMMAP_EVCNT_DEFINE(ubimerge) 135UVMMAP_EVCNT_DEFINE(ubimerge)
136UVMMAP_EVCNT_DEFINE(unomerge) 136UVMMAP_EVCNT_DEFINE(unomerge)
137UVMMAP_EVCNT_DEFINE(kbackmerge) 137UVMMAP_EVCNT_DEFINE(kbackmerge)
138UVMMAP_EVCNT_DEFINE(kforwmerge) 138UVMMAP_EVCNT_DEFINE(kforwmerge)
139UVMMAP_EVCNT_DEFINE(kbimerge) 139UVMMAP_EVCNT_DEFINE(kbimerge)
140UVMMAP_EVCNT_DEFINE(knomerge) 140UVMMAP_EVCNT_DEFINE(knomerge)
141UVMMAP_EVCNT_DEFINE(map_call) 141UVMMAP_EVCNT_DEFINE(map_call)
142UVMMAP_EVCNT_DEFINE(mlk_call) 142UVMMAP_EVCNT_DEFINE(mlk_call)
143UVMMAP_EVCNT_DEFINE(mlk_hint) 143UVMMAP_EVCNT_DEFINE(mlk_hint)
144UVMMAP_EVCNT_DEFINE(mlk_tree) 144UVMMAP_EVCNT_DEFINE(mlk_tree)
145UVMMAP_EVCNT_DEFINE(mlk_treeloop) 145UVMMAP_EVCNT_DEFINE(mlk_treeloop)
146 146
147const char vmmapbsy[] = "vmmapbsy"; 147const char vmmapbsy[] = "vmmapbsy";
148 148
149/* 149/*
150 * cache for vmspace structures. 150 * cache for vmspace structures.
151 */ 151 */
152 152
153static struct pool_cache uvm_vmspace_cache; 153static struct pool_cache uvm_vmspace_cache;
154 154
155/* 155/*
156 * cache for dynamically-allocated map entries. 156 * cache for dynamically-allocated map entries.
157 */ 157 */
158 158
159static struct pool_cache uvm_map_entry_cache; 159static struct pool_cache uvm_map_entry_cache;
160 160
161#ifdef PMAP_GROWKERNEL 161#ifdef PMAP_GROWKERNEL
162/* 162/*
163 * This global represents the end of the kernel virtual address 163 * This global represents the end of the kernel virtual address
164 * space. If we want to exceed this, we must grow the kernel 164 * space. If we want to exceed this, we must grow the kernel
165 * virtual address space dynamically. 165 * virtual address space dynamically.
166 * 166 *
167 * Note, this variable is locked by kernel_map's lock. 167 * Note, this variable is locked by kernel_map's lock.
168 */ 168 */
169vaddr_t uvm_maxkaddr; 169vaddr_t uvm_maxkaddr;
170#endif 170#endif
171 171
172#ifndef __USER_VA0_IS_SAFE 172#ifndef __USER_VA0_IS_SAFE
173#ifndef __USER_VA0_DISABLE_DEFAULT 173#ifndef __USER_VA0_DISABLE_DEFAULT
174#define __USER_VA0_DISABLE_DEFAULT 1 174#define __USER_VA0_DISABLE_DEFAULT 1
175#endif 175#endif
176#ifdef USER_VA0_DISABLE_DEFAULT /* kernel config option overrides */ 176#ifdef USER_VA0_DISABLE_DEFAULT /* kernel config option overrides */
177#undef __USER_VA0_DISABLE_DEFAULT 177#undef __USER_VA0_DISABLE_DEFAULT
178#define __USER_VA0_DISABLE_DEFAULT USER_VA0_DISABLE_DEFAULT 178#define __USER_VA0_DISABLE_DEFAULT USER_VA0_DISABLE_DEFAULT
179#endif 179#endif
180int user_va0_disable = __USER_VA0_DISABLE_DEFAULT; 180int user_va0_disable = __USER_VA0_DISABLE_DEFAULT;
181#endif 181#endif
182 182
183/* 183/*
184 * macros 184 * macros
185 */ 185 */
186 186
187/* 187/*
188 * uvm_map_align_va: round down or up virtual address 188 * uvm_map_align_va: round down or up virtual address
189 */ 189 */
190static __inline void 190static __inline void
191uvm_map_align_va(vaddr_t *vap, vsize_t align, int topdown) 191uvm_map_align_va(vaddr_t *vap, vsize_t align, int topdown)
192{ 192{
193 193
194 KASSERT(powerof2(align)); 194 KASSERT(powerof2(align));
195 195
196 if (align != 0 && (*vap & (align - 1)) != 0) { 196 if (align != 0 && (*vap & (align - 1)) != 0) {
197 if (topdown) 197 if (topdown)
198 *vap = rounddown2(*vap, align); 198 *vap = rounddown2(*vap, align);
199 else 199 else
200 *vap = roundup2(*vap, align); 200 *vap = roundup2(*vap, align);
201 } 201 }
202} 202}
203 203
204/* 204/*
205 * UVM_ET_ISCOMPATIBLE: check some requirements for map entry merging 205 * UVM_ET_ISCOMPATIBLE: check some requirements for map entry merging
206 */ 206 */
207extern struct vm_map *pager_map; 207extern struct vm_map *pager_map;
208 208
209#define UVM_ET_ISCOMPATIBLE(ent, type, uobj, meflags, \ 209#define UVM_ET_ISCOMPATIBLE(ent, type, uobj, meflags, \
210 prot, maxprot, inh, adv, wire) \ 210 prot, maxprot, inh, adv, wire) \
211 ((ent)->etype == (type) && \ 211 ((ent)->etype == (type) && \
212 (((ent)->flags ^ (meflags)) & (UVM_MAP_NOMERGE)) == 0 && \ 212 (((ent)->flags ^ (meflags)) & (UVM_MAP_NOMERGE)) == 0 && \
213 (ent)->object.uvm_obj == (uobj) && \ 213 (ent)->object.uvm_obj == (uobj) && \
214 (ent)->protection == (prot) && \ 214 (ent)->protection == (prot) && \
215 (ent)->max_protection == (maxprot) && \ 215 (ent)->max_protection == (maxprot) && \
216 (ent)->inheritance == (inh) && \ 216 (ent)->inheritance == (inh) && \
217 (ent)->advice == (adv) && \ 217 (ent)->advice == (adv) && \
218 (ent)->wired_count == (wire)) 218 (ent)->wired_count == (wire))
219 219
220/* 220/*
221 * uvm_map_entry_link: insert entry into a map 221 * uvm_map_entry_link: insert entry into a map
222 * 222 *
223 * => map must be locked 223 * => map must be locked
224 */ 224 */
225#define uvm_map_entry_link(map, after_where, entry) do { \ 225#define uvm_map_entry_link(map, after_where, entry) do { \
226 uvm_mapent_check(entry); \ 226 uvm_mapent_check(entry); \
227 (map)->nentries++; \ 227 (map)->nentries++; \
228 (entry)->prev = (after_where); \ 228 (entry)->prev = (after_where); \
229 (entry)->next = (after_where)->next; \ 229 (entry)->next = (after_where)->next; \
230 (entry)->prev->next = (entry); \ 230 (entry)->prev->next = (entry); \
231 (entry)->next->prev = (entry); \ 231 (entry)->next->prev = (entry); \
232 uvm_rb_insert((map), (entry)); \ 232 uvm_rb_insert((map), (entry)); \
233} while (/*CONSTCOND*/ 0) 233} while (/*CONSTCOND*/ 0)
234 234
235/* 235/*
236 * uvm_map_entry_unlink: remove entry from a map 236 * uvm_map_entry_unlink: remove entry from a map
237 * 237 *
238 * => map must be locked 238 * => map must be locked
239 */ 239 */
240#define uvm_map_entry_unlink(map, entry) do { \ 240#define uvm_map_entry_unlink(map, entry) do { \
241 KASSERT((entry) != (map)->first_free); \ 241 KASSERT((entry) != (map)->first_free); \
242 KASSERT((entry) != (map)->hint); \ 242 KASSERT((entry) != (map)->hint); \
243 uvm_mapent_check(entry); \ 243 uvm_mapent_check(entry); \
244 (map)->nentries--; \ 244 (map)->nentries--; \
245 (entry)->next->prev = (entry)->prev; \ 245 (entry)->next->prev = (entry)->prev; \
246 (entry)->prev->next = (entry)->next; \ 246 (entry)->prev->next = (entry)->next; \
247 uvm_rb_remove((map), (entry)); \ 247 uvm_rb_remove((map), (entry)); \
248} while (/*CONSTCOND*/ 0) 248} while (/*CONSTCOND*/ 0)
249 249
250/* 250/*
251 * SAVE_HINT: saves the specified entry as the hint for future lookups. 251 * SAVE_HINT: saves the specified entry as the hint for future lookups.
252 * 252 *
253 * => map need not be locked. 253 * => map need not be locked.
254 */ 254 */
255#define SAVE_HINT(map, check, value) do { \ 255#define SAVE_HINT(map, check, value) do { \
256 if ((map)->hint == (check)) \ 256 if ((map)->hint == (check)) \
257 (map)->hint = (value); \ 257 (map)->hint = (value); \
258} while (/*CONSTCOND*/ 0) 258} while (/*CONSTCOND*/ 0)
259 259
260/* 260/*
261 * clear_hints: ensure that hints don't point to the entry. 261 * clear_hints: ensure that hints don't point to the entry.
262 * 262 *
263 * => map must be write-locked. 263 * => map must be write-locked.
264 */ 264 */
265static void 265static void
266clear_hints(struct vm_map *map, struct vm_map_entry *ent) 266clear_hints(struct vm_map *map, struct vm_map_entry *ent)
267{ 267{
268 268
269 SAVE_HINT(map, ent, ent->prev); 269 SAVE_HINT(map, ent, ent->prev);
270 if (map->first_free == ent) { 270 if (map->first_free == ent) {
271 map->first_free = ent->prev; 271 map->first_free = ent->prev;
272 } 272 }
273} 273}
274 274
275/* 275/*
276 * VM_MAP_RANGE_CHECK: check and correct range 276 * VM_MAP_RANGE_CHECK: check and correct range
277 * 277 *
278 * => map must at least be read locked 278 * => map must at least be read locked
279 */ 279 */
280 280
281#define VM_MAP_RANGE_CHECK(map, start, end) do { \ 281#define VM_MAP_RANGE_CHECK(map, start, end) do { \
282 if (start < vm_map_min(map)) \ 282 if (start < vm_map_min(map)) \
283 start = vm_map_min(map); \ 283 start = vm_map_min(map); \
284 if (end > vm_map_max(map)) \ 284 if (end > vm_map_max(map)) \
285 end = vm_map_max(map); \ 285 end = vm_map_max(map); \
286 if (start > end) \ 286 if (start > end) \
287 start = end; \ 287 start = end; \
288} while (/*CONSTCOND*/ 0) 288} while (/*CONSTCOND*/ 0)
289 289
290/* 290/*
291 * local prototypes 291 * local prototypes
292 */ 292 */
293 293
294static struct vm_map_entry * 294static struct vm_map_entry *
295 uvm_mapent_alloc(struct vm_map *, int); 295 uvm_mapent_alloc(struct vm_map *, int);
296static void uvm_mapent_copy(struct vm_map_entry *, struct vm_map_entry *); 296static void uvm_mapent_copy(struct vm_map_entry *, struct vm_map_entry *);
297static void uvm_mapent_free(struct vm_map_entry *); 297static void uvm_mapent_free(struct vm_map_entry *);
298#if defined(DEBUG) 298#if defined(DEBUG)
299static void _uvm_mapent_check(const struct vm_map_entry *, const char *, 299static void _uvm_mapent_check(const struct vm_map_entry *, const char *,
300 int); 300 int);
301#define uvm_mapent_check(map) _uvm_mapent_check(map, __FILE__, __LINE__) 301#define uvm_mapent_check(map) _uvm_mapent_check(map, __FILE__, __LINE__)
302#else /* defined(DEBUG) */ 302#else /* defined(DEBUG) */
303#define uvm_mapent_check(e) /* nothing */ 303#define uvm_mapent_check(e) /* nothing */
304#endif /* defined(DEBUG) */ 304#endif /* defined(DEBUG) */
305 305
306static void uvm_map_entry_unwire(struct vm_map *, struct vm_map_entry *); 306static void uvm_map_entry_unwire(struct vm_map *, struct vm_map_entry *);
307static void uvm_map_reference_amap(struct vm_map_entry *, int); 307static void uvm_map_reference_amap(struct vm_map_entry *, int);
308static int uvm_map_space_avail(vaddr_t *, vsize_t, voff_t, vsize_t, int, 308static int uvm_map_space_avail(vaddr_t *, vsize_t, voff_t, vsize_t, int,
309 int, struct vm_map_entry *); 309 int, struct vm_map_entry *);
310static void uvm_map_unreference_amap(struct vm_map_entry *, int); 310static void uvm_map_unreference_amap(struct vm_map_entry *, int);
311 311
312int _uvm_map_sanity(struct vm_map *); 312int _uvm_map_sanity(struct vm_map *);
313int _uvm_tree_sanity(struct vm_map *); 313int _uvm_tree_sanity(struct vm_map *);
314static vsize_t uvm_rb_maxgap(const struct vm_map_entry *); 314static vsize_t uvm_rb_maxgap(const struct vm_map_entry *);
315 315
316#define ROOT_ENTRY(map) ((struct vm_map_entry *)(map)->rb_tree.rbt_root) 316#define ROOT_ENTRY(map) ((struct vm_map_entry *)(map)->rb_tree.rbt_root)
317#define LEFT_ENTRY(entry) ((struct vm_map_entry *)(entry)->rb_node.rb_left) 317#define LEFT_ENTRY(entry) ((struct vm_map_entry *)(entry)->rb_node.rb_left)
318#define RIGHT_ENTRY(entry) ((struct vm_map_entry *)(entry)->rb_node.rb_right) 318#define RIGHT_ENTRY(entry) ((struct vm_map_entry *)(entry)->rb_node.rb_right)
319#define PARENT_ENTRY(map, entry) \ 319#define PARENT_ENTRY(map, entry) \
320 (ROOT_ENTRY(map) == (entry) \ 320 (ROOT_ENTRY(map) == (entry) \
321 ? NULL : (struct vm_map_entry *)RB_FATHER(&(entry)->rb_node)) 321 ? NULL : (struct vm_map_entry *)RB_FATHER(&(entry)->rb_node))
322 322
323/* 323/*
324 * These get filled in if/when SYSVSHM shared memory code is loaded 324 * These get filled in if/when SYSVSHM shared memory code is loaded
325 * 325 *
326 * We do this with function pointers rather the #ifdef SYSVSHM so the 326 * We do this with function pointers rather the #ifdef SYSVSHM so the
327 * SYSVSHM code can be loaded and unloaded 327 * SYSVSHM code can be loaded and unloaded
328 */ 328 */
329void (*uvm_shmexit)(struct vmspace *) = NULL; 329void (*uvm_shmexit)(struct vmspace *) = NULL;
330void (*uvm_shmfork)(struct vmspace *, struct vmspace *) = NULL; 330void (*uvm_shmfork)(struct vmspace *, struct vmspace *) = NULL;
331 331
332static int 332static int
333uvm_map_compare_nodes(void *ctx, const void *nparent, const void *nkey) 333uvm_map_compare_nodes(void *ctx, const void *nparent, const void *nkey)
334{ 334{
335 const struct vm_map_entry *eparent = nparent; 335 const struct vm_map_entry *eparent = nparent;
336 const struct vm_map_entry *ekey = nkey; 336 const struct vm_map_entry *ekey = nkey;
337 337
338 KASSERT(eparent->start < ekey->start || eparent->start >= ekey->end); 338 KASSERT(eparent->start < ekey->start || eparent->start >= ekey->end);
339 KASSERT(ekey->start < eparent->start || ekey->start >= eparent->end); 339 KASSERT(ekey->start < eparent->start || ekey->start >= eparent->end);
340 340
341 if (eparent->start < ekey->start) 341 if (eparent->start < ekey->start)
342 return -1; 342 return -1;
343 if (eparent->end >= ekey->start) 343 if (eparent->end >= ekey->start)
344 return 1; 344 return 1;
345 return 0; 345 return 0;
346} 346}
347 347
348static int 348static int
349uvm_map_compare_key(void *ctx, const void *nparent, const void *vkey) 349uvm_map_compare_key(void *ctx, const void *nparent, const void *vkey)
350{ 350{
351 const struct vm_map_entry *eparent = nparent; 351 const struct vm_map_entry *eparent = nparent;
352 const vaddr_t va = *(const vaddr_t *) vkey; 352 const vaddr_t va = *(const vaddr_t *) vkey;
353 353
354 if (eparent->start < va) 354 if (eparent->start < va)
355 return -1; 355 return -1;
356 if (eparent->end >= va) 356 if (eparent->end >= va)
357 return 1; 357 return 1;
358 return 0; 358 return 0;
359} 359}
360 360
361static const rb_tree_ops_t uvm_map_tree_ops = { 361static const rb_tree_ops_t uvm_map_tree_ops = {
362 .rbto_compare_nodes = uvm_map_compare_nodes, 362 .rbto_compare_nodes = uvm_map_compare_nodes,
363 .rbto_compare_key = uvm_map_compare_key, 363 .rbto_compare_key = uvm_map_compare_key,
364 .rbto_node_offset = offsetof(struct vm_map_entry, rb_node), 364 .rbto_node_offset = offsetof(struct vm_map_entry, rb_node),
365 .rbto_context = NULL 365 .rbto_context = NULL
366}; 366};
367 367
368/* 368/*
369 * uvm_rb_gap: return the gap size between our entry and next entry. 369 * uvm_rb_gap: return the gap size between our entry and next entry.
370 */ 370 */
371static inline vsize_t 371static inline vsize_t
372uvm_rb_gap(const struct vm_map_entry *entry) 372uvm_rb_gap(const struct vm_map_entry *entry)
373{ 373{
374 374
375 KASSERT(entry->next != NULL); 375 KASSERT(entry->next != NULL);
376 return entry->next->start - entry->end; 376 return entry->next->start - entry->end;
377} 377}
378 378
379static vsize_t 379static vsize_t
380uvm_rb_maxgap(const struct vm_map_entry *entry) 380uvm_rb_maxgap(const struct vm_map_entry *entry)
381{ 381{
382 struct vm_map_entry *child; 382 struct vm_map_entry *child;
383 vsize_t maxgap = entry->gap; 383 vsize_t maxgap = entry->gap;
384 384
385 /* 385 /*
386 * We need maxgap to be the largest gap of us or any of our 386 * We need maxgap to be the largest gap of us or any of our
387 * descendents. Since each of our children's maxgap is the 387 * descendents. Since each of our children's maxgap is the
388 * cached value of their largest gap of themselves or their 388 * cached value of their largest gap of themselves or their
389 * descendents, we can just use that value and avoid recursing 389 * descendents, we can just use that value and avoid recursing
390 * down the tree to calculate it. 390 * down the tree to calculate it.
391 */ 391 */
392 if ((child = LEFT_ENTRY(entry)) != NULL && maxgap < child->maxgap) 392 if ((child = LEFT_ENTRY(entry)) != NULL && maxgap < child->maxgap)
393 maxgap = child->maxgap; 393 maxgap = child->maxgap;
394 394
395 if ((child = RIGHT_ENTRY(entry)) != NULL && maxgap < child->maxgap) 395 if ((child = RIGHT_ENTRY(entry)) != NULL && maxgap < child->maxgap)
396 maxgap = child->maxgap; 396 maxgap = child->maxgap;
397 397
398 return maxgap; 398 return maxgap;
399} 399}
400 400
401static void 401static void
402uvm_rb_fixup(struct vm_map *map, struct vm_map_entry *entry) 402uvm_rb_fixup(struct vm_map *map, struct vm_map_entry *entry)
403{ 403{
404 struct vm_map_entry *parent; 404 struct vm_map_entry *parent;
405 405
406 KASSERT(entry->gap == uvm_rb_gap(entry)); 406 KASSERT(entry->gap == uvm_rb_gap(entry));
407 entry->maxgap = uvm_rb_maxgap(entry); 407 entry->maxgap = uvm_rb_maxgap(entry);
408 408
409 while ((parent = PARENT_ENTRY(map, entry)) != NULL) { 409 while ((parent = PARENT_ENTRY(map, entry)) != NULL) {
410 struct vm_map_entry *brother; 410 struct vm_map_entry *brother;
411 vsize_t maxgap = parent->gap; 411 vsize_t maxgap = parent->gap;
412 unsigned int which; 412 unsigned int which;
413 413
414 KDASSERT(parent->gap == uvm_rb_gap(parent)); 414 KDASSERT(parent->gap == uvm_rb_gap(parent));
415 if (maxgap < entry->maxgap) 415 if (maxgap < entry->maxgap)
416 maxgap = entry->maxgap; 416 maxgap = entry->maxgap;
417 /* 417 /*
418 * Since we work towards the root, we know entry's maxgap 418 * Since we work towards the root, we know entry's maxgap
419 * value is OK, but its brothers may now be out-of-date due 419 * value is OK, but its brothers may now be out-of-date due
420 * to rebalancing. So refresh it. 420 * to rebalancing. So refresh it.
421 */ 421 */
422 which = RB_POSITION(&entry->rb_node) ^ RB_DIR_OTHER; 422 which = RB_POSITION(&entry->rb_node) ^ RB_DIR_OTHER;
423 brother = (struct vm_map_entry *)parent->rb_node.rb_nodes[which]; 423 brother = (struct vm_map_entry *)parent->rb_node.rb_nodes[which];
424 if (brother != NULL) { 424 if (brother != NULL) {
425 KDASSERT(brother->gap == uvm_rb_gap(brother)); 425 KDASSERT(brother->gap == uvm_rb_gap(brother));
426 brother->maxgap = uvm_rb_maxgap(brother); 426 brother->maxgap = uvm_rb_maxgap(brother);
427 if (maxgap < brother->maxgap) 427 if (maxgap < brother->maxgap)
428 maxgap = brother->maxgap; 428 maxgap = brother->maxgap;
429 } 429 }
430 430
431 parent->maxgap = maxgap; 431 parent->maxgap = maxgap;
432 entry = parent; 432 entry = parent;
433 } 433 }
434} 434}
435 435
436static void 436static void
437uvm_rb_insert(struct vm_map *map, struct vm_map_entry *entry) 437uvm_rb_insert(struct vm_map *map, struct vm_map_entry *entry)
438{ 438{
439 struct vm_map_entry *ret __diagused; 439 struct vm_map_entry *ret __diagused;
440 440
441 entry->gap = entry->maxgap = uvm_rb_gap(entry); 441 entry->gap = entry->maxgap = uvm_rb_gap(entry);
442 if (entry->prev != &map->header) 442 if (entry->prev != &map->header)
443 entry->prev->gap = uvm_rb_gap(entry->prev); 443 entry->prev->gap = uvm_rb_gap(entry->prev);
444 444
445 ret = rb_tree_insert_node(&map->rb_tree, entry); 445 ret = rb_tree_insert_node(&map->rb_tree, entry);
446 KASSERTMSG(ret == entry, 446 KASSERTMSG(ret == entry,
447 "uvm_rb_insert: map %p: duplicate entry %p", map, ret); 447 "uvm_rb_insert: map %p: duplicate entry %p", map, ret);
448 448
449 /* 449 /*
450 * If the previous entry is not our immediate left child, then it's an 450 * If the previous entry is not our immediate left child, then it's an
451 * ancestor and will be fixed up on the way to the root. We don't 451 * ancestor and will be fixed up on the way to the root. We don't
452 * have to check entry->prev against &map->header since &map->header 452 * have to check entry->prev against &map->header since &map->header
453 * will never be in the tree. 453 * will never be in the tree.
454 */ 454 */
455 uvm_rb_fixup(map, 455 uvm_rb_fixup(map,
456 LEFT_ENTRY(entry) == entry->prev ? entry->prev : entry); 456 LEFT_ENTRY(entry) == entry->prev ? entry->prev : entry);
457} 457}
458 458
459static void 459static void
460uvm_rb_remove(struct vm_map *map, struct vm_map_entry *entry) 460uvm_rb_remove(struct vm_map *map, struct vm_map_entry *entry)
461{ 461{
462 struct vm_map_entry *prev_parent = NULL, *next_parent = NULL; 462 struct vm_map_entry *prev_parent = NULL, *next_parent = NULL;
463 463
464 /* 464 /*
465 * If we are removing an interior node, then an adjacent node will 465 * If we are removing an interior node, then an adjacent node will
466 * be used to replace its position in the tree. Therefore we will 466 * be used to replace its position in the tree. Therefore we will
467 * need to fixup the tree starting at the parent of the replacement 467 * need to fixup the tree starting at the parent of the replacement
468 * node. So record their parents for later use. 468 * node. So record their parents for later use.
469 */ 469 */
470 if (entry->prev != &map->header) 470 if (entry->prev != &map->header)
471 prev_parent = PARENT_ENTRY(map, entry->prev); 471 prev_parent = PARENT_ENTRY(map, entry->prev);
472 if (entry->next != &map->header) 472 if (entry->next != &map->header)
473 next_parent = PARENT_ENTRY(map, entry->next); 473 next_parent = PARENT_ENTRY(map, entry->next);
474 474
475 rb_tree_remove_node(&map->rb_tree, entry); 475 rb_tree_remove_node(&map->rb_tree, entry);
476 476
477 /* 477 /*
478 * If the previous node has a new parent, fixup the tree starting 478 * If the previous node has a new parent, fixup the tree starting
479 * at the previous node's old parent. 479 * at the previous node's old parent.
480 */ 480 */
481 if (entry->prev != &map->header) { 481 if (entry->prev != &map->header) {
482 /* 482 /*
483 * Update the previous entry's gap due to our absence. 483 * Update the previous entry's gap due to our absence.
484 */ 484 */
485 entry->prev->gap = uvm_rb_gap(entry->prev); 485 entry->prev->gap = uvm_rb_gap(entry->prev);
486 uvm_rb_fixup(map, entry->prev); 486 uvm_rb_fixup(map, entry->prev);
487 if (prev_parent != NULL 487 if (prev_parent != NULL
488 && prev_parent != entry 488 && prev_parent != entry
489 && prev_parent != PARENT_ENTRY(map, entry->prev)) 489 && prev_parent != PARENT_ENTRY(map, entry->prev))
490 uvm_rb_fixup(map, prev_parent); 490 uvm_rb_fixup(map, prev_parent);
491 } 491 }
492 492
493 /* 493 /*
494 * If the next node has a new parent, fixup the tree starting 494 * If the next node has a new parent, fixup the tree starting
495 * at the next node's old parent. 495 * at the next node's old parent.
496 */ 496 */
497 if (entry->next != &map->header) { 497 if (entry->next != &map->header) {
498 uvm_rb_fixup(map, entry->next); 498 uvm_rb_fixup(map, entry->next);
499 if (next_parent != NULL 499 if (next_parent != NULL
500 && next_parent != entry 500 && next_parent != entry
501 && next_parent != PARENT_ENTRY(map, entry->next)) 501 && next_parent != PARENT_ENTRY(map, entry->next))
502 uvm_rb_fixup(map, next_parent); 502 uvm_rb_fixup(map, next_parent);
503 } 503 }
504} 504}
505 505
506#if defined(DEBUG) 506#if defined(DEBUG)
507int uvm_debug_check_map = 0; 507int uvm_debug_check_map = 0;
508int uvm_debug_check_rbtree = 0; 508int uvm_debug_check_rbtree = 0;
509#define uvm_map_check(map, name) \ 509#define uvm_map_check(map, name) \
510 _uvm_map_check((map), (name), __FILE__, __LINE__) 510 _uvm_map_check((map), (name), __FILE__, __LINE__)
511static void 511static void
512_uvm_map_check(struct vm_map *map, const char *name, 512_uvm_map_check(struct vm_map *map, const char *name,
513 const char *file, int line) 513 const char *file, int line)
514{ 514{
515 515
516 if ((uvm_debug_check_map && _uvm_map_sanity(map)) || 516 if ((uvm_debug_check_map && _uvm_map_sanity(map)) ||
517 (uvm_debug_check_rbtree && _uvm_tree_sanity(map))) { 517 (uvm_debug_check_rbtree && _uvm_tree_sanity(map))) {
518 panic("uvm_map_check failed: \"%s\" map=%p (%s:%d)", 518 panic("uvm_map_check failed: \"%s\" map=%p (%s:%d)",
519 name, map, file, line); 519 name, map, file, line);
520 } 520 }
521} 521}
522#else /* defined(DEBUG) */ 522#else /* defined(DEBUG) */
523#define uvm_map_check(map, name) /* nothing */ 523#define uvm_map_check(map, name) /* nothing */
524#endif /* defined(DEBUG) */ 524#endif /* defined(DEBUG) */
525 525
526#if defined(DEBUG) || defined(DDB) 526#if defined(DEBUG) || defined(DDB)
527int 527int
528_uvm_map_sanity(struct vm_map *map) 528_uvm_map_sanity(struct vm_map *map)
529{ 529{
530 bool first_free_found = false; 530 bool first_free_found = false;
531 bool hint_found = false; 531 bool hint_found = false;
532 const struct vm_map_entry *e; 532 const struct vm_map_entry *e;
533 struct vm_map_entry *hint = map->hint; 533 struct vm_map_entry *hint = map->hint;
534 534
535 e = &map->header; 535 e = &map->header;
536 for (;;) { 536 for (;;) {
537 if (map->first_free == e) { 537 if (map->first_free == e) {
538 first_free_found = true; 538 first_free_found = true;
539 } else if (!first_free_found && e->next->start > e->end) { 539 } else if (!first_free_found && e->next->start > e->end) {
540 printf("first_free %p should be %p\n", 540 printf("first_free %p should be %p\n",
541 map->first_free, e); 541 map->first_free, e);
542 return -1; 542 return -1;
543 } 543 }
544 if (hint == e) { 544 if (hint == e) {
545 hint_found = true; 545 hint_found = true;
546 } 546 }
547 547
548 e = e->next; 548 e = e->next;
549 if (e == &map->header) { 549 if (e == &map->header) {
550 break; 550 break;
551 } 551 }
552 } 552 }
553 if (!first_free_found) { 553 if (!first_free_found) {
554 printf("stale first_free\n"); 554 printf("stale first_free\n");
555 return -1; 555 return -1;
556 } 556 }
557 if (!hint_found) { 557 if (!hint_found) {
558 printf("stale hint\n"); 558 printf("stale hint\n");
559 return -1; 559 return -1;
560 } 560 }
561 return 0; 561 return 0;
562} 562}
563 563
564int 564int
565_uvm_tree_sanity(struct vm_map *map) 565_uvm_tree_sanity(struct vm_map *map)
566{ 566{
567 struct vm_map_entry *tmp, *trtmp; 567 struct vm_map_entry *tmp, *trtmp;
568 int n = 0, i = 1; 568 int n = 0, i = 1;
569 569
570 for (tmp = map->header.next; tmp != &map->header; tmp = tmp->next) { 570 for (tmp = map->header.next; tmp != &map->header; tmp = tmp->next) {
571 if (tmp->gap != uvm_rb_gap(tmp)) { 571 if (tmp->gap != uvm_rb_gap(tmp)) {
572 printf("%d/%d gap %#lx != %#lx %s\n", 572 printf("%d/%d gap %#lx != %#lx %s\n",
573 n + 1, map->nentries, 573 n + 1, map->nentries,
574 (ulong)tmp->gap, (ulong)uvm_rb_gap(tmp), 574 (ulong)tmp->gap, (ulong)uvm_rb_gap(tmp),
575 tmp->next == &map->header ? "(last)" : ""); 575 tmp->next == &map->header ? "(last)" : "");
576 goto error; 576 goto error;
577 } 577 }
578 /* 578 /*
579 * If any entries are out of order, tmp->gap will be unsigned 579 * If any entries are out of order, tmp->gap will be unsigned
580 * and will likely exceed the size of the map. 580 * and will likely exceed the size of the map.
581 */ 581 */
582 if (tmp->gap >= vm_map_max(map) - vm_map_min(map)) { 582 if (tmp->gap >= vm_map_max(map) - vm_map_min(map)) {
583 printf("too large gap %zu\n", (size_t)tmp->gap); 583 printf("too large gap %zu\n", (size_t)tmp->gap);
584 goto error; 584 goto error;
585 } 585 }
586 n++; 586 n++;
587 } 587 }
588 588
589 if (n != map->nentries) { 589 if (n != map->nentries) {
590 printf("nentries: %d vs %d\n", n, map->nentries); 590 printf("nentries: %d vs %d\n", n, map->nentries);
591 goto error; 591 goto error;
592 } 592 }
593 593
594 trtmp = NULL; 594 trtmp = NULL;
595 for (tmp = map->header.next; tmp != &map->header; tmp = tmp->next) { 595 for (tmp = map->header.next; tmp != &map->header; tmp = tmp->next) {
596 if (tmp->maxgap != uvm_rb_maxgap(tmp)) { 596 if (tmp->maxgap != uvm_rb_maxgap(tmp)) {
597 printf("maxgap %#lx != %#lx\n", 597 printf("maxgap %#lx != %#lx\n",
598 (ulong)tmp->maxgap, 598 (ulong)tmp->maxgap,
599 (ulong)uvm_rb_maxgap(tmp)); 599 (ulong)uvm_rb_maxgap(tmp));
600 goto error; 600 goto error;
601 } 601 }
602 if (trtmp != NULL && trtmp->start >= tmp->start) { 602 if (trtmp != NULL && trtmp->start >= tmp->start) {
603 printf("corrupt: 0x%"PRIxVADDR"x >= 0x%"PRIxVADDR"x\n", 603 printf("corrupt: 0x%"PRIxVADDR"x >= 0x%"PRIxVADDR"x\n",
604 trtmp->start, tmp->start); 604 trtmp->start, tmp->start);
605 goto error; 605 goto error;
606 } 606 }
607 607
608 trtmp = tmp; 608 trtmp = tmp;
609 } 609 }
610 610
611 for (tmp = map->header.next; tmp != &map->header; 611 for (tmp = map->header.next; tmp != &map->header;
612 tmp = tmp->next, i++) { 612 tmp = tmp->next, i++) {
613 trtmp = rb_tree_iterate(&map->rb_tree, tmp, RB_DIR_LEFT); 613 trtmp = rb_tree_iterate(&map->rb_tree, tmp, RB_DIR_LEFT);
614 if (trtmp == NULL) 614 if (trtmp == NULL)
615 trtmp = &map->header; 615 trtmp = &map->header;
616 if (tmp->prev != trtmp) { 616 if (tmp->prev != trtmp) {
617 printf("lookup: %d: %p->prev=%p: %p\n", 617 printf("lookup: %d: %p->prev=%p: %p\n",
618 i, tmp, tmp->prev, trtmp); 618 i, tmp, tmp->prev, trtmp);
619 goto error; 619 goto error;
620 } 620 }
621 trtmp = rb_tree_iterate(&map->rb_tree, tmp, RB_DIR_RIGHT); 621 trtmp = rb_tree_iterate(&map->rb_tree, tmp, RB_DIR_RIGHT);
622 if (trtmp == NULL) 622 if (trtmp == NULL)
623 trtmp = &map->header; 623 trtmp = &map->header;
624 if (tmp->next != trtmp) { 624 if (tmp->next != trtmp) {
625 printf("lookup: %d: %p->next=%p: %p\n", 625 printf("lookup: %d: %p->next=%p: %p\n",
626 i, tmp, tmp->next, trtmp); 626 i, tmp, tmp->next, trtmp);
627 goto error; 627 goto error;
628 } 628 }
629 trtmp = rb_tree_find_node(&map->rb_tree, &tmp->start); 629 trtmp = rb_tree_find_node(&map->rb_tree, &tmp->start);
630 if (trtmp != tmp) { 630 if (trtmp != tmp) {
631 printf("lookup: %d: %p - %p: %p\n", i, tmp, trtmp, 631 printf("lookup: %d: %p - %p: %p\n", i, tmp, trtmp,
632 PARENT_ENTRY(map, tmp)); 632 PARENT_ENTRY(map, tmp));
633 goto error; 633 goto error;
634 } 634 }
635 } 635 }
636 636
637 return (0); 637 return (0);
638 error: 638 error:
639 return (-1); 639 return (-1);
640} 640}
641#endif /* defined(DEBUG) || defined(DDB) */ 641#endif /* defined(DEBUG) || defined(DDB) */
642 642
643/* 643/*
644 * vm_map_lock: acquire an exclusive (write) lock on a map. 644 * vm_map_lock: acquire an exclusive (write) lock on a map.
645 * 645 *
646 * => The locking protocol provides for guaranteed upgrade from shared -> 646 * => The locking protocol provides for guaranteed upgrade from shared ->
647 * exclusive by whichever thread currently has the map marked busy. 647 * exclusive by whichever thread currently has the map marked busy.
648 * See "LOCKING PROTOCOL NOTES" in uvm_map.h. This is horrible; among 648 * See "LOCKING PROTOCOL NOTES" in uvm_map.h. This is horrible; among
649 * other problems, it defeats any fairness guarantees provided by RW 649 * other problems, it defeats any fairness guarantees provided by RW
650 * locks. 650 * locks.
651 */ 651 */
652 652
653void 653void
654vm_map_lock(struct vm_map *map) 654vm_map_lock(struct vm_map *map)
655{ 655{
656 656
657 for (;;) { 657 for (;;) {
658 rw_enter(&map->lock, RW_WRITER); 658 rw_enter(&map->lock, RW_WRITER);
659 if (map->busy == NULL || map->busy == curlwp) { 659 if (map->busy == NULL || map->busy == curlwp) {
660 break; 660 break;
661 } 661 }
662 mutex_enter(&map->misc_lock); 662 mutex_enter(&map->misc_lock);
663 rw_exit(&map->lock); 663 rw_exit(&map->lock);
664 if (map->busy != NULL) { 664 if (map->busy != NULL) {
665 cv_wait(&map->cv, &map->misc_lock); 665 cv_wait(&map->cv, &map->misc_lock);
666 } 666 }
667 mutex_exit(&map->misc_lock); 667 mutex_exit(&map->misc_lock);
668 } 668 }
669 map->timestamp++; 669 map->timestamp++;
670} 670}
671 671
672/* 672/*
673 * vm_map_lock_try: try to lock a map, failing if it is already locked. 673 * vm_map_lock_try: try to lock a map, failing if it is already locked.
674 */ 674 */
675 675
676bool 676bool
677vm_map_lock_try(struct vm_map *map) 677vm_map_lock_try(struct vm_map *map)
678{ 678{
679 679
680 if (!rw_tryenter(&map->lock, RW_WRITER)) { 680 if (!rw_tryenter(&map->lock, RW_WRITER)) {
681 return false; 681 return false;
682 } 682 }
683 if (map->busy != NULL) { 683 if (map->busy != NULL) {
684 rw_exit(&map->lock); 684 rw_exit(&map->lock);
685 return false; 685 return false;
686 } 686 }
687 map->timestamp++; 687 map->timestamp++;
688 return true; 688 return true;
689} 689}
690 690
691/* 691/*
692 * vm_map_unlock: release an exclusive lock on a map. 692 * vm_map_unlock: release an exclusive lock on a map.
693 */ 693 */
694 694
695void 695void
696vm_map_unlock(struct vm_map *map) 696vm_map_unlock(struct vm_map *map)
697{ 697{
698 698
699 KASSERT(rw_write_held(&map->lock)); 699 KASSERT(rw_write_held(&map->lock));
700 KASSERT(map->busy == NULL || map->busy == curlwp); 700 KASSERT(map->busy == NULL || map->busy == curlwp);
701 rw_exit(&map->lock); 701 rw_exit(&map->lock);
702} 702}
703 703
704/* 704/*
705 * vm_map_unbusy: mark the map as unbusy, and wake any waiters that 705 * vm_map_unbusy: mark the map as unbusy, and wake any waiters that
706 * want an exclusive lock. 706 * want an exclusive lock.
707 */ 707 */
708 708
709void 709void
710vm_map_unbusy(struct vm_map *map) 710vm_map_unbusy(struct vm_map *map)
711{ 711{
712 712
713 KASSERT(map->busy == curlwp); 713 KASSERT(map->busy == curlwp);
714 714
715 /* 715 /*
716 * Safe to clear 'busy' and 'waiters' with only a read lock held: 716 * Safe to clear 'busy' and 'waiters' with only a read lock held:
717 * 717 *
718 * o they can only be set with a write lock held 718 * o they can only be set with a write lock held
719 * o writers are blocked out with a read or write hold 719 * o writers are blocked out with a read or write hold
720 * o at any time, only one thread owns the set of values 720 * o at any time, only one thread owns the set of values
721 */ 721 */
722 mutex_enter(&map->misc_lock); 722 mutex_enter(&map->misc_lock);
723 map->busy = NULL; 723 map->busy = NULL;
724 cv_broadcast(&map->cv); 724 cv_broadcast(&map->cv);
725 mutex_exit(&map->misc_lock); 725 mutex_exit(&map->misc_lock);
726} 726}
727 727
728/* 728/*
729 * vm_map_lock_read: acquire a shared (read) lock on a map. 729 * vm_map_lock_read: acquire a shared (read) lock on a map.
730 */ 730 */
731 731
732void 732void
733vm_map_lock_read(struct vm_map *map) 733vm_map_lock_read(struct vm_map *map)
734{ 734{
735 735
736 rw_enter(&map->lock, RW_READER); 736 rw_enter(&map->lock, RW_READER);
737} 737}
738 738
739/* 739/*
740 * vm_map_unlock_read: release a shared lock on a map. 740 * vm_map_unlock_read: release a shared lock on a map.
741 */ 741 */
742 742
743void 743void
744vm_map_unlock_read(struct vm_map *map) 744vm_map_unlock_read(struct vm_map *map)
745{ 745{
746 746
747 rw_exit(&map->lock); 747 rw_exit(&map->lock);
748} 748}
749 749
750/* 750/*
751 * vm_map_busy: mark a map as busy. 751 * vm_map_busy: mark a map as busy.
752 * 752 *
753 * => the caller must hold the map write locked 753 * => the caller must hold the map write locked
754 */ 754 */
755 755
756void 756void
757vm_map_busy(struct vm_map *map) 757vm_map_busy(struct vm_map *map)
758{ 758{
759 759
760 KASSERT(rw_write_held(&map->lock)); 760 KASSERT(rw_write_held(&map->lock));
761 KASSERT(map->busy == NULL); 761 KASSERT(map->busy == NULL);
762 762
763 map->busy = curlwp; 763 map->busy = curlwp;
764} 764}
765 765
766/* 766/*
767 * vm_map_locked_p: return true if the map is write locked. 767 * vm_map_locked_p: return true if the map is write locked.
768 * 768 *
769 * => only for debug purposes like KASSERTs. 769 * => only for debug purposes like KASSERTs.
770 * => should not be used to verify that a map is not locked. 770 * => should not be used to verify that a map is not locked.
771 */ 771 */
772 772
773bool 773bool
774vm_map_locked_p(struct vm_map *map) 774vm_map_locked_p(struct vm_map *map)
775{ 775{
776 776
777 return rw_write_held(&map->lock); 777 return rw_write_held(&map->lock);
778} 778}
779 779
780/* 780/*
781 * uvm_mapent_alloc: allocate a map entry 781 * uvm_mapent_alloc: allocate a map entry
782 */ 782 */
783 783
784static struct vm_map_entry * 784static struct vm_map_entry *
785uvm_mapent_alloc(struct vm_map *map, int flags) 785uvm_mapent_alloc(struct vm_map *map, int flags)
786{ 786{
787 struct vm_map_entry *me; 787 struct vm_map_entry *me;
788 int pflags = (flags & UVM_FLAG_NOWAIT) ? PR_NOWAIT : PR_WAITOK; 788 int pflags = (flags & UVM_FLAG_NOWAIT) ? PR_NOWAIT : PR_WAITOK;
789 UVMHIST_FUNC("uvm_mapent_alloc"); UVMHIST_CALLED(maphist); 789 UVMHIST_FUNC("uvm_mapent_alloc"); UVMHIST_CALLED(maphist);
790 790
791 me = pool_cache_get(&uvm_map_entry_cache, pflags); 791 me = pool_cache_get(&uvm_map_entry_cache, pflags);
792 if (__predict_false(me == NULL)) { 792 if (__predict_false(me == NULL)) {
793 return NULL; 793 return NULL;
794 } 794 }
795 me->flags = 0; 795 me->flags = 0;
796 796
797 UVMHIST_LOG(maphist, "<- new entry=%#jx [kentry=%jd]", (uintptr_t)me, 797 UVMHIST_LOG(maphist, "<- new entry=%#jx [kentry=%jd]", (uintptr_t)me,
798 (map == kernel_map), 0, 0); 798 (map == kernel_map), 0, 0);
799 return me; 799 return me;
800} 800}
801 801
802/* 802/*
803 * uvm_mapent_free: free map entry 803 * uvm_mapent_free: free map entry
804 */ 804 */
805 805
806static void 806static void
807uvm_mapent_free(struct vm_map_entry *me) 807uvm_mapent_free(struct vm_map_entry *me)
808{ 808{
809 UVMHIST_FUNC("uvm_mapent_free"); UVMHIST_CALLED(maphist); 809 UVMHIST_FUNC("uvm_mapent_free"); UVMHIST_CALLED(maphist);
810 810
811 UVMHIST_LOG(maphist,"<- freeing map entry=%#jx [flags=%jd]", 811 UVMHIST_LOG(maphist,"<- freeing map entry=%#jx [flags=%jd]",
812 (uintptr_t)me, me->flags, 0, 0); 812 (uintptr_t)me, me->flags, 0, 0);
813 pool_cache_put(&uvm_map_entry_cache, me); 813 pool_cache_put(&uvm_map_entry_cache, me);
814} 814}
815 815
816/* 816/*
817 * uvm_mapent_copy: copy a map entry, preserving flags 817 * uvm_mapent_copy: copy a map entry, preserving flags
818 */ 818 */
819 819
820static inline void 820static inline void
821uvm_mapent_copy(struct vm_map_entry *src, struct vm_map_entry *dst) 821uvm_mapent_copy(struct vm_map_entry *src, struct vm_map_entry *dst)
822{ 822{
823 823
824 memcpy(dst, src, sizeof(*dst)); 824 memcpy(dst, src, sizeof(*dst));
825 dst->flags = 0; 825 dst->flags = 0;
826} 826}
827 827
828#if defined(DEBUG) 828#if defined(DEBUG)
829static void 829static void
830_uvm_mapent_check(const struct vm_map_entry *entry, const char *file, int line) 830_uvm_mapent_check(const struct vm_map_entry *entry, const char *file, int line)
831{ 831{
832 832
833 if (entry->start >= entry->end) { 833 if (entry->start >= entry->end) {
834 goto bad; 834 goto bad;
835 } 835 }
836 if (UVM_ET_ISOBJ(entry)) { 836 if (UVM_ET_ISOBJ(entry)) {
837 if (entry->object.uvm_obj == NULL) { 837 if (entry->object.uvm_obj == NULL) {
838 goto bad; 838 goto bad;
839 } 839 }
840 } else if (UVM_ET_ISSUBMAP(entry)) { 840 } else if (UVM_ET_ISSUBMAP(entry)) {
841 if (entry->object.sub_map == NULL) { 841 if (entry->object.sub_map == NULL) {
842 goto bad; 842 goto bad;
843 } 843 }
844 } else { 844 } else {
845 if (entry->object.uvm_obj != NULL || 845 if (entry->object.uvm_obj != NULL ||
846 entry->object.sub_map != NULL) { 846 entry->object.sub_map != NULL) {
847 goto bad; 847 goto bad;
848 } 848 }
849 } 849 }
850 if (!UVM_ET_ISOBJ(entry)) { 850 if (!UVM_ET_ISOBJ(entry)) {
851 if (entry->offset != 0) { 851 if (entry->offset != 0) {
852 goto bad; 852 goto bad;
853 } 853 }
854 } 854 }
855 855
856 return; 856 return;
857 857
858bad: 858bad:
859 panic("%s: bad entry %p (%s:%d)", __func__, entry, file, line); 859 panic("%s: bad entry %p (%s:%d)", __func__, entry, file, line);
860} 860}
861#endif /* defined(DEBUG) */ 861#endif /* defined(DEBUG) */
862 862
863/* 863/*
864 * uvm_map_entry_unwire: unwire a map entry 864 * uvm_map_entry_unwire: unwire a map entry
865 * 865 *
866 * => map should be locked by caller 866 * => map should be locked by caller
867 */ 867 */
868 868
869static inline void 869static inline void
870uvm_map_entry_unwire(struct vm_map *map, struct vm_map_entry *entry) 870uvm_map_entry_unwire(struct vm_map *map, struct vm_map_entry *entry)
871{ 871{
872 872
873 entry->wired_count = 0; 873 entry->wired_count = 0;
874 uvm_fault_unwire_locked(map, entry->start, entry->end); 874 uvm_fault_unwire_locked(map, entry->start, entry->end);
875} 875}
876 876
877 877
878/* 878/*
879 * wrapper for calling amap_ref() 879 * wrapper for calling amap_ref()
880 */ 880 */
881static inline void 881static inline void
882uvm_map_reference_amap(struct vm_map_entry *entry, int flags) 882uvm_map_reference_amap(struct vm_map_entry *entry, int flags)
883{ 883{
884 884
885 amap_ref(entry->aref.ar_amap, entry->aref.ar_pageoff, 885 amap_ref(entry->aref.ar_amap, entry->aref.ar_pageoff,
886 (entry->end - entry->start) >> PAGE_SHIFT, flags); 886 (entry->end - entry->start) >> PAGE_SHIFT, flags);
887} 887}
888 888
889 889
890/* 890/*
891 * wrapper for calling amap_unref() 891 * wrapper for calling amap_unref()
892 */ 892 */
893static inline void 893static inline void
894uvm_map_unreference_amap(struct vm_map_entry *entry, int flags) 894uvm_map_unreference_amap(struct vm_map_entry *entry, int flags)
895{ 895{
896 896
897 amap_unref(entry->aref.ar_amap, entry->aref.ar_pageoff, 897 amap_unref(entry->aref.ar_amap, entry->aref.ar_pageoff,
898 (entry->end - entry->start) >> PAGE_SHIFT, flags); 898 (entry->end - entry->start) >> PAGE_SHIFT, flags);
899} 899}
900 900
901 901
902/* 902/*
903 * uvm_map_init: init mapping system at boot time. 903 * uvm_map_init: init mapping system at boot time.
904 */ 904 */
905 905
906void 906void
907uvm_map_init(void) 907uvm_map_init(void)
908{ 908{
909#if defined(UVMHIST) 909#if defined(UVMHIST)
910 static struct kern_history_ent pdhistbuf[UVMHIST_PDHIST_SIZE]; 910 static struct kern_history_ent pdhistbuf[UVMHIST_PDHIST_SIZE];
911#endif 911#endif
912 912
913 /* 913 /*
914 * first, init logging system. 914 * first, init logging system.
915 */ 915 */
916 916
917 UVMHIST_FUNC("uvm_map_init"); 917 UVMHIST_FUNC("uvm_map_init");
918 UVMHIST_LINK_STATIC(maphist); 918 UVMHIST_LINK_STATIC(maphist);
919 UVMHIST_INIT_STATIC(pdhist, pdhistbuf); 919 UVMHIST_INIT_STATIC(pdhist, pdhistbuf);
920 UVMHIST_CALLED(maphist); 920 UVMHIST_CALLED(maphist);
921 UVMHIST_LOG(maphist,"<starting uvm map system>", 0, 0, 0, 0); 921 UVMHIST_LOG(maphist,"<starting uvm map system>", 0, 0, 0, 0);
922 922
923 /* 923 /*
924 * initialize the global lock for kernel map entry. 924 * initialize the global lock for kernel map entry.
925 */ 925 */
926 926
927 mutex_init(&uvm_kentry_lock, MUTEX_DRIVER, IPL_VM); 927 mutex_init(&uvm_kentry_lock, MUTEX_DRIVER, IPL_VM);
928} 928}
929 929
930/* 930/*
931 * uvm_map_init_caches: init mapping system caches. 931 * uvm_map_init_caches: init mapping system caches.
932 */ 932 */
933void 933void
934uvm_map_init_caches(void) 934uvm_map_init_caches(void)
935{ 935{
936 /* 936 /*
937 * initialize caches. 937 * initialize caches.
938 */ 938 */
939 939
940 pool_cache_bootstrap(&uvm_map_entry_cache, sizeof(struct vm_map_entry), 940 pool_cache_bootstrap(&uvm_map_entry_cache, sizeof(struct vm_map_entry),
941 coherency_unit, 0, PR_LARGECACHE, "vmmpepl", NULL, IPL_NONE, NULL, 941 coherency_unit, 0, PR_LARGECACHE, "vmmpepl", NULL, IPL_NONE, NULL,
942 NULL, NULL); 942 NULL, NULL);
943 pool_cache_bootstrap(&uvm_vmspace_cache, sizeof(struct vmspace), 943 pool_cache_bootstrap(&uvm_vmspace_cache, sizeof(struct vmspace),
944 0, 0, 0, "vmsppl", NULL, IPL_NONE, NULL, NULL, NULL); 944 0, 0, 0, "vmsppl", NULL, IPL_NONE, NULL, NULL, NULL);
945} 945}
946 946
947/* 947/*
948 * clippers 948 * clippers
949 */ 949 */
950 950
951/* 951/*
952 * uvm_mapent_splitadj: adjust map entries for splitting, after uvm_mapent_copy. 952 * uvm_mapent_splitadj: adjust map entries for splitting, after uvm_mapent_copy.
953 */ 953 */
954 954
955static void 955static void
956uvm_mapent_splitadj(struct vm_map_entry *entry1, struct vm_map_entry *entry2, 956uvm_mapent_splitadj(struct vm_map_entry *entry1, struct vm_map_entry *entry2,
957 vaddr_t splitat) 957 vaddr_t splitat)
958{ 958{
959 vaddr_t adj; 959 vaddr_t adj;
960 960
961 KASSERT(entry1->start < splitat); 961 KASSERT(entry1->start < splitat);
962 KASSERT(splitat < entry1->end); 962 KASSERT(splitat < entry1->end);
963 963
964 adj = splitat - entry1->start; 964 adj = splitat - entry1->start;
965 entry1->end = entry2->start = splitat; 965 entry1->end = entry2->start = splitat;
966 966
967 if (entry1->aref.ar_amap) { 967 if (entry1->aref.ar_amap) {
968 amap_splitref(&entry1->aref, &entry2->aref, adj); 968 amap_splitref(&entry1->aref, &entry2->aref, adj);
969 } 969 }
970 if (UVM_ET_ISSUBMAP(entry1)) { 970 if (UVM_ET_ISSUBMAP(entry1)) {
971 /* ... unlikely to happen, but play it safe */ 971 /* ... unlikely to happen, but play it safe */
972 uvm_map_reference(entry1->object.sub_map); 972 uvm_map_reference(entry1->object.sub_map);
973 } else if (UVM_ET_ISOBJ(entry1)) { 973 } else if (UVM_ET_ISOBJ(entry1)) {
974 KASSERT(entry1->object.uvm_obj != NULL); /* suppress coverity */ 974 KASSERT(entry1->object.uvm_obj != NULL); /* suppress coverity */
975 entry2->offset += adj; 975 entry2->offset += adj;
976 if (entry1->object.uvm_obj->pgops && 976 if (entry1->object.uvm_obj->pgops &&
977 entry1->object.uvm_obj->pgops->pgo_reference) 977 entry1->object.uvm_obj->pgops->pgo_reference)
978 entry1->object.uvm_obj->pgops->pgo_reference( 978 entry1->object.uvm_obj->pgops->pgo_reference(
979 entry1->object.uvm_obj); 979 entry1->object.uvm_obj);
980 } 980 }
981} 981}
982 982
983/* 983/*
984 * uvm_map_clip_start: ensure that the entry begins at or after 984 * uvm_map_clip_start: ensure that the entry begins at or after
985 * the starting address, if it doesn't we split the entry. 985 * the starting address, if it doesn't we split the entry.
986 * 986 *
987 * => caller should use UVM_MAP_CLIP_START macro rather than calling 987 * => caller should use UVM_MAP_CLIP_START macro rather than calling
988 * this directly 988 * this directly
989 * => map must be locked by caller 989 * => map must be locked by caller
990 */ 990 */
991 991
992void 992void
993uvm_map_clip_start(struct vm_map *map, struct vm_map_entry *entry, 993uvm_map_clip_start(struct vm_map *map, struct vm_map_entry *entry,
994 vaddr_t start) 994 vaddr_t start)
995{ 995{
996 struct vm_map_entry *new_entry; 996 struct vm_map_entry *new_entry;
997 997
998 /* uvm_map_simplify_entry(map, entry); */ /* XXX */ 998 /* uvm_map_simplify_entry(map, entry); */ /* XXX */
999 999
1000 uvm_map_check(map, "clip_start entry"); 1000 uvm_map_check(map, "clip_start entry");
1001 uvm_mapent_check(entry); 1001 uvm_mapent_check(entry);
1002 1002
1003 /* 1003 /*
1004 * Split off the front portion. note that we must insert the new 1004 * Split off the front portion. note that we must insert the new
1005 * entry BEFORE this one, so that this entry has the specified 1005 * entry BEFORE this one, so that this entry has the specified
1006 * starting address. 1006 * starting address.
1007 */ 1007 */
1008 new_entry = uvm_mapent_alloc(map, 0); 1008 new_entry = uvm_mapent_alloc(map, 0);
1009 uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */ 1009 uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */
1010 uvm_mapent_splitadj(new_entry, entry, start); 1010 uvm_mapent_splitadj(new_entry, entry, start);
1011 uvm_map_entry_link(map, entry->prev, new_entry); 1011 uvm_map_entry_link(map, entry->prev, new_entry);
1012 1012
1013 uvm_map_check(map, "clip_start leave"); 1013 uvm_map_check(map, "clip_start leave");
1014} 1014}
1015 1015
1016/* 1016/*
1017 * uvm_map_clip_end: ensure that the entry ends at or before 1017 * uvm_map_clip_end: ensure that the entry ends at or before
1018 * the ending address, if it does't we split the reference 1018 * the ending address, if it does't we split the reference
1019 * 1019 *
1020 * => caller should use UVM_MAP_CLIP_END macro rather than calling 1020 * => caller should use UVM_MAP_CLIP_END macro rather than calling
1021 * this directly 1021 * this directly
1022 * => map must be locked by caller 1022 * => map must be locked by caller
1023 */ 1023 */
1024 1024
1025void 1025void
1026uvm_map_clip_end(struct vm_map *map, struct vm_map_entry *entry, vaddr_t end) 1026uvm_map_clip_end(struct vm_map *map, struct vm_map_entry *entry, vaddr_t end)
1027{ 1027{
1028 struct vm_map_entry *new_entry; 1028 struct vm_map_entry *new_entry;
1029 1029
1030 uvm_map_check(map, "clip_end entry"); 1030 uvm_map_check(map, "clip_end entry");
1031 uvm_mapent_check(entry); 1031 uvm_mapent_check(entry);
1032 1032
1033 /* 1033 /*
1034 * Create a new entry and insert it 1034 * Create a new entry and insert it
1035 * AFTER the specified entry 1035 * AFTER the specified entry
1036 */ 1036 */
1037 new_entry = uvm_mapent_alloc(map, 0); 1037 new_entry = uvm_mapent_alloc(map, 0);
1038 uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */ 1038 uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */
1039 uvm_mapent_splitadj(entry, new_entry, end); 1039 uvm_mapent_splitadj(entry, new_entry, end);
1040 uvm_map_entry_link(map, entry, new_entry); 1040 uvm_map_entry_link(map, entry, new_entry);
1041 1041
1042 uvm_map_check(map, "clip_end leave"); 1042 uvm_map_check(map, "clip_end leave");
1043} 1043}
1044 1044
1045/* 1045/*
1046 * M A P - m a i n e n t r y p o i n t 1046 * M A P - m a i n e n t r y p o i n t
1047 */ 1047 */
1048/* 1048/*
1049 * uvm_map: establish a valid mapping in a map 1049 * uvm_map: establish a valid mapping in a map
1050 * 1050 *
1051 * => assume startp is page aligned. 1051 * => assume startp is page aligned.
1052 * => assume size is a multiple of PAGE_SIZE. 1052 * => assume size is a multiple of PAGE_SIZE.
1053 * => assume sys_mmap provides enough of a "hint" to have us skip 1053 * => assume sys_mmap provides enough of a "hint" to have us skip
1054 * over text/data/bss area. 1054 * over text/data/bss area.
1055 * => map must be unlocked (we will lock it) 1055 * => map must be unlocked (we will lock it)
1056 * => <uobj,uoffset> value meanings (4 cases): 1056 * => <uobj,uoffset> value meanings (4 cases):
1057 * [1] <NULL,uoffset> == uoffset is a hint for PMAP_PREFER 1057 * [1] <NULL,uoffset> == uoffset is a hint for PMAP_PREFER
1058 * [2] <NULL,UVM_UNKNOWN_OFFSET> == don't PMAP_PREFER 1058 * [2] <NULL,UVM_UNKNOWN_OFFSET> == don't PMAP_PREFER
1059 * [3] <uobj,uoffset> == normal mapping 1059 * [3] <uobj,uoffset> == normal mapping
1060 * [4] <uobj,UVM_UNKNOWN_OFFSET> == uvm_map finds offset based on VA 1060 * [4] <uobj,UVM_UNKNOWN_OFFSET> == uvm_map finds offset based on VA
1061 * 1061 *
1062 * case [4] is for kernel mappings where we don't know the offset until 1062 * case [4] is for kernel mappings where we don't know the offset until
1063 * we've found a virtual address. note that kernel object offsets are 1063 * we've found a virtual address. note that kernel object offsets are
1064 * always relative to vm_map_min(kernel_map). 1064 * always relative to vm_map_min(kernel_map).
1065 * 1065 *
1066 * => if `align' is non-zero, we align the virtual address to the specified 1066 * => if `align' is non-zero, we align the virtual address to the specified
1067 * alignment. 1067 * alignment.
1068 * this is provided as a mechanism for large pages. 1068 * this is provided as a mechanism for large pages.
@@ -1617,3715 +1617,3715 @@ nomerge: @@ -1617,3715 +1617,3715 @@ nomerge:
1617 } 1617 }
1618 1618
1619 map->size += size; 1619 map->size += size;
1620 1620
1621 UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0); 1621 UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
1622 1622
1623 error = 0; 1623 error = 0;
1624 1624
1625done: 1625done:
1626 vm_map_unlock(map); 1626 vm_map_unlock(map);
1627 1627
1628 if (new_entry) { 1628 if (new_entry) {
1629 uvm_mapent_free(new_entry); 1629 uvm_mapent_free(new_entry);
1630 } 1630 }
1631 if (dead) { 1631 if (dead) {
1632 KDASSERT(merged); 1632 KDASSERT(merged);
1633 uvm_mapent_free(dead); 1633 uvm_mapent_free(dead);
1634 } 1634 }
1635 if (dead_entries) 1635 if (dead_entries)
1636 uvm_unmap_detach(dead_entries, 0); 1636 uvm_unmap_detach(dead_entries, 0);
1637 1637
1638 return error; 1638 return error;
1639} 1639}
1640 1640
1641/* 1641/*
1642 * uvm_map_lookup_entry_bytree: lookup an entry in tree 1642 * uvm_map_lookup_entry_bytree: lookup an entry in tree
1643 */ 1643 */
1644 1644
1645static inline bool 1645static inline bool
1646uvm_map_lookup_entry_bytree(struct vm_map *map, vaddr_t address, 1646uvm_map_lookup_entry_bytree(struct vm_map *map, vaddr_t address,
1647 struct vm_map_entry **entry /* OUT */) 1647 struct vm_map_entry **entry /* OUT */)
1648{ 1648{
1649 struct vm_map_entry *prev = &map->header; 1649 struct vm_map_entry *prev = &map->header;
1650 struct vm_map_entry *cur = ROOT_ENTRY(map); 1650 struct vm_map_entry *cur = ROOT_ENTRY(map);
1651 1651
1652 while (cur) { 1652 while (cur) {
1653 UVMMAP_EVCNT_INCR(mlk_treeloop); 1653 UVMMAP_EVCNT_INCR(mlk_treeloop);
1654 if (address >= cur->start) { 1654 if (address >= cur->start) {
1655 if (address < cur->end) { 1655 if (address < cur->end) {
1656 *entry = cur; 1656 *entry = cur;
1657 return true; 1657 return true;
1658 } 1658 }
1659 prev = cur; 1659 prev = cur;
1660 cur = RIGHT_ENTRY(cur); 1660 cur = RIGHT_ENTRY(cur);
1661 } else 1661 } else
1662 cur = LEFT_ENTRY(cur); 1662 cur = LEFT_ENTRY(cur);
1663 } 1663 }
1664 *entry = prev; 1664 *entry = prev;
1665 return false; 1665 return false;
1666} 1666}
1667 1667
1668/* 1668/*
1669 * uvm_map_lookup_entry: find map entry at or before an address 1669 * uvm_map_lookup_entry: find map entry at or before an address
1670 * 1670 *
1671 * => map must at least be read-locked by caller 1671 * => map must at least be read-locked by caller
1672 * => entry is returned in "entry" 1672 * => entry is returned in "entry"
1673 * => return value is true if address is in the returned entry 1673 * => return value is true if address is in the returned entry
1674 */ 1674 */
1675 1675
1676bool 1676bool
1677uvm_map_lookup_entry(struct vm_map *map, vaddr_t address, 1677uvm_map_lookup_entry(struct vm_map *map, vaddr_t address,
1678 struct vm_map_entry **entry /* OUT */) 1678 struct vm_map_entry **entry /* OUT */)
1679{ 1679{
1680 struct vm_map_entry *cur; 1680 struct vm_map_entry *cur;
1681 UVMHIST_FUNC("uvm_map_lookup_entry"); 1681 UVMHIST_FUNC("uvm_map_lookup_entry");
1682 UVMHIST_CALLED(maphist); 1682 UVMHIST_CALLED(maphist);
1683 1683
1684 UVMHIST_LOG(maphist,"(map=%#jx,addr=%#jx,ent=%#jx)", 1684 UVMHIST_LOG(maphist,"(map=%#jx,addr=%#jx,ent=%#jx)",
1685 (uintptr_t)map, address, (uintptr_t)entry, 0); 1685 (uintptr_t)map, address, (uintptr_t)entry, 0);
1686 1686
1687 /* 1687 /*
1688 * make a quick check to see if we are already looking at 1688 * make a quick check to see if we are already looking at
1689 * the entry we want (which is usually the case). note also 1689 * the entry we want (which is usually the case). note also
1690 * that we don't need to save the hint here... it is the 1690 * that we don't need to save the hint here... it is the
1691 * same hint (unless we are at the header, in which case the 1691 * same hint (unless we are at the header, in which case the
1692 * hint didn't buy us anything anyway). 1692 * hint didn't buy us anything anyway).
1693 */ 1693 */
1694 1694
1695 cur = map->hint; 1695 cur = map->hint;
1696 UVMMAP_EVCNT_INCR(mlk_call); 1696 UVMMAP_EVCNT_INCR(mlk_call);
1697 if (cur != &map->header && 1697 if (cur != &map->header &&
1698 address >= cur->start && cur->end > address) { 1698 address >= cur->start && cur->end > address) {
1699 UVMMAP_EVCNT_INCR(mlk_hint); 1699 UVMMAP_EVCNT_INCR(mlk_hint);
1700 *entry = cur; 1700 *entry = cur;
1701 UVMHIST_LOG(maphist,"<- got it via hint (%#jx)", 1701 UVMHIST_LOG(maphist,"<- got it via hint (%#jx)",
1702 (uintptr_t)cur, 0, 0, 0); 1702 (uintptr_t)cur, 0, 0, 0);
1703 uvm_mapent_check(*entry); 1703 uvm_mapent_check(*entry);
1704 return (true); 1704 return (true);
1705 } 1705 }
1706 uvm_map_check(map, __func__); 1706 uvm_map_check(map, __func__);
1707 1707
1708 /* 1708 /*
1709 * lookup in the tree. 1709 * lookup in the tree.
1710 */ 1710 */
1711 1711
1712 UVMMAP_EVCNT_INCR(mlk_tree); 1712 UVMMAP_EVCNT_INCR(mlk_tree);
1713 if (__predict_true(uvm_map_lookup_entry_bytree(map, address, entry))) { 1713 if (__predict_true(uvm_map_lookup_entry_bytree(map, address, entry))) {
1714 SAVE_HINT(map, map->hint, *entry); 1714 SAVE_HINT(map, map->hint, *entry);
1715 UVMHIST_LOG(maphist,"<- search got it (%#jx)", 1715 UVMHIST_LOG(maphist,"<- search got it (%#jx)",
1716 (uintptr_t)cur, 0, 0, 0); 1716 (uintptr_t)cur, 0, 0, 0);
1717 KDASSERT((*entry)->start <= address); 1717 KDASSERT((*entry)->start <= address);
1718 KDASSERT(address < (*entry)->end); 1718 KDASSERT(address < (*entry)->end);
1719 uvm_mapent_check(*entry); 1719 uvm_mapent_check(*entry);
1720 return (true); 1720 return (true);
1721 } 1721 }
1722 1722
1723 SAVE_HINT(map, map->hint, *entry); 1723 SAVE_HINT(map, map->hint, *entry);
1724 UVMHIST_LOG(maphist,"<- failed!",0,0,0,0); 1724 UVMHIST_LOG(maphist,"<- failed!",0,0,0,0);
1725 KDASSERT((*entry) == &map->header || (*entry)->end <= address); 1725 KDASSERT((*entry) == &map->header || (*entry)->end <= address);
1726 KDASSERT((*entry)->next == &map->header || 1726 KDASSERT((*entry)->next == &map->header ||
1727 address < (*entry)->next->start); 1727 address < (*entry)->next->start);
1728 return (false); 1728 return (false);
1729} 1729}
1730 1730
1731/* 1731/*
1732 * See if the range between start and start + length fits in the gap 1732 * See if the range between start and start + length fits in the gap
1733 * entry->next->start and entry->end. Returns 1 if fits, 0 if doesn't 1733 * entry->next->start and entry->end. Returns 1 if fits, 0 if doesn't
1734 * fit, and -1 address wraps around. 1734 * fit, and -1 address wraps around.
1735 */ 1735 */
1736static int 1736static int
1737uvm_map_space_avail(vaddr_t *start, vsize_t length, voff_t uoffset, 1737uvm_map_space_avail(vaddr_t *start, vsize_t length, voff_t uoffset,
1738 vsize_t align, int flags, int topdown, struct vm_map_entry *entry) 1738 vsize_t align, int flags, int topdown, struct vm_map_entry *entry)
1739{ 1739{
1740 vaddr_t end; 1740 vaddr_t end;
1741 1741
1742#ifdef PMAP_PREFER 1742#ifdef PMAP_PREFER
1743 /* 1743 /*
1744 * push start address forward as needed to avoid VAC alias problems. 1744 * push start address forward as needed to avoid VAC alias problems.
1745 * we only do this if a valid offset is specified. 1745 * we only do this if a valid offset is specified.
1746 */ 1746 */
1747 1747
1748 if (uoffset != UVM_UNKNOWN_OFFSET) 1748 if (uoffset != UVM_UNKNOWN_OFFSET)
1749 PMAP_PREFER(uoffset, start, length, topdown); 1749 PMAP_PREFER(uoffset, start, length, topdown);
1750#endif 1750#endif
1751 if ((flags & UVM_FLAG_COLORMATCH) != 0) { 1751 if ((flags & UVM_FLAG_COLORMATCH) != 0) {
1752 KASSERT(align < uvmexp.ncolors); 1752 KASSERT(align < uvmexp.ncolors);
1753 if (uvmexp.ncolors > 1) { 1753 if (uvmexp.ncolors > 1) {
1754 const u_int colormask = uvmexp.colormask; 1754 const u_int colormask = uvmexp.colormask;
1755 const u_int colorsize = colormask + 1; 1755 const u_int colorsize = colormask + 1;
1756 vaddr_t hint = atop(*start); 1756 vaddr_t hint = atop(*start);
1757 const u_int color = hint & colormask; 1757 const u_int color = hint & colormask;
1758 if (color != align) { 1758 if (color != align) {
1759 hint -= color; /* adjust to color boundary */ 1759 hint -= color; /* adjust to color boundary */
1760 KASSERT((hint & colormask) == 0); 1760 KASSERT((hint & colormask) == 0);
1761 if (topdown) { 1761 if (topdown) {
1762 if (align > color) 1762 if (align > color)
1763 hint -= colorsize; 1763 hint -= colorsize;
1764 } else { 1764 } else {
1765 if (align < color) 1765 if (align < color)
1766 hint += colorsize; 1766 hint += colorsize;
1767 } 1767 }
1768 *start = ptoa(hint + align); /* adjust to color */ 1768 *start = ptoa(hint + align); /* adjust to color */
1769 } 1769 }
1770 } 1770 }
1771 } else { 1771 } else {
1772 KASSERT(powerof2(align)); 1772 KASSERT(powerof2(align));
1773 uvm_map_align_va(start, align, topdown); 1773 uvm_map_align_va(start, align, topdown);
1774 /* 1774 /*
1775 * XXX Should we PMAP_PREFER() here again? 1775 * XXX Should we PMAP_PREFER() here again?
1776 * eh...i think we're okay 1776 * eh...i think we're okay
1777 */ 1777 */
1778 } 1778 }
1779 1779
1780 /* 1780 /*
1781 * Find the end of the proposed new region. Be sure we didn't 1781 * Find the end of the proposed new region. Be sure we didn't
1782 * wrap around the address; if so, we lose. Otherwise, if the 1782 * wrap around the address; if so, we lose. Otherwise, if the
1783 * proposed new region fits before the next entry, we win. 1783 * proposed new region fits before the next entry, we win.
1784 */ 1784 */
1785 1785
1786 end = *start + length; 1786 end = *start + length;
1787 if (end < *start) 1787 if (end < *start)
1788 return (-1); 1788 return (-1);
1789 1789
1790 if (entry->next->start >= end && *start >= entry->end) 1790 if (entry->next->start >= end && *start >= entry->end)
1791 return (1); 1791 return (1);
1792 1792
1793 return (0); 1793 return (0);
1794} 1794}
1795 1795
1796/* 1796/*
1797 * uvm_map_findspace: find "length" sized space in "map". 1797 * uvm_map_findspace: find "length" sized space in "map".
1798 * 1798 *
1799 * => "hint" is a hint about where we want it, unless UVM_FLAG_FIXED is 1799 * => "hint" is a hint about where we want it, unless UVM_FLAG_FIXED is
1800 * set in "flags" (in which case we insist on using "hint"). 1800 * set in "flags" (in which case we insist on using "hint").
1801 * => "result" is VA returned 1801 * => "result" is VA returned
1802 * => uobj/uoffset are to be used to handle VAC alignment, if required 1802 * => uobj/uoffset are to be used to handle VAC alignment, if required
1803 * => if "align" is non-zero, we attempt to align to that value. 1803 * => if "align" is non-zero, we attempt to align to that value.
1804 * => caller must at least have read-locked map 1804 * => caller must at least have read-locked map
1805 * => returns NULL on failure, or pointer to prev. map entry if success 1805 * => returns NULL on failure, or pointer to prev. map entry if success
1806 * => note this is a cross between the old vm_map_findspace and vm_map_find 1806 * => note this is a cross between the old vm_map_findspace and vm_map_find
1807 */ 1807 */
1808 1808
1809struct vm_map_entry * 1809struct vm_map_entry *
1810uvm_map_findspace(struct vm_map *map, vaddr_t hint, vsize_t length, 1810uvm_map_findspace(struct vm_map *map, vaddr_t hint, vsize_t length,
1811 vaddr_t *result /* OUT */, struct uvm_object *uobj, voff_t uoffset, 1811 vaddr_t *result /* OUT */, struct uvm_object *uobj, voff_t uoffset,
1812 vsize_t align, int flags) 1812 vsize_t align, int flags)
1813{ 1813{
1814 struct vm_map_entry *entry; 1814 struct vm_map_entry *entry;
1815 struct vm_map_entry *child, *prev, *tmp; 1815 struct vm_map_entry *child, *prev, *tmp;
1816 vaddr_t orig_hint __diagused; 1816 vaddr_t orig_hint __diagused;
1817 const int topdown = map->flags & VM_MAP_TOPDOWN; 1817 const int topdown = map->flags & VM_MAP_TOPDOWN;
1818 UVMHIST_FUNC("uvm_map_findspace"); 1818 UVMHIST_FUNC("uvm_map_findspace");
1819 UVMHIST_CALLED(maphist); 1819 UVMHIST_CALLED(maphist);
1820 1820
1821 UVMHIST_LOG(maphist, "(map=%#jx, hint=%#jx, len=%ju, flags=%#jx)", 1821 UVMHIST_LOG(maphist, "(map=%#jx, hint=%#jx, len=%ju, flags=%#jx)",
1822 (uintptr_t)map, hint, length, flags); 1822 (uintptr_t)map, hint, length, flags);
1823 KASSERT((flags & UVM_FLAG_COLORMATCH) != 0 || powerof2(align)); 1823 KASSERT((flags & UVM_FLAG_COLORMATCH) != 0 || powerof2(align));
1824 KASSERT((flags & UVM_FLAG_COLORMATCH) == 0 || align < uvmexp.ncolors); 1824 KASSERT((flags & UVM_FLAG_COLORMATCH) == 0 || align < uvmexp.ncolors);
1825 KASSERT((flags & UVM_FLAG_FIXED) == 0 || align == 0); 1825 KASSERT((flags & UVM_FLAG_FIXED) == 0 || align == 0);
1826 1826
1827 uvm_map_check(map, "map_findspace entry"); 1827 uvm_map_check(map, "map_findspace entry");
1828 1828
1829 /* 1829 /*
1830 * remember the original hint. if we are aligning, then we 1830 * remember the original hint. if we are aligning, then we
1831 * may have to try again with no alignment constraint if 1831 * may have to try again with no alignment constraint if
1832 * we fail the first time. 1832 * we fail the first time.
1833 */ 1833 */
1834 1834
1835 orig_hint = hint; 1835 orig_hint = hint;
1836 if (hint < vm_map_min(map)) { /* check ranges ... */ 1836 if (hint < vm_map_min(map)) { /* check ranges ... */
1837 if (flags & UVM_FLAG_FIXED) { 1837 if (flags & UVM_FLAG_FIXED) {
1838 UVMHIST_LOG(maphist,"<- VA below map range",0,0,0,0); 1838 UVMHIST_LOG(maphist,"<- VA below map range",0,0,0,0);
1839 return (NULL); 1839 return (NULL);
1840 } 1840 }
1841 hint = vm_map_min(map); 1841 hint = vm_map_min(map);
1842 } 1842 }
1843 if (hint > vm_map_max(map)) { 1843 if (hint > vm_map_max(map)) {
1844 UVMHIST_LOG(maphist,"<- VA %#jx > range [%#jx->%#jx]", 1844 UVMHIST_LOG(maphist,"<- VA %#jx > range [%#jx->%#jx]",
1845 hint, vm_map_min(map), vm_map_max(map), 0); 1845 hint, vm_map_min(map), vm_map_max(map), 0);
1846 return (NULL); 1846 return (NULL);
1847 } 1847 }
1848 1848
1849 /* 1849 /*
1850 * hint may not be aligned properly; we need round up or down it 1850 * hint may not be aligned properly; we need round up or down it
1851 * before proceeding further. 1851 * before proceeding further.
1852 */ 1852 */
1853 if ((flags & UVM_FLAG_COLORMATCH) == 0) 1853 if ((flags & UVM_FLAG_COLORMATCH) == 0)
1854 uvm_map_align_va(&hint, align, topdown); 1854 uvm_map_align_va(&hint, align, topdown);
1855 1855
1856 /* 1856 /*
1857 * Look for the first possible address; if there's already 1857 * Look for the first possible address; if there's already
1858 * something at this address, we have to start after it. 1858 * something at this address, we have to start after it.
1859 */ 1859 */
1860 1860
1861 /* 1861 /*
1862 * @@@: there are four, no, eight cases to consider. 1862 * @@@: there are four, no, eight cases to consider.
1863 * 1863 *
1864 * 0: found, fixed, bottom up -> fail 1864 * 0: found, fixed, bottom up -> fail
1865 * 1: found, fixed, top down -> fail 1865 * 1: found, fixed, top down -> fail
1866 * 2: found, not fixed, bottom up -> start after entry->end, 1866 * 2: found, not fixed, bottom up -> start after entry->end,
1867 * loop up 1867 * loop up
1868 * 3: found, not fixed, top down -> start before entry->start, 1868 * 3: found, not fixed, top down -> start before entry->start,
1869 * loop down 1869 * loop down
1870 * 4: not found, fixed, bottom up -> check entry->next->start, fail 1870 * 4: not found, fixed, bottom up -> check entry->next->start, fail
1871 * 5: not found, fixed, top down -> check entry->next->start, fail 1871 * 5: not found, fixed, top down -> check entry->next->start, fail
1872 * 6: not found, not fixed, bottom up -> check entry->next->start, 1872 * 6: not found, not fixed, bottom up -> check entry->next->start,
1873 * loop up 1873 * loop up
1874 * 7: not found, not fixed, top down -> check entry->next->start, 1874 * 7: not found, not fixed, top down -> check entry->next->start,
1875 * loop down 1875 * loop down
1876 * 1876 *
1877 * as you can see, it reduces to roughly five cases, and that 1877 * as you can see, it reduces to roughly five cases, and that
1878 * adding top down mapping only adds one unique case (without 1878 * adding top down mapping only adds one unique case (without
1879 * it, there would be four cases). 1879 * it, there would be four cases).
1880 */ 1880 */
1881 1881
1882 if ((flags & UVM_FLAG_FIXED) == 0 && hint == vm_map_min(map)) { 1882 if ((flags & UVM_FLAG_FIXED) == 0 && hint == vm_map_min(map)) {
1883 entry = map->first_free; 1883 entry = map->first_free;
1884 } else { 1884 } else {
1885 if (uvm_map_lookup_entry(map, hint, &entry)) { 1885 if (uvm_map_lookup_entry(map, hint, &entry)) {
1886 /* "hint" address already in use ... */ 1886 /* "hint" address already in use ... */
1887 if (flags & UVM_FLAG_FIXED) { 1887 if (flags & UVM_FLAG_FIXED) {
1888 UVMHIST_LOG(maphist, "<- fixed & VA in use", 1888 UVMHIST_LOG(maphist, "<- fixed & VA in use",
1889 0, 0, 0, 0); 1889 0, 0, 0, 0);
1890 return (NULL); 1890 return (NULL);
1891 } 1891 }
1892 if (topdown) 1892 if (topdown)
1893 /* Start from lower gap. */ 1893 /* Start from lower gap. */
1894 entry = entry->prev; 1894 entry = entry->prev;
1895 } else if (flags & UVM_FLAG_FIXED) { 1895 } else if (flags & UVM_FLAG_FIXED) {
1896 if (entry->next->start >= hint + length && 1896 if (entry->next->start >= hint + length &&
1897 hint + length > hint) 1897 hint + length > hint)
1898 goto found; 1898 goto found;
1899 1899
1900 /* "hint" address is gap but too small */ 1900 /* "hint" address is gap but too small */
1901 UVMHIST_LOG(maphist, "<- fixed mapping failed", 1901 UVMHIST_LOG(maphist, "<- fixed mapping failed",
1902 0, 0, 0, 0); 1902 0, 0, 0, 0);
1903 return (NULL); /* only one shot at it ... */ 1903 return (NULL); /* only one shot at it ... */
1904 } else { 1904 } else {
1905 /* 1905 /*
1906 * See if given hint fits in this gap. 1906 * See if given hint fits in this gap.
1907 */ 1907 */
1908 switch (uvm_map_space_avail(&hint, length, 1908 switch (uvm_map_space_avail(&hint, length,
1909 uoffset, align, flags, topdown, entry)) { 1909 uoffset, align, flags, topdown, entry)) {
1910 case 1: 1910 case 1:
1911 goto found; 1911 goto found;
1912 case -1: 1912 case -1:
1913 goto wraparound; 1913 goto wraparound;
1914 } 1914 }
1915 1915
1916 if (topdown) { 1916 if (topdown) {
1917 /* 1917 /*
1918 * Still there is a chance to fit 1918 * Still there is a chance to fit
1919 * if hint > entry->end. 1919 * if hint > entry->end.
1920 */ 1920 */
1921 } else { 1921 } else {
1922 /* Start from higher gap. */ 1922 /* Start from higher gap. */
1923 entry = entry->next; 1923 entry = entry->next;
1924 if (entry == &map->header) 1924 if (entry == &map->header)
1925 goto notfound; 1925 goto notfound;
1926 goto nextgap; 1926 goto nextgap;
1927 } 1927 }
1928 } 1928 }
1929 } 1929 }
1930 1930
1931 /* 1931 /*
1932 * Note that all UVM_FLAGS_FIXED case is already handled. 1932 * Note that all UVM_FLAGS_FIXED case is already handled.
1933 */ 1933 */
1934 KDASSERT((flags & UVM_FLAG_FIXED) == 0); 1934 KDASSERT((flags & UVM_FLAG_FIXED) == 0);
1935 1935
1936 /* Try to find the space in the red-black tree */ 1936 /* Try to find the space in the red-black tree */
1937 1937
1938 /* Check slot before any entry */ 1938 /* Check slot before any entry */
1939 hint = topdown ? entry->next->start - length : entry->end; 1939 hint = topdown ? entry->next->start - length : entry->end;
1940 switch (uvm_map_space_avail(&hint, length, uoffset, align, flags, 1940 switch (uvm_map_space_avail(&hint, length, uoffset, align, flags,
1941 topdown, entry)) { 1941 topdown, entry)) {
1942 case 1: 1942 case 1:
1943 goto found; 1943 goto found;
1944 case -1: 1944 case -1:
1945 goto wraparound; 1945 goto wraparound;
1946 } 1946 }
1947 1947
1948nextgap: 1948nextgap:
1949 KDASSERT((flags & UVM_FLAG_FIXED) == 0); 1949 KDASSERT((flags & UVM_FLAG_FIXED) == 0);
1950 /* If there is not enough space in the whole tree, we fail */ 1950 /* If there is not enough space in the whole tree, we fail */
1951 tmp = ROOT_ENTRY(map); 1951 tmp = ROOT_ENTRY(map);
1952 if (tmp == NULL || tmp->maxgap < length) 1952 if (tmp == NULL || tmp->maxgap < length)
1953 goto notfound; 1953 goto notfound;
1954 1954
1955 prev = NULL; /* previous candidate */ 1955 prev = NULL; /* previous candidate */
1956 1956
1957 /* Find an entry close to hint that has enough space */ 1957 /* Find an entry close to hint that has enough space */
1958 for (; tmp;) { 1958 for (; tmp;) {
1959 KASSERT(tmp->next->start == tmp->end + tmp->gap); 1959 KASSERT(tmp->next->start == tmp->end + tmp->gap);
1960 if (topdown) { 1960 if (topdown) {
1961 if (tmp->next->start < hint + length && 1961 if (tmp->next->start < hint + length &&
1962 (prev == NULL || tmp->end > prev->end)) { 1962 (prev == NULL || tmp->end > prev->end)) {
1963 if (tmp->gap >= length) 1963 if (tmp->gap >= length)
1964 prev = tmp; 1964 prev = tmp;
1965 else if ((child = LEFT_ENTRY(tmp)) != NULL 1965 else if ((child = LEFT_ENTRY(tmp)) != NULL
1966 && child->maxgap >= length) 1966 && child->maxgap >= length)
1967 prev = tmp; 1967 prev = tmp;
1968 } 1968 }
1969 } else { 1969 } else {
1970 if (tmp->end >= hint && 1970 if (tmp->end >= hint &&
1971 (prev == NULL || tmp->end < prev->end)) { 1971 (prev == NULL || tmp->end < prev->end)) {
1972 if (tmp->gap >= length) 1972 if (tmp->gap >= length)
1973 prev = tmp; 1973 prev = tmp;
1974 else if ((child = RIGHT_ENTRY(tmp)) != NULL 1974 else if ((child = RIGHT_ENTRY(tmp)) != NULL
1975 && child->maxgap >= length) 1975 && child->maxgap >= length)
1976 prev = tmp; 1976 prev = tmp;
1977 } 1977 }
1978 } 1978 }
1979 if (tmp->next->start < hint + length) 1979 if (tmp->next->start < hint + length)
1980 child = RIGHT_ENTRY(tmp); 1980 child = RIGHT_ENTRY(tmp);
1981 else if (tmp->end > hint) 1981 else if (tmp->end > hint)
1982 child = LEFT_ENTRY(tmp); 1982 child = LEFT_ENTRY(tmp);
1983 else { 1983 else {
1984 if (tmp->gap >= length) 1984 if (tmp->gap >= length)
1985 break; 1985 break;
1986 if (topdown) 1986 if (topdown)
1987 child = LEFT_ENTRY(tmp); 1987 child = LEFT_ENTRY(tmp);
1988 else 1988 else
1989 child = RIGHT_ENTRY(tmp); 1989 child = RIGHT_ENTRY(tmp);
1990 } 1990 }
1991 if (child == NULL || child->maxgap < length) 1991 if (child == NULL || child->maxgap < length)
1992 break; 1992 break;
1993 tmp = child; 1993 tmp = child;
1994 } 1994 }
1995 1995
1996 if (tmp != NULL && tmp->start < hint && hint < tmp->next->start) { 1996 if (tmp != NULL && tmp->start < hint && hint < tmp->next->start) {
1997 /* 1997 /*
1998 * Check if the entry that we found satifies the 1998 * Check if the entry that we found satifies the
1999 * space requirement 1999 * space requirement
2000 */ 2000 */
2001 if (topdown) { 2001 if (topdown) {
2002 if (hint > tmp->next->start - length) 2002 if (hint > tmp->next->start - length)
2003 hint = tmp->next->start - length; 2003 hint = tmp->next->start - length;
2004 } else { 2004 } else {
2005 if (hint < tmp->end) 2005 if (hint < tmp->end)
2006 hint = tmp->end; 2006 hint = tmp->end;
2007 } 2007 }
2008 switch (uvm_map_space_avail(&hint, length, uoffset, align, 2008 switch (uvm_map_space_avail(&hint, length, uoffset, align,
2009 flags, topdown, tmp)) { 2009 flags, topdown, tmp)) {
2010 case 1: 2010 case 1:
2011 entry = tmp; 2011 entry = tmp;
2012 goto found; 2012 goto found;
2013 case -1: 2013 case -1:
2014 goto wraparound; 2014 goto wraparound;
2015 } 2015 }
2016 if (tmp->gap >= length) 2016 if (tmp->gap >= length)
2017 goto listsearch; 2017 goto listsearch;
2018 } 2018 }
2019 if (prev == NULL) 2019 if (prev == NULL)
2020 goto notfound; 2020 goto notfound;
2021 2021
2022 if (topdown) { 2022 if (topdown) {
2023 KASSERT(orig_hint >= prev->next->start - length || 2023 KASSERT(orig_hint >= prev->next->start - length ||
2024 prev->next->start - length > prev->next->start); 2024 prev->next->start - length > prev->next->start);
2025 hint = prev->next->start - length; 2025 hint = prev->next->start - length;
2026 } else { 2026 } else {
2027 KASSERT(orig_hint <= prev->end); 2027 KASSERT(orig_hint <= prev->end);
2028 hint = prev->end; 2028 hint = prev->end;
2029 } 2029 }
2030 switch (uvm_map_space_avail(&hint, length, uoffset, align, 2030 switch (uvm_map_space_avail(&hint, length, uoffset, align,
2031 flags, topdown, prev)) { 2031 flags, topdown, prev)) {
2032 case 1: 2032 case 1:
2033 entry = prev; 2033 entry = prev;
2034 goto found; 2034 goto found;
2035 case -1: 2035 case -1:
2036 goto wraparound; 2036 goto wraparound;
2037 } 2037 }
2038 if (prev->gap >= length) 2038 if (prev->gap >= length)
2039 goto listsearch; 2039 goto listsearch;
2040 2040
2041 if (topdown) 2041 if (topdown)
2042 tmp = LEFT_ENTRY(prev); 2042 tmp = LEFT_ENTRY(prev);
2043 else 2043 else
2044 tmp = RIGHT_ENTRY(prev); 2044 tmp = RIGHT_ENTRY(prev);
2045 for (;;) { 2045 for (;;) {
2046 KASSERT(tmp && tmp->maxgap >= length); 2046 KASSERT(tmp && tmp->maxgap >= length);
2047 if (topdown) 2047 if (topdown)
2048 child = RIGHT_ENTRY(tmp); 2048 child = RIGHT_ENTRY(tmp);
2049 else 2049 else
2050 child = LEFT_ENTRY(tmp); 2050 child = LEFT_ENTRY(tmp);
2051 if (child && child->maxgap >= length) { 2051 if (child && child->maxgap >= length) {
2052 tmp = child; 2052 tmp = child;
2053 continue; 2053 continue;
2054 } 2054 }
2055 if (tmp->gap >= length) 2055 if (tmp->gap >= length)
2056 break; 2056 break;
2057 if (topdown) 2057 if (topdown)
2058 tmp = LEFT_ENTRY(tmp); 2058 tmp = LEFT_ENTRY(tmp);
2059 else 2059 else
2060 tmp = RIGHT_ENTRY(tmp); 2060 tmp = RIGHT_ENTRY(tmp);
2061 } 2061 }
2062 2062
2063 if (topdown) { 2063 if (topdown) {
2064 KASSERT(orig_hint >= tmp->next->start - length || 2064 KASSERT(orig_hint >= tmp->next->start - length ||
2065 tmp->next->start - length > tmp->next->start); 2065 tmp->next->start - length > tmp->next->start);
2066 hint = tmp->next->start - length; 2066 hint = tmp->next->start - length;
2067 } else { 2067 } else {
2068 KASSERT(orig_hint <= tmp->end); 2068 KASSERT(orig_hint <= tmp->end);
2069 hint = tmp->end; 2069 hint = tmp->end;
2070 } 2070 }
2071 switch (uvm_map_space_avail(&hint, length, uoffset, align, 2071 switch (uvm_map_space_avail(&hint, length, uoffset, align,
2072 flags, topdown, tmp)) { 2072 flags, topdown, tmp)) {
2073 case 1: 2073 case 1:
2074 entry = tmp; 2074 entry = tmp;
2075 goto found; 2075 goto found;
2076 case -1: 2076 case -1:
2077 goto wraparound; 2077 goto wraparound;
2078 } 2078 }
2079 2079
2080 /* 2080 /*
2081 * The tree fails to find an entry because of offset or alignment 2081 * The tree fails to find an entry because of offset or alignment
2082 * restrictions. Search the list instead. 2082 * restrictions. Search the list instead.
2083 */ 2083 */
2084 listsearch: 2084 listsearch:
2085 /* 2085 /*
2086 * Look through the rest of the map, trying to fit a new region in 2086 * Look through the rest of the map, trying to fit a new region in
2087 * the gap between existing regions, or after the very last region. 2087 * the gap between existing regions, or after the very last region.
2088 * note: entry->end = base VA of current gap, 2088 * note: entry->end = base VA of current gap,
2089 * entry->next->start = VA of end of current gap 2089 * entry->next->start = VA of end of current gap
2090 */ 2090 */
2091 2091
2092 for (;;) { 2092 for (;;) {
2093 /* Update hint for current gap. */ 2093 /* Update hint for current gap. */
2094 hint = topdown ? entry->next->start - length : entry->end; 2094 hint = topdown ? entry->next->start - length : entry->end;
2095 2095
2096 /* See if it fits. */ 2096 /* See if it fits. */
2097 switch (uvm_map_space_avail(&hint, length, uoffset, align, 2097 switch (uvm_map_space_avail(&hint, length, uoffset, align,
2098 flags, topdown, entry)) { 2098 flags, topdown, entry)) {
2099 case 1: 2099 case 1:
2100 goto found; 2100 goto found;
2101 case -1: 2101 case -1:
2102 goto wraparound; 2102 goto wraparound;
2103 } 2103 }
2104 2104
2105 /* Advance to next/previous gap */ 2105 /* Advance to next/previous gap */
2106 if (topdown) { 2106 if (topdown) {
2107 if (entry == &map->header) { 2107 if (entry == &map->header) {
2108 UVMHIST_LOG(maphist, "<- failed (off start)", 2108 UVMHIST_LOG(maphist, "<- failed (off start)",
2109 0,0,0,0); 2109 0,0,0,0);
2110 goto notfound; 2110 goto notfound;
2111 } 2111 }
2112 entry = entry->prev; 2112 entry = entry->prev;
2113 } else { 2113 } else {
2114 entry = entry->next; 2114 entry = entry->next;
2115 if (entry == &map->header) { 2115 if (entry == &map->header) {
2116 UVMHIST_LOG(maphist, "<- failed (off end)", 2116 UVMHIST_LOG(maphist, "<- failed (off end)",
2117 0,0,0,0); 2117 0,0,0,0);
2118 goto notfound; 2118 goto notfound;
2119 } 2119 }
2120 } 2120 }
2121 } 2121 }
2122 2122
2123 found: 2123 found:
2124 SAVE_HINT(map, map->hint, entry); 2124 SAVE_HINT(map, map->hint, entry);
2125 *result = hint; 2125 *result = hint;
2126 UVMHIST_LOG(maphist,"<- got it! (result=%#jx)", hint, 0,0,0); 2126 UVMHIST_LOG(maphist,"<- got it! (result=%#jx)", hint, 0,0,0);
2127 KASSERTMSG( topdown || hint >= orig_hint, "hint: %jx, orig_hint: %jx", 2127 KASSERTMSG( topdown || hint >= orig_hint, "hint: %jx, orig_hint: %jx",
2128 (uintmax_t)hint, (uintmax_t)orig_hint); 2128 (uintmax_t)hint, (uintmax_t)orig_hint);
2129 KASSERTMSG(!topdown || hint <= orig_hint, "hint: %jx, orig_hint: %jx", 2129 KASSERTMSG(!topdown || hint <= orig_hint, "hint: %jx, orig_hint: %jx",
2130 (uintmax_t)hint, (uintmax_t)orig_hint); 2130 (uintmax_t)hint, (uintmax_t)orig_hint);
2131 KASSERT(entry->end <= hint); 2131 KASSERT(entry->end <= hint);
2132 KASSERT(hint + length <= entry->next->start); 2132 KASSERT(hint + length <= entry->next->start);
2133 return (entry); 2133 return (entry);
2134 2134
2135 wraparound: 2135 wraparound:
2136 UVMHIST_LOG(maphist, "<- failed (wrap around)", 0,0,0,0); 2136 UVMHIST_LOG(maphist, "<- failed (wrap around)", 0,0,0,0);
2137 2137
2138 return (NULL); 2138 return (NULL);
2139 2139
2140 notfound: 2140 notfound:
2141 UVMHIST_LOG(maphist, "<- failed (notfound)", 0,0,0,0); 2141 UVMHIST_LOG(maphist, "<- failed (notfound)", 0,0,0,0);
2142 2142
2143 return (NULL); 2143 return (NULL);
2144} 2144}
2145 2145
2146/* 2146/*
2147 * U N M A P - m a i n h e l p e r f u n c t i o n s 2147 * U N M A P - m a i n h e l p e r f u n c t i o n s
2148 */ 2148 */
2149 2149
2150/* 2150/*
2151 * uvm_unmap_remove: remove mappings from a vm_map (from "start" up to "stop") 2151 * uvm_unmap_remove: remove mappings from a vm_map (from "start" up to "stop")
2152 * 2152 *
2153 * => caller must check alignment and size 2153 * => caller must check alignment and size
2154 * => map must be locked by caller 2154 * => map must be locked by caller
2155 * => we return a list of map entries that we've remove from the map 2155 * => we return a list of map entries that we've remove from the map
2156 * in "entry_list" 2156 * in "entry_list"
2157 */ 2157 */
2158 2158
2159void 2159void
2160uvm_unmap_remove(struct vm_map *map, vaddr_t start, vaddr_t end, 2160uvm_unmap_remove(struct vm_map *map, vaddr_t start, vaddr_t end,
2161 struct vm_map_entry **entry_list /* OUT */, int flags) 2161 struct vm_map_entry **entry_list /* OUT */, int flags)
2162{ 2162{
2163 struct vm_map_entry *entry, *first_entry, *next; 2163 struct vm_map_entry *entry, *first_entry, *next;
2164 vaddr_t len; 2164 vaddr_t len;
2165 UVMHIST_FUNC("uvm_unmap_remove"); UVMHIST_CALLED(maphist); 2165 UVMHIST_FUNC("uvm_unmap_remove"); UVMHIST_CALLED(maphist);
2166 2166
2167 UVMHIST_LOG(maphist,"(map=%#jx, start=%#jx, end=%#jx)", 2167 UVMHIST_LOG(maphist,"(map=%#jx, start=%#jx, end=%#jx)",
2168 (uintptr_t)map, start, end, 0); 2168 (uintptr_t)map, start, end, 0);
2169 VM_MAP_RANGE_CHECK(map, start, end); 2169 VM_MAP_RANGE_CHECK(map, start, end);
2170 2170
2171 uvm_map_check(map, "unmap_remove entry"); 2171 uvm_map_check(map, "unmap_remove entry");
2172 2172
2173 /* 2173 /*
2174 * find first entry 2174 * find first entry
2175 */ 2175 */
2176 2176
2177 if (uvm_map_lookup_entry(map, start, &first_entry) == true) { 2177 if (uvm_map_lookup_entry(map, start, &first_entry) == true) {
2178 /* clip and go... */ 2178 /* clip and go... */
2179 entry = first_entry; 2179 entry = first_entry;
2180 UVM_MAP_CLIP_START(map, entry, start); 2180 UVM_MAP_CLIP_START(map, entry, start);
2181 /* critical! prevents stale hint */ 2181 /* critical! prevents stale hint */
2182 SAVE_HINT(map, entry, entry->prev); 2182 SAVE_HINT(map, entry, entry->prev);
2183 } else { 2183 } else {
2184 entry = first_entry->next; 2184 entry = first_entry->next;
2185 } 2185 }
2186 2186
2187 /* 2187 /*
2188 * save the free space hint 2188 * save the free space hint
2189 */ 2189 */
2190 2190
2191 if (map->first_free != &map->header && map->first_free->start >= start) 2191 if (map->first_free != &map->header && map->first_free->start >= start)
2192 map->first_free = entry->prev; 2192 map->first_free = entry->prev;
2193 2193
2194 /* 2194 /*
2195 * note: we now re-use first_entry for a different task. we remove 2195 * note: we now re-use first_entry for a different task. we remove
2196 * a number of map entries from the map and save them in a linked 2196 * a number of map entries from the map and save them in a linked
2197 * list headed by "first_entry". once we remove them from the map 2197 * list headed by "first_entry". once we remove them from the map
2198 * the caller should unlock the map and drop the references to the 2198 * the caller should unlock the map and drop the references to the
2199 * backing objects [c.f. uvm_unmap_detach]. the object is to 2199 * backing objects [c.f. uvm_unmap_detach]. the object is to
2200 * separate unmapping from reference dropping. why? 2200 * separate unmapping from reference dropping. why?
2201 * [1] the map has to be locked for unmapping 2201 * [1] the map has to be locked for unmapping
2202 * [2] the map need not be locked for reference dropping 2202 * [2] the map need not be locked for reference dropping
2203 * [3] dropping references may trigger pager I/O, and if we hit 2203 * [3] dropping references may trigger pager I/O, and if we hit
2204 * a pager that does synchronous I/O we may have to wait for it. 2204 * a pager that does synchronous I/O we may have to wait for it.
2205 * [4] we would like all waiting for I/O to occur with maps unlocked 2205 * [4] we would like all waiting for I/O to occur with maps unlocked
2206 * so that we don't block other threads. 2206 * so that we don't block other threads.
2207 */ 2207 */
2208 2208
2209 first_entry = NULL; 2209 first_entry = NULL;
2210 *entry_list = NULL; 2210 *entry_list = NULL;
2211 2211
2212 /* 2212 /*
2213 * break up the area into map entry sized regions and unmap. note 2213 * break up the area into map entry sized regions and unmap. note
2214 * that all mappings have to be removed before we can even consider 2214 * that all mappings have to be removed before we can even consider
2215 * dropping references to amaps or VM objects (otherwise we could end 2215 * dropping references to amaps or VM objects (otherwise we could end
2216 * up with a mapping to a page on the free list which would be very bad) 2216 * up with a mapping to a page on the free list which would be very bad)
2217 */ 2217 */
2218 2218
2219 while ((entry != &map->header) && (entry->start < end)) { 2219 while ((entry != &map->header) && (entry->start < end)) {
2220 KASSERT((entry->flags & UVM_MAP_STATIC) == 0); 2220 KASSERT((entry->flags & UVM_MAP_STATIC) == 0);
2221 2221
2222 UVM_MAP_CLIP_END(map, entry, end); 2222 UVM_MAP_CLIP_END(map, entry, end);
2223 next = entry->next; 2223 next = entry->next;
2224 len = entry->end - entry->start; 2224 len = entry->end - entry->start;
2225 2225
2226 /* 2226 /*
2227 * unwire before removing addresses from the pmap; otherwise 2227 * unwire before removing addresses from the pmap; otherwise
2228 * unwiring will put the entries back into the pmap (XXX). 2228 * unwiring will put the entries back into the pmap (XXX).
2229 */ 2229 */
2230 2230
2231 if (VM_MAPENT_ISWIRED(entry)) { 2231 if (VM_MAPENT_ISWIRED(entry)) {
2232 uvm_map_entry_unwire(map, entry); 2232 uvm_map_entry_unwire(map, entry);
2233 } 2233 }
2234 if (flags & UVM_FLAG_VAONLY) { 2234 if (flags & UVM_FLAG_VAONLY) {
2235 2235
2236 /* nothing */ 2236 /* nothing */
2237 2237
2238 } else if ((map->flags & VM_MAP_PAGEABLE) == 0) { 2238 } else if ((map->flags & VM_MAP_PAGEABLE) == 0) {
2239 2239
2240 /* 2240 /*
2241 * if the map is non-pageable, any pages mapped there 2241 * if the map is non-pageable, any pages mapped there
2242 * must be wired and entered with pmap_kenter_pa(), 2242 * must be wired and entered with pmap_kenter_pa(),
2243 * and we should free any such pages immediately. 2243 * and we should free any such pages immediately.
2244 * this is mostly used for kmem_map. 2244 * this is mostly used for kmem_map.
2245 */ 2245 */
2246 KASSERT(vm_map_pmap(map) == pmap_kernel()); 2246 KASSERT(vm_map_pmap(map) == pmap_kernel());
2247 2247
2248 uvm_km_pgremove_intrsafe(map, entry->start, entry->end); 2248 uvm_km_pgremove_intrsafe(map, entry->start, entry->end);
2249 } else if (UVM_ET_ISOBJ(entry) && 2249 } else if (UVM_ET_ISOBJ(entry) &&
2250 UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj)) { 2250 UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj)) {
2251 panic("%s: kernel object %p %p\n", 2251 panic("%s: kernel object %p %p\n",
2252 __func__, map, entry); 2252 __func__, map, entry);
2253 } else if (UVM_ET_ISOBJ(entry) || entry->aref.ar_amap) { 2253 } else if (UVM_ET_ISOBJ(entry) || entry->aref.ar_amap) {
2254 /* 2254 /*
2255 * remove mappings the standard way. lock object 2255 * remove mappings the standard way. lock object
2256 * and/or amap to ensure vm_page state does not 2256 * and/or amap to ensure vm_page state does not
2257 * change while in pmap_remove(). 2257 * change while in pmap_remove().
2258 */ 2258 */
2259 2259
2260#ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */ 2260#ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */
2261 uvm_map_lock_entry(entry, RW_WRITER); 2261 uvm_map_lock_entry(entry, RW_WRITER);
2262#else 2262#else
2263 uvm_map_lock_entry(entry, RW_READER); 2263 uvm_map_lock_entry(entry, RW_READER);
2264#endif 2264#endif
2265 pmap_remove(map->pmap, entry->start, entry->end); 2265 pmap_remove(map->pmap, entry->start, entry->end);
2266 2266
2267 /* 2267 /*
2268 * note: if map is dying, leave pmap_update() for 2268 * note: if map is dying, leave pmap_update() for
2269 * later. if the map is to be reused (exec) then 2269 * later. if the map is to be reused (exec) then
2270 * pmap_update() will be called. if the map is 2270 * pmap_update() will be called. if the map is
2271 * being disposed of (exit) then pmap_destroy() 2271 * being disposed of (exit) then pmap_destroy()
2272 * will be called. 2272 * will be called.
2273 */ 2273 */
2274 2274
2275 if ((map->flags & VM_MAP_DYING) == 0) { 2275 if ((map->flags & VM_MAP_DYING) == 0) {
2276 pmap_update(vm_map_pmap(map)); 2276 pmap_update(vm_map_pmap(map));
2277 } else { 2277 } else {
2278 KASSERT(vm_map_pmap(map) != pmap_kernel()); 2278 KASSERT(vm_map_pmap(map) != pmap_kernel());
2279 } 2279 }
2280 2280
2281 uvm_map_unlock_entry(entry); 2281 uvm_map_unlock_entry(entry);
2282 } 2282 }
2283 2283
2284#if defined(UVMDEBUG) 2284#if defined(UVMDEBUG)
2285 /* 2285 /*
2286 * check if there's remaining mapping, 2286 * check if there's remaining mapping,
2287 * which is a bug in caller. 2287 * which is a bug in caller.
2288 */ 2288 */
2289 2289
2290 vaddr_t va; 2290 vaddr_t va;
2291 for (va = entry->start; va < entry->end; 2291 for (va = entry->start; va < entry->end;
2292 va += PAGE_SIZE) { 2292 va += PAGE_SIZE) {
2293 if (pmap_extract(vm_map_pmap(map), va, NULL)) { 2293 if (pmap_extract(vm_map_pmap(map), va, NULL)) {
2294 panic("%s: %#"PRIxVADDR" has mapping", 2294 panic("%s: %#"PRIxVADDR" has mapping",
2295 __func__, va); 2295 __func__, va);
2296 } 2296 }
2297 } 2297 }
2298 2298
2299 if (VM_MAP_IS_KERNEL(map) && (flags & UVM_FLAG_NOWAIT) == 0) { 2299 if (VM_MAP_IS_KERNEL(map) && (flags & UVM_FLAG_NOWAIT) == 0) {
2300 uvm_km_check_empty(map, entry->start, 2300 uvm_km_check_empty(map, entry->start,
2301 entry->end); 2301 entry->end);
2302 } 2302 }
2303#endif /* defined(UVMDEBUG) */ 2303#endif /* defined(UVMDEBUG) */
2304 2304
2305 /* 2305 /*
2306 * remove entry from map and put it on our list of entries 2306 * remove entry from map and put it on our list of entries
2307 * that we've nuked. then go to next entry. 2307 * that we've nuked. then go to next entry.
2308 */ 2308 */
2309 2309
2310 UVMHIST_LOG(maphist, " removed map entry %#jx", 2310 UVMHIST_LOG(maphist, " removed map entry %#jx",
2311 (uintptr_t)entry, 0, 0, 0); 2311 (uintptr_t)entry, 0, 0, 0);
2312 2312
2313 /* critical! prevents stale hint */ 2313 /* critical! prevents stale hint */
2314 SAVE_HINT(map, entry, entry->prev); 2314 SAVE_HINT(map, entry, entry->prev);
2315 2315
2316 uvm_map_entry_unlink(map, entry); 2316 uvm_map_entry_unlink(map, entry);
2317 KASSERT(map->size >= len); 2317 KASSERT(map->size >= len);
2318 map->size -= len; 2318 map->size -= len;
2319 entry->prev = NULL; 2319 entry->prev = NULL;
2320 entry->next = first_entry; 2320 entry->next = first_entry;
2321 first_entry = entry; 2321 first_entry = entry;
2322 entry = next; 2322 entry = next;
2323 } 2323 }
2324 2324
2325 uvm_map_check(map, "unmap_remove leave"); 2325 uvm_map_check(map, "unmap_remove leave");
2326 2326
2327 /* 2327 /*
2328 * now we've cleaned up the map and are ready for the caller to drop 2328 * now we've cleaned up the map and are ready for the caller to drop
2329 * references to the mapped objects. 2329 * references to the mapped objects.
2330 */ 2330 */
2331 2331
2332 *entry_list = first_entry; 2332 *entry_list = first_entry;
2333 UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0); 2333 UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
2334 2334
2335 if (map->flags & VM_MAP_WANTVA) { 2335 if (map->flags & VM_MAP_WANTVA) {
2336 mutex_enter(&map->misc_lock); 2336 mutex_enter(&map->misc_lock);
2337 map->flags &= ~VM_MAP_WANTVA; 2337 map->flags &= ~VM_MAP_WANTVA;
2338 cv_broadcast(&map->cv); 2338 cv_broadcast(&map->cv);
2339 mutex_exit(&map->misc_lock); 2339 mutex_exit(&map->misc_lock);
2340 } 2340 }
2341} 2341}
2342 2342
2343/* 2343/*
2344 * uvm_unmap_detach: drop references in a chain of map entries 2344 * uvm_unmap_detach: drop references in a chain of map entries
2345 * 2345 *
2346 * => we will free the map entries as we traverse the list. 2346 * => we will free the map entries as we traverse the list.
2347 */ 2347 */
2348 2348
2349void 2349void
2350uvm_unmap_detach(struct vm_map_entry *first_entry, int flags) 2350uvm_unmap_detach(struct vm_map_entry *first_entry, int flags)
2351{ 2351{
2352 struct vm_map_entry *next_entry; 2352 struct vm_map_entry *next_entry;
2353 UVMHIST_FUNC("uvm_unmap_detach"); UVMHIST_CALLED(maphist); 2353 UVMHIST_FUNC("uvm_unmap_detach"); UVMHIST_CALLED(maphist);
2354 2354
2355 while (first_entry) { 2355 while (first_entry) {
2356 KASSERT(!VM_MAPENT_ISWIRED(first_entry)); 2356 KASSERT(!VM_MAPENT_ISWIRED(first_entry));
2357 UVMHIST_LOG(maphist, 2357 UVMHIST_LOG(maphist,
2358 " detach %#jx: amap=%#jx, obj=%#jx, submap?=%jd", 2358 " detach %#jx: amap=%#jx, obj=%#jx, submap?=%jd",
2359 (uintptr_t)first_entry, 2359 (uintptr_t)first_entry,
2360 (uintptr_t)first_entry->aref.ar_amap, 2360 (uintptr_t)first_entry->aref.ar_amap,
2361 (uintptr_t)first_entry->object.uvm_obj, 2361 (uintptr_t)first_entry->object.uvm_obj,
2362 UVM_ET_ISSUBMAP(first_entry)); 2362 UVM_ET_ISSUBMAP(first_entry));
2363 2363
2364 /* 2364 /*
2365 * drop reference to amap, if we've got one 2365 * drop reference to amap, if we've got one
2366 */ 2366 */
2367 2367
2368 if (first_entry->aref.ar_amap) 2368 if (first_entry->aref.ar_amap)
2369 uvm_map_unreference_amap(first_entry, flags); 2369 uvm_map_unreference_amap(first_entry, flags);
2370 2370
2371 /* 2371 /*
2372 * drop reference to our backing object, if we've got one 2372 * drop reference to our backing object, if we've got one
2373 */ 2373 */
2374 2374
2375 KASSERT(!UVM_ET_ISSUBMAP(first_entry)); 2375 KASSERT(!UVM_ET_ISSUBMAP(first_entry));
2376 if (UVM_ET_ISOBJ(first_entry) && 2376 if (UVM_ET_ISOBJ(first_entry) &&
2377 first_entry->object.uvm_obj->pgops->pgo_detach) { 2377 first_entry->object.uvm_obj->pgops->pgo_detach) {
2378 (*first_entry->object.uvm_obj->pgops->pgo_detach) 2378 (*first_entry->object.uvm_obj->pgops->pgo_detach)
2379 (first_entry->object.uvm_obj); 2379 (first_entry->object.uvm_obj);
2380 } 2380 }
2381 next_entry = first_entry->next; 2381 next_entry = first_entry->next;
2382 uvm_mapent_free(first_entry); 2382 uvm_mapent_free(first_entry);
2383 first_entry = next_entry; 2383 first_entry = next_entry;
2384 } 2384 }
2385 UVMHIST_LOG(maphist, "<- done", 0,0,0,0); 2385 UVMHIST_LOG(maphist, "<- done", 0,0,0,0);
2386} 2386}
2387 2387
2388/* 2388/*
2389 * E X T R A C T I O N F U N C T I O N S 2389 * E X T R A C T I O N F U N C T I O N S
2390 */ 2390 */
2391 2391
2392/* 2392/*
2393 * uvm_map_reserve: reserve space in a vm_map for future use. 2393 * uvm_map_reserve: reserve space in a vm_map for future use.
2394 * 2394 *
2395 * => we reserve space in a map by putting a dummy map entry in the 2395 * => we reserve space in a map by putting a dummy map entry in the
2396 * map (dummy means obj=NULL, amap=NULL, prot=VM_PROT_NONE) 2396 * map (dummy means obj=NULL, amap=NULL, prot=VM_PROT_NONE)
2397 * => map should be unlocked (we will write lock it) 2397 * => map should be unlocked (we will write lock it)
2398 * => we return true if we were able to reserve space 2398 * => we return true if we were able to reserve space
2399 * => XXXCDC: should be inline? 2399 * => XXXCDC: should be inline?
2400 */ 2400 */
2401 2401
2402int 2402int
2403uvm_map_reserve(struct vm_map *map, vsize_t size, 2403uvm_map_reserve(struct vm_map *map, vsize_t size,
2404 vaddr_t offset /* hint for pmap_prefer */, 2404 vaddr_t offset /* hint for pmap_prefer */,
2405 vsize_t align /* alignment */, 2405 vsize_t align /* alignment */,
2406 vaddr_t *raddr /* IN:hint, OUT: reserved VA */, 2406 vaddr_t *raddr /* IN:hint, OUT: reserved VA */,
2407 uvm_flag_t flags /* UVM_FLAG_FIXED or UVM_FLAG_COLORMATCH or 0 */) 2407 uvm_flag_t flags /* UVM_FLAG_FIXED or UVM_FLAG_COLORMATCH or 0 */)
2408{ 2408{
2409 UVMHIST_FUNC("uvm_map_reserve"); UVMHIST_CALLED(maphist); 2409 UVMHIST_FUNC("uvm_map_reserve"); UVMHIST_CALLED(maphist);
2410 2410
2411 UVMHIST_LOG(maphist, "(map=%#jx, size=%#jx, offset=%#jx, addr=%#jx)", 2411 UVMHIST_LOG(maphist, "(map=%#jx, size=%#jx, offset=%#jx, addr=%#jx)",
2412 (uintptr_t)map, size, offset, (uintptr_t)raddr); 2412 (uintptr_t)map, size, offset, (uintptr_t)raddr);
2413 2413
2414 size = round_page(size); 2414 size = round_page(size);
2415 2415
2416 /* 2416 /*
2417 * reserve some virtual space. 2417 * reserve some virtual space.
2418 */ 2418 */
2419 2419
2420 if (uvm_map(map, raddr, size, NULL, offset, align, 2420 if (uvm_map(map, raddr, size, NULL, offset, align,
2421 UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE, 2421 UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE,
2422 UVM_ADV_RANDOM, UVM_FLAG_NOMERGE|flags)) != 0) { 2422 UVM_ADV_RANDOM, UVM_FLAG_NOMERGE|flags)) != 0) {
2423 UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0); 2423 UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0);
2424 return (false); 2424 return (false);
2425 } 2425 }
2426 2426
2427 UVMHIST_LOG(maphist, "<- done (*raddr=%#jx)", *raddr,0,0,0); 2427 UVMHIST_LOG(maphist, "<- done (*raddr=%#jx)", *raddr,0,0,0);
2428 return (true); 2428 return (true);
2429} 2429}
2430 2430
2431/* 2431/*
2432 * uvm_map_replace: replace a reserved (blank) area of memory with 2432 * uvm_map_replace: replace a reserved (blank) area of memory with
2433 * real mappings. 2433 * real mappings.
2434 * 2434 *
2435 * => caller must WRITE-LOCK the map 2435 * => caller must WRITE-LOCK the map
2436 * => we return true if replacement was a success 2436 * => we return true if replacement was a success
2437 * => we expect the newents chain to have nnewents entrys on it and 2437 * => we expect the newents chain to have nnewents entrys on it and
2438 * we expect newents->prev to point to the last entry on the list 2438 * we expect newents->prev to point to the last entry on the list
2439 * => note newents is allowed to be NULL 2439 * => note newents is allowed to be NULL
2440 */ 2440 */
2441 2441
2442static int 2442static int
2443uvm_map_replace(struct vm_map *map, vaddr_t start, vaddr_t end, 2443uvm_map_replace(struct vm_map *map, vaddr_t start, vaddr_t end,
2444 struct vm_map_entry *newents, int nnewents, vsize_t nsize, 2444 struct vm_map_entry *newents, int nnewents, vsize_t nsize,
2445 struct vm_map_entry **oldentryp) 2445 struct vm_map_entry **oldentryp)
2446{ 2446{
2447 struct vm_map_entry *oldent, *last; 2447 struct vm_map_entry *oldent, *last;
2448 2448
2449 uvm_map_check(map, "map_replace entry"); 2449 uvm_map_check(map, "map_replace entry");
2450 2450
2451 /* 2451 /*
2452 * first find the blank map entry at the specified address 2452 * first find the blank map entry at the specified address
2453 */ 2453 */
2454 2454
2455 if (!uvm_map_lookup_entry(map, start, &oldent)) { 2455 if (!uvm_map_lookup_entry(map, start, &oldent)) {
2456 return (false); 2456 return (false);
2457 } 2457 }
2458 2458
2459 /* 2459 /*
2460 * check to make sure we have a proper blank entry 2460 * check to make sure we have a proper blank entry
2461 */ 2461 */
2462 2462
2463 if (end < oldent->end) { 2463 if (end < oldent->end) {
2464 UVM_MAP_CLIP_END(map, oldent, end); 2464 UVM_MAP_CLIP_END(map, oldent, end);
2465 } 2465 }
2466 if (oldent->start != start || oldent->end != end || 2466 if (oldent->start != start || oldent->end != end ||
2467 oldent->object.uvm_obj != NULL || oldent->aref.ar_amap != NULL) { 2467 oldent->object.uvm_obj != NULL || oldent->aref.ar_amap != NULL) {
2468 return (false); 2468 return (false);
2469 } 2469 }
2470 2470
2471#ifdef DIAGNOSTIC 2471#ifdef DIAGNOSTIC
2472 2472
2473 /* 2473 /*
2474 * sanity check the newents chain 2474 * sanity check the newents chain
2475 */ 2475 */
2476 2476
2477 { 2477 {
2478 struct vm_map_entry *tmpent = newents; 2478 struct vm_map_entry *tmpent = newents;
2479 int nent = 0; 2479 int nent = 0;
2480 vsize_t sz = 0; 2480 vsize_t sz = 0;
2481 vaddr_t cur = start; 2481 vaddr_t cur = start;
2482 2482
2483 while (tmpent) { 2483 while (tmpent) {
2484 nent++; 2484 nent++;
2485 sz += tmpent->end - tmpent->start; 2485 sz += tmpent->end - tmpent->start;
2486 if (tmpent->start < cur) 2486 if (tmpent->start < cur)
2487 panic("uvm_map_replace1"); 2487 panic("uvm_map_replace1");
2488 if (tmpent->start >= tmpent->end || tmpent->end > end) { 2488 if (tmpent->start >= tmpent->end || tmpent->end > end) {
2489 panic("uvm_map_replace2: " 2489 panic("uvm_map_replace2: "
2490 "tmpent->start=%#"PRIxVADDR 2490 "tmpent->start=%#"PRIxVADDR
2491 ", tmpent->end=%#"PRIxVADDR 2491 ", tmpent->end=%#"PRIxVADDR
2492 ", end=%#"PRIxVADDR, 2492 ", end=%#"PRIxVADDR,
2493 tmpent->start, tmpent->end, end); 2493 tmpent->start, tmpent->end, end);
2494 } 2494 }
2495 cur = tmpent->end; 2495 cur = tmpent->end;
2496 if (tmpent->next) { 2496 if (tmpent->next) {
2497 if (tmpent->next->prev != tmpent) 2497 if (tmpent->next->prev != tmpent)
2498 panic("uvm_map_replace3"); 2498 panic("uvm_map_replace3");
2499 } else { 2499 } else {
2500 if (newents->prev != tmpent) 2500 if (newents->prev != tmpent)
2501 panic("uvm_map_replace4"); 2501 panic("uvm_map_replace4");
2502 } 2502 }
2503 tmpent = tmpent->next; 2503 tmpent = tmpent->next;
2504 } 2504 }
2505 if (nent != nnewents) 2505 if (nent != nnewents)
2506 panic("uvm_map_replace5"); 2506 panic("uvm_map_replace5");
2507 if (sz != nsize) 2507 if (sz != nsize)
2508 panic("uvm_map_replace6"); 2508 panic("uvm_map_replace6");
2509 } 2509 }
2510#endif 2510#endif
2511 2511
2512 /* 2512 /*
2513 * map entry is a valid blank! replace it. (this does all the 2513 * map entry is a valid blank! replace it. (this does all the
2514 * work of map entry link/unlink...). 2514 * work of map entry link/unlink...).
2515 */ 2515 */
2516 2516
2517 if (newents) { 2517 if (newents) {
2518 last = newents->prev; 2518 last = newents->prev;
2519 2519
2520 /* critical: flush stale hints out of map */ 2520 /* critical: flush stale hints out of map */
2521 SAVE_HINT(map, map->hint, newents); 2521 SAVE_HINT(map, map->hint, newents);
2522 if (map->first_free == oldent) 2522 if (map->first_free == oldent)
2523 map->first_free = last; 2523 map->first_free = last;
2524 2524
2525 last->next = oldent->next; 2525 last->next = oldent->next;
2526 last->next->prev = last; 2526 last->next->prev = last;
2527 2527
2528 /* Fix RB tree */ 2528 /* Fix RB tree */
2529 uvm_rb_remove(map, oldent); 2529 uvm_rb_remove(map, oldent);
2530 2530
2531 newents->prev = oldent->prev; 2531 newents->prev = oldent->prev;
2532 newents->prev->next = newents; 2532 newents->prev->next = newents;
2533 map->nentries = map->nentries + (nnewents - 1); 2533 map->nentries = map->nentries + (nnewents - 1);
2534 2534
2535 /* Fixup the RB tree */ 2535 /* Fixup the RB tree */
2536 { 2536 {
2537 int i; 2537 int i;
2538 struct vm_map_entry *tmp; 2538 struct vm_map_entry *tmp;
2539 2539
2540 tmp = newents; 2540 tmp = newents;
2541 for (i = 0; i < nnewents && tmp; i++) { 2541 for (i = 0; i < nnewents && tmp; i++) {
2542 uvm_rb_insert(map, tmp); 2542 uvm_rb_insert(map, tmp);
2543 tmp = tmp->next; 2543 tmp = tmp->next;
2544 } 2544 }
2545 } 2545 }
2546 } else { 2546 } else {
2547 /* NULL list of new entries: just remove the old one */ 2547 /* NULL list of new entries: just remove the old one */
2548 clear_hints(map, oldent); 2548 clear_hints(map, oldent);
2549 uvm_map_entry_unlink(map, oldent); 2549 uvm_map_entry_unlink(map, oldent);
2550 } 2550 }
2551 map->size -= end - start - nsize; 2551 map->size -= end - start - nsize;
2552 2552
2553 uvm_map_check(map, "map_replace leave"); 2553 uvm_map_check(map, "map_replace leave");
2554 2554
2555 /* 2555 /*
2556 * now we can free the old blank entry and return. 2556 * now we can free the old blank entry and return.
2557 */ 2557 */
2558 2558
2559 *oldentryp = oldent; 2559 *oldentryp = oldent;
2560 return (true); 2560 return (true);
2561} 2561}
2562 2562
2563/* 2563/*
2564 * uvm_map_extract: extract a mapping from a map and put it somewhere 2564 * uvm_map_extract: extract a mapping from a map and put it somewhere
2565 * (maybe removing the old mapping) 2565 * (maybe removing the old mapping)
2566 * 2566 *
2567 * => maps should be unlocked (we will write lock them) 2567 * => maps should be unlocked (we will write lock them)
2568 * => returns 0 on success, error code otherwise 2568 * => returns 0 on success, error code otherwise
2569 * => start must be page aligned 2569 * => start must be page aligned
2570 * => len must be page sized 2570 * => len must be page sized
2571 * => flags: 2571 * => flags:
2572 * UVM_EXTRACT_REMOVE: remove mappings from srcmap 2572 * UVM_EXTRACT_REMOVE: remove mappings from srcmap
2573 * UVM_EXTRACT_CONTIG: abort if unmapped area (advisory only) 2573 * UVM_EXTRACT_CONTIG: abort if unmapped area (advisory only)
2574 * UVM_EXTRACT_QREF: for a temporary extraction do quick obj refs 2574 * UVM_EXTRACT_QREF: for a temporary extraction do quick obj refs
2575 * UVM_EXTRACT_FIXPROT: set prot to maxprot as we go 2575 * UVM_EXTRACT_FIXPROT: set prot to maxprot as we go
2576 * UVM_EXTRACT_PROT_ALL: set prot to UVM_PROT_ALL as we go 2576 * UVM_EXTRACT_PROT_ALL: set prot to UVM_PROT_ALL as we go
2577 * >>>NOTE: if you set REMOVE, you are not allowed to use CONTIG or QREF!<<< 2577 * >>>NOTE: if you set REMOVE, you are not allowed to use CONTIG or QREF!<<<
2578 * >>>NOTE: QREF's must be unmapped via the QREF path, thus should only 2578 * >>>NOTE: QREF's must be unmapped via the QREF path, thus should only
2579 * be used from within the kernel in a kernel level map <<< 2579 * be used from within the kernel in a kernel level map <<<
2580 */ 2580 */
2581 2581
2582int 2582int
2583uvm_map_extract(struct vm_map *srcmap, vaddr_t start, vsize_t len, 2583uvm_map_extract(struct vm_map *srcmap, vaddr_t start, vsize_t len,
2584 struct vm_map *dstmap, vaddr_t *dstaddrp, int flags) 2584 struct vm_map *dstmap, vaddr_t *dstaddrp, int flags)
2585{ 2585{
2586 vaddr_t dstaddr, end, newend, oldoffset, fudge, orig_fudge; 2586 vaddr_t dstaddr, end, newend, oldoffset, fudge, orig_fudge;
2587 struct vm_map_entry *chain, *endchain, *entry, *orig_entry, *newentry, 2587 struct vm_map_entry *chain, *endchain, *entry, *orig_entry, *newentry,
2588 *deadentry, *oldentry; 2588 *deadentry, *oldentry;
2589 struct vm_map_entry *resentry = NULL; /* a dummy reservation entry */ 2589 struct vm_map_entry *resentry = NULL; /* a dummy reservation entry */
2590 vsize_t elen __unused; 2590 vsize_t elen __unused;
2591 int nchain, error, copy_ok; 2591 int nchain, error, copy_ok;
2592 vsize_t nsize; 2592 vsize_t nsize;
2593 UVMHIST_FUNC("uvm_map_extract"); UVMHIST_CALLED(maphist); 2593 UVMHIST_FUNC("uvm_map_extract"); UVMHIST_CALLED(maphist);
2594 2594
2595 UVMHIST_LOG(maphist,"(srcmap=%#jx,start=%#jx, len=%#jx", 2595 UVMHIST_LOG(maphist,"(srcmap=%#jx,start=%#jx, len=%#jx",
2596 (uintptr_t)srcmap, start, len, 0); 2596 (uintptr_t)srcmap, start, len, 0);
2597 UVMHIST_LOG(maphist," ...,dstmap=%#jx, flags=%#jx)", 2597 UVMHIST_LOG(maphist," ...,dstmap=%#jx, flags=%#jx)",
2598 (uintptr_t)dstmap, flags, 0, 0); 2598 (uintptr_t)dstmap, flags, 0, 0);
2599 2599
2600 /* 2600 /*
2601 * step 0: sanity check: start must be on a page boundary, length 2601 * step 0: sanity check: start must be on a page boundary, length
2602 * must be page sized. can't ask for CONTIG/QREF if you asked for 2602 * must be page sized. can't ask for CONTIG/QREF if you asked for
2603 * REMOVE. 2603 * REMOVE.
2604 */ 2604 */
2605 2605
2606 KASSERT((start & PAGE_MASK) == 0 && (len & PAGE_MASK) == 0); 2606 KASSERT((start & PAGE_MASK) == 0 && (len & PAGE_MASK) == 0);
2607 KASSERT((flags & UVM_EXTRACT_REMOVE) == 0 || 2607 KASSERT((flags & UVM_EXTRACT_REMOVE) == 0 ||
2608 (flags & (UVM_EXTRACT_CONTIG|UVM_EXTRACT_QREF)) == 0); 2608 (flags & (UVM_EXTRACT_CONTIG|UVM_EXTRACT_QREF)) == 0);
2609 2609
2610 /* 2610 /*
2611 * step 1: reserve space in the target map for the extracted area 2611 * step 1: reserve space in the target map for the extracted area
2612 */ 2612 */
2613 2613
2614 if ((flags & UVM_EXTRACT_RESERVED) == 0) { 2614 if ((flags & UVM_EXTRACT_RESERVED) == 0) {
2615 dstaddr = vm_map_min(dstmap); 2615 dstaddr = vm_map_min(dstmap);
2616 if (!uvm_map_reserve(dstmap, len, start,  2616 if (!uvm_map_reserve(dstmap, len, start,
2617 atop(start) & uvmexp.colormask, &dstaddr, 2617 atop(start) & uvmexp.colormask, &dstaddr,
2618 UVM_FLAG_COLORMATCH)) 2618 UVM_FLAG_COLORMATCH))
2619 return (ENOMEM); 2619 return (ENOMEM);
2620 KASSERT((atop(start ^ dstaddr) & uvmexp.colormask) == 0); 2620 KASSERT((atop(start ^ dstaddr) & uvmexp.colormask) == 0);
2621 *dstaddrp = dstaddr; /* pass address back to caller */ 2621 *dstaddrp = dstaddr; /* pass address back to caller */
2622 UVMHIST_LOG(maphist, " dstaddr=%#jx", dstaddr,0,0,0); 2622 UVMHIST_LOG(maphist, " dstaddr=%#jx", dstaddr,0,0,0);
2623 } else { 2623 } else {
2624 dstaddr = *dstaddrp; 2624 dstaddr = *dstaddrp;
2625 } 2625 }
2626 2626
2627 /* 2627 /*
2628 * step 2: setup for the extraction process loop by init'ing the 2628 * step 2: setup for the extraction process loop by init'ing the
2629 * map entry chain, locking src map, and looking up the first useful 2629 * map entry chain, locking src map, and looking up the first useful
2630 * entry in the map. 2630 * entry in the map.
2631 */ 2631 */
2632 2632
2633 end = start + len; 2633 end = start + len;
2634 newend = dstaddr + len; 2634 newend = dstaddr + len;
2635 chain = endchain = NULL; 2635 chain = endchain = NULL;
2636 nchain = 0; 2636 nchain = 0;
2637 nsize = 0; 2637 nsize = 0;
2638 vm_map_lock(srcmap); 2638 vm_map_lock(srcmap);
2639 2639
2640 if (uvm_map_lookup_entry(srcmap, start, &entry)) { 2640 if (uvm_map_lookup_entry(srcmap, start, &entry)) {
2641 2641
2642 /* "start" is within an entry */ 2642 /* "start" is within an entry */
2643 if (flags & UVM_EXTRACT_QREF) { 2643 if (flags & UVM_EXTRACT_QREF) {
2644 2644
2645 /* 2645 /*
2646 * for quick references we don't clip the entry, so 2646 * for quick references we don't clip the entry, so
2647 * the entry may map space "before" the starting 2647 * the entry may map space "before" the starting
2648 * virtual address... this is the "fudge" factor 2648 * virtual address... this is the "fudge" factor
2649 * (which can be non-zero only the first time 2649 * (which can be non-zero only the first time
2650 * through the "while" loop in step 3). 2650 * through the "while" loop in step 3).
2651 */ 2651 */
2652 2652
2653 fudge = start - entry->start; 2653 fudge = start - entry->start;
2654 } else { 2654 } else {
2655 2655
2656 /* 2656 /*
2657 * normal reference: we clip the map to fit (thus 2657 * normal reference: we clip the map to fit (thus
2658 * fudge is zero) 2658 * fudge is zero)
2659 */ 2659 */
2660 2660
2661 UVM_MAP_CLIP_START(srcmap, entry, start); 2661 UVM_MAP_CLIP_START(srcmap, entry, start);
2662 SAVE_HINT(srcmap, srcmap->hint, entry->prev); 2662 SAVE_HINT(srcmap, srcmap->hint, entry->prev);
2663 fudge = 0; 2663 fudge = 0;
2664 } 2664 }
2665 } else { 2665 } else {
2666 2666
2667 /* "start" is not within an entry ... skip to next entry */ 2667 /* "start" is not within an entry ... skip to next entry */
2668 if (flags & UVM_EXTRACT_CONTIG) { 2668 if (flags & UVM_EXTRACT_CONTIG) {
2669 error = EINVAL; 2669 error = EINVAL;
2670 goto bad; /* definite hole here ... */ 2670 goto bad; /* definite hole here ... */
2671 } 2671 }
2672 2672
2673 entry = entry->next; 2673 entry = entry->next;
2674 fudge = 0; 2674 fudge = 0;
2675 } 2675 }
2676 2676
2677 /* save values from srcmap for step 6 */ 2677 /* save values from srcmap for step 6 */
2678 orig_entry = entry; 2678 orig_entry = entry;
2679 orig_fudge = fudge; 2679 orig_fudge = fudge;
2680 2680
2681 /* 2681 /*
2682 * step 3: now start looping through the map entries, extracting 2682 * step 3: now start looping through the map entries, extracting
2683 * as we go. 2683 * as we go.
2684 */ 2684 */
2685 2685
2686 while (entry->start < end && entry != &srcmap->header) { 2686 while (entry->start < end && entry != &srcmap->header) {
2687 2687
2688 /* if we are not doing a quick reference, clip it */ 2688 /* if we are not doing a quick reference, clip it */
2689 if ((flags & UVM_EXTRACT_QREF) == 0) 2689 if ((flags & UVM_EXTRACT_QREF) == 0)
2690 UVM_MAP_CLIP_END(srcmap, entry, end); 2690 UVM_MAP_CLIP_END(srcmap, entry, end);
2691 2691
2692 /* clear needs_copy (allow chunking) */ 2692 /* clear needs_copy (allow chunking) */
2693 if (UVM_ET_ISNEEDSCOPY(entry)) { 2693 if (UVM_ET_ISNEEDSCOPY(entry)) {
2694 amap_copy(srcmap, entry, 2694 amap_copy(srcmap, entry,
2695 AMAP_COPY_NOWAIT|AMAP_COPY_NOMERGE, start, end); 2695 AMAP_COPY_NOWAIT|AMAP_COPY_NOMERGE, start, end);
2696 if (UVM_ET_ISNEEDSCOPY(entry)) { /* failed? */ 2696 if (UVM_ET_ISNEEDSCOPY(entry)) { /* failed? */
2697 error = ENOMEM; 2697 error = ENOMEM;
2698 goto bad; 2698 goto bad;
2699 } 2699 }
2700 2700
2701 /* amap_copy could clip (during chunk)! update fudge */ 2701 /* amap_copy could clip (during chunk)! update fudge */
2702 if (fudge) { 2702 if (fudge) {
2703 fudge = start - entry->start; 2703 fudge = start - entry->start;
2704 orig_fudge = fudge; 2704 orig_fudge = fudge;
2705 } 2705 }
2706 } 2706 }
2707 2707
2708 /* calculate the offset of this from "start" */ 2708 /* calculate the offset of this from "start" */
2709 oldoffset = (entry->start + fudge) - start; 2709 oldoffset = (entry->start + fudge) - start;
2710 2710
2711 /* allocate a new map entry */ 2711 /* allocate a new map entry */
2712 newentry = uvm_mapent_alloc(dstmap, 0); 2712 newentry = uvm_mapent_alloc(dstmap, 0);
2713 if (newentry == NULL) { 2713 if (newentry == NULL) {
2714 error = ENOMEM; 2714 error = ENOMEM;
2715 goto bad; 2715 goto bad;
2716 } 2716 }
2717 2717
2718 /* set up new map entry */ 2718 /* set up new map entry */
2719 newentry->next = NULL; 2719 newentry->next = NULL;
2720 newentry->prev = endchain; 2720 newentry->prev = endchain;
2721 newentry->start = dstaddr + oldoffset; 2721 newentry->start = dstaddr + oldoffset;
2722 newentry->end = 2722 newentry->end =
2723 newentry->start + (entry->end - (entry->start + fudge)); 2723 newentry->start + (entry->end - (entry->start + fudge));
2724 if (newentry->end > newend || newentry->end < newentry->start) 2724 if (newentry->end > newend || newentry->end < newentry->start)
2725 newentry->end = newend; 2725 newentry->end = newend;
2726 newentry->object.uvm_obj = entry->object.uvm_obj; 2726 newentry->object.uvm_obj = entry->object.uvm_obj;
2727 if (newentry->object.uvm_obj) { 2727 if (newentry->object.uvm_obj) {
2728 if (newentry->object.uvm_obj->pgops->pgo_reference) 2728 if (newentry->object.uvm_obj->pgops->pgo_reference)
2729 newentry->object.uvm_obj->pgops-> 2729 newentry->object.uvm_obj->pgops->
2730 pgo_reference(newentry->object.uvm_obj); 2730 pgo_reference(newentry->object.uvm_obj);
2731 newentry->offset = entry->offset + fudge; 2731 newentry->offset = entry->offset + fudge;
2732 } else { 2732 } else {
2733 newentry->offset = 0; 2733 newentry->offset = 0;
2734 } 2734 }
2735 newentry->etype = entry->etype; 2735 newentry->etype = entry->etype;
2736 if (flags & UVM_EXTRACT_PROT_ALL) { 2736 if (flags & UVM_EXTRACT_PROT_ALL) {
2737 newentry->protection = newentry->max_protection = 2737 newentry->protection = newentry->max_protection =
2738 UVM_PROT_ALL; 2738 UVM_PROT_ALL;
2739 } else { 2739 } else {
2740 newentry->protection = (flags & UVM_EXTRACT_FIXPROT) ? 2740 newentry->protection = (flags & UVM_EXTRACT_FIXPROT) ?
2741 entry->max_protection : entry->protection; 2741 entry->max_protection : entry->protection;
2742 newentry->max_protection = entry->max_protection; 2742 newentry->max_protection = entry->max_protection;
2743 } 2743 }
2744 newentry->inheritance = entry->inheritance; 2744 newentry->inheritance = entry->inheritance;
2745 newentry->wired_count = 0; 2745 newentry->wired_count = 0;
2746 newentry->aref.ar_amap = entry->aref.ar_amap; 2746 newentry->aref.ar_amap = entry->aref.ar_amap;
2747 if (newentry->aref.ar_amap) { 2747 if (newentry->aref.ar_amap) {
2748 newentry->aref.ar_pageoff = 2748 newentry->aref.ar_pageoff =
2749 entry->aref.ar_pageoff + (fudge >> PAGE_SHIFT); 2749 entry->aref.ar_pageoff + (fudge >> PAGE_SHIFT);
2750 uvm_map_reference_amap(newentry, AMAP_SHARED | 2750 uvm_map_reference_amap(newentry, AMAP_SHARED |
2751 ((flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0)); 2751 ((flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0));
2752 } else { 2752 } else {
2753 newentry->aref.ar_pageoff = 0; 2753 newentry->aref.ar_pageoff = 0;
2754 } 2754 }
2755 newentry->advice = entry->advice; 2755 newentry->advice = entry->advice;
2756 if ((flags & UVM_EXTRACT_QREF) != 0) { 2756 if ((flags & UVM_EXTRACT_QREF) != 0) {
2757 newentry->flags |= UVM_MAP_NOMERGE; 2757 newentry->flags |= UVM_MAP_NOMERGE;
2758 } 2758 }
2759 2759
2760 /* now link it on the chain */ 2760 /* now link it on the chain */
2761 nchain++; 2761 nchain++;
2762 nsize += newentry->end - newentry->start; 2762 nsize += newentry->end - newentry->start;
2763 if (endchain == NULL) { 2763 if (endchain == NULL) {
2764 chain = endchain = newentry; 2764 chain = endchain = newentry;
2765 } else { 2765 } else {
2766 endchain->next = newentry; 2766 endchain->next = newentry;
2767 endchain = newentry; 2767 endchain = newentry;
2768 } 2768 }
2769 2769
2770 /* end of 'while' loop! */ 2770 /* end of 'while' loop! */
2771 if ((flags & UVM_EXTRACT_CONTIG) && entry->end < end && 2771 if ((flags & UVM_EXTRACT_CONTIG) && entry->end < end &&
2772 (entry->next == &srcmap->header || 2772 (entry->next == &srcmap->header ||
2773 entry->next->start != entry->end)) { 2773 entry->next->start != entry->end)) {
2774 error = EINVAL; 2774 error = EINVAL;
2775 goto bad; 2775 goto bad;
2776 } 2776 }
2777 entry = entry->next; 2777 entry = entry->next;
2778 fudge = 0; 2778 fudge = 0;
2779 } 2779 }
2780 2780
2781 /* 2781 /*
2782 * step 4: close off chain (in format expected by uvm_map_replace) 2782 * step 4: close off chain (in format expected by uvm_map_replace)
2783 */ 2783 */
2784 2784
2785 if (chain) 2785 if (chain)
2786 chain->prev = endchain; 2786 chain->prev = endchain;
2787 2787
2788 /* 2788 /*
2789 * step 5: attempt to lock the dest map so we can pmap_copy. 2789 * step 5: attempt to lock the dest map so we can pmap_copy.
2790 * note usage of copy_ok: 2790 * note usage of copy_ok:
2791 * 1 => dstmap locked, pmap_copy ok, and we "replace" here (step 5) 2791 * 1 => dstmap locked, pmap_copy ok, and we "replace" here (step 5)
2792 * 0 => dstmap unlocked, NO pmap_copy, and we will "replace" in step 7 2792 * 0 => dstmap unlocked, NO pmap_copy, and we will "replace" in step 7
2793 */ 2793 */
2794 2794
2795 if (srcmap == dstmap || vm_map_lock_try(dstmap) == true) { 2795 if (srcmap == dstmap || vm_map_lock_try(dstmap) == true) {
2796 copy_ok = 1; 2796 copy_ok = 1;
2797 if (!uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain, 2797 if (!uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain,
2798 nchain, nsize, &resentry)) { 2798 nchain, nsize, &resentry)) {
2799 if (srcmap != dstmap) 2799 if (srcmap != dstmap)
2800 vm_map_unlock(dstmap); 2800 vm_map_unlock(dstmap);
2801 error = EIO; 2801 error = EIO;
2802 goto bad; 2802 goto bad;
2803 } 2803 }
2804 } else { 2804 } else {
2805 copy_ok = 0; 2805 copy_ok = 0;
2806 /* replace defered until step 7 */ 2806 /* replace defered until step 7 */
2807 } 2807 }
2808 2808
2809 /* 2809 /*
2810 * step 6: traverse the srcmap a second time to do the following: 2810 * step 6: traverse the srcmap a second time to do the following:
2811 * - if we got a lock on the dstmap do pmap_copy 2811 * - if we got a lock on the dstmap do pmap_copy
2812 * - if UVM_EXTRACT_REMOVE remove the entries 2812 * - if UVM_EXTRACT_REMOVE remove the entries
2813 * we make use of orig_entry and orig_fudge (saved in step 2) 2813 * we make use of orig_entry and orig_fudge (saved in step 2)
2814 */ 2814 */
2815 2815
2816 if (copy_ok || (flags & UVM_EXTRACT_REMOVE)) { 2816 if (copy_ok || (flags & UVM_EXTRACT_REMOVE)) {
2817 2817
2818 /* purge possible stale hints from srcmap */ 2818 /* purge possible stale hints from srcmap */
2819 if (flags & UVM_EXTRACT_REMOVE) { 2819 if (flags & UVM_EXTRACT_REMOVE) {
2820 SAVE_HINT(srcmap, srcmap->hint, orig_entry->prev); 2820 SAVE_HINT(srcmap, srcmap->hint, orig_entry->prev);
2821 if (srcmap->first_free != &srcmap->header && 2821 if (srcmap->first_free != &srcmap->header &&
2822 srcmap->first_free->start >= start) 2822 srcmap->first_free->start >= start)
2823 srcmap->first_free = orig_entry->prev; 2823 srcmap->first_free = orig_entry->prev;
2824 } 2824 }
2825 2825
2826 entry = orig_entry; 2826 entry = orig_entry;
2827 fudge = orig_fudge; 2827 fudge = orig_fudge;
2828 deadentry = NULL; /* for UVM_EXTRACT_REMOVE */ 2828 deadentry = NULL; /* for UVM_EXTRACT_REMOVE */
2829 2829
2830 while (entry->start < end && entry != &srcmap->header) { 2830 while (entry->start < end && entry != &srcmap->header) {
2831 if (copy_ok) { 2831 if (copy_ok) {
2832 oldoffset = (entry->start + fudge) - start; 2832 oldoffset = (entry->start + fudge) - start;
2833 elen = MIN(end, entry->end) - 2833 elen = MIN(end, entry->end) -
2834 (entry->start + fudge); 2834 (entry->start + fudge);
2835 pmap_copy(dstmap->pmap, srcmap->pmap, 2835 pmap_copy(dstmap->pmap, srcmap->pmap,
2836 dstaddr + oldoffset, elen, 2836 dstaddr + oldoffset, elen,
2837 entry->start + fudge); 2837 entry->start + fudge);
2838 } 2838 }
2839 2839
2840 /* we advance "entry" in the following if statement */ 2840 /* we advance "entry" in the following if statement */
2841 if (flags & UVM_EXTRACT_REMOVE) { 2841 if (flags & UVM_EXTRACT_REMOVE) {
2842#ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */ 2842#ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */
2843 uvm_map_lock_entry(entry, RW_WRITER); 2843 uvm_map_lock_entry(entry, RW_WRITER);
2844#else 2844#else
2845 uvm_map_lock_entry(entry, RW_READER); 2845 uvm_map_lock_entry(entry, RW_READER);
2846#endif 2846#endif
2847 pmap_remove(srcmap->pmap, entry->start, 2847 pmap_remove(srcmap->pmap, entry->start,
2848 entry->end); 2848 entry->end);
2849 uvm_map_unlock_entry(entry); 2849 uvm_map_unlock_entry(entry);
2850 oldentry = entry; /* save entry */ 2850 oldentry = entry; /* save entry */
2851 entry = entry->next; /* advance */ 2851 entry = entry->next; /* advance */
2852 uvm_map_entry_unlink(srcmap, oldentry); 2852 uvm_map_entry_unlink(srcmap, oldentry);
2853 /* add to dead list */ 2853 /* add to dead list */
2854 oldentry->next = deadentry; 2854 oldentry->next = deadentry;
2855 deadentry = oldentry; 2855 deadentry = oldentry;
2856 } else { 2856 } else {
2857 entry = entry->next; /* advance */ 2857 entry = entry->next; /* advance */
2858 } 2858 }
2859 2859
2860 /* end of 'while' loop */ 2860 /* end of 'while' loop */
2861 fudge = 0; 2861 fudge = 0;
2862 } 2862 }
2863 pmap_update(srcmap->pmap); 2863 pmap_update(srcmap->pmap);
2864 2864
2865 /* 2865 /*
2866 * unlock dstmap. we will dispose of deadentry in 2866 * unlock dstmap. we will dispose of deadentry in
2867 * step 7 if needed 2867 * step 7 if needed
2868 */ 2868 */
2869 2869
2870 if (copy_ok && srcmap != dstmap) 2870 if (copy_ok && srcmap != dstmap)
2871 vm_map_unlock(dstmap); 2871 vm_map_unlock(dstmap);
2872 2872
2873 } else { 2873 } else {
2874 deadentry = NULL; 2874 deadentry = NULL;
2875 } 2875 }
2876 2876
2877 /* 2877 /*
2878 * step 7: we are done with the source map, unlock. if copy_ok 2878 * step 7: we are done with the source map, unlock. if copy_ok
2879 * is 0 then we have not replaced the dummy mapping in dstmap yet 2879 * is 0 then we have not replaced the dummy mapping in dstmap yet
2880 * and we need to do so now. 2880 * and we need to do so now.
2881 */ 2881 */
2882 2882
2883 vm_map_unlock(srcmap); 2883 vm_map_unlock(srcmap);
2884 if ((flags & UVM_EXTRACT_REMOVE) && deadentry) 2884 if ((flags & UVM_EXTRACT_REMOVE) && deadentry)
2885 uvm_unmap_detach(deadentry, 0); /* dispose of old entries */ 2885 uvm_unmap_detach(deadentry, 0); /* dispose of old entries */
2886 2886
2887 /* now do the replacement if we didn't do it in step 5 */ 2887 /* now do the replacement if we didn't do it in step 5 */
2888 if (copy_ok == 0) { 2888 if (copy_ok == 0) {
2889 vm_map_lock(dstmap); 2889 vm_map_lock(dstmap);
2890 error = uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain, 2890 error = uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain,
2891 nchain, nsize, &resentry); 2891 nchain, nsize, &resentry);
2892 vm_map_unlock(dstmap); 2892 vm_map_unlock(dstmap);
2893 2893
2894 if (error == false) { 2894 if (error == false) {
2895 error = EIO; 2895 error = EIO;
2896 goto bad2; 2896 goto bad2;
2897 } 2897 }
2898 } 2898 }
2899 2899
2900 if (resentry != NULL) 2900 if (resentry != NULL)
2901 uvm_mapent_free(resentry); 2901 uvm_mapent_free(resentry);
2902 2902
2903 return (0); 2903 return (0);
2904 2904
2905 /* 2905 /*
2906 * bad: failure recovery 2906 * bad: failure recovery
2907 */ 2907 */
2908bad: 2908bad:
2909 vm_map_unlock(srcmap); 2909 vm_map_unlock(srcmap);
2910bad2: /* src already unlocked */ 2910bad2: /* src already unlocked */
2911 if (chain) 2911 if (chain)
2912 uvm_unmap_detach(chain, 2912 uvm_unmap_detach(chain,
2913 (flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0); 2913 (flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0);
2914 2914
2915 if (resentry != NULL) 2915 if (resentry != NULL)
2916 uvm_mapent_free(resentry); 2916 uvm_mapent_free(resentry);
2917 2917
2918 if ((flags & UVM_EXTRACT_RESERVED) == 0) { 2918 if ((flags & UVM_EXTRACT_RESERVED) == 0) {
2919 uvm_unmap(dstmap, dstaddr, dstaddr+len); /* ??? */ 2919 uvm_unmap(dstmap, dstaddr, dstaddr+len); /* ??? */
2920 } 2920 }
2921 return (error); 2921 return (error);
2922} 2922}
2923 2923
2924/* end of extraction functions */ 2924/* end of extraction functions */
2925 2925
2926/* 2926/*
2927 * uvm_map_submap: punch down part of a map into a submap 2927 * uvm_map_submap: punch down part of a map into a submap
2928 * 2928 *
2929 * => only the kernel_map is allowed to be submapped 2929 * => only the kernel_map is allowed to be submapped
2930 * => the purpose of submapping is to break up the locking granularity 2930 * => the purpose of submapping is to break up the locking granularity
2931 * of a larger map 2931 * of a larger map
2932 * => the range specified must have been mapped previously with a uvm_map() 2932 * => the range specified must have been mapped previously with a uvm_map()
2933 * call [with uobj==NULL] to create a blank map entry in the main map. 2933 * call [with uobj==NULL] to create a blank map entry in the main map.
2934 * [And it had better still be blank!] 2934 * [And it had better still be blank!]
2935 * => maps which contain submaps should never be copied or forked. 2935 * => maps which contain submaps should never be copied or forked.
2936 * => to remove a submap, use uvm_unmap() on the main map 2936 * => to remove a submap, use uvm_unmap() on the main map
2937 * and then uvm_map_deallocate() the submap. 2937 * and then uvm_map_deallocate() the submap.
2938 * => main map must be unlocked. 2938 * => main map must be unlocked.
2939 * => submap must have been init'd and have a zero reference count. 2939 * => submap must have been init'd and have a zero reference count.
2940 * [need not be locked as we don't actually reference it] 2940 * [need not be locked as we don't actually reference it]
2941 */ 2941 */
2942 2942
2943int 2943int
2944uvm_map_submap(struct vm_map *map, vaddr_t start, vaddr_t end, 2944uvm_map_submap(struct vm_map *map, vaddr_t start, vaddr_t end,
2945 struct vm_map *submap) 2945 struct vm_map *submap)
2946{ 2946{
2947 struct vm_map_entry *entry; 2947 struct vm_map_entry *entry;
2948 int error; 2948 int error;
2949 2949
2950 vm_map_lock(map); 2950 vm_map_lock(map);
2951 VM_MAP_RANGE_CHECK(map, start, end); 2951 VM_MAP_RANGE_CHECK(map, start, end);
2952 2952
2953 if (uvm_map_lookup_entry(map, start, &entry)) { 2953 if (uvm_map_lookup_entry(map, start, &entry)) {
2954 UVM_MAP_CLIP_START(map, entry, start); 2954 UVM_MAP_CLIP_START(map, entry, start);
2955 UVM_MAP_CLIP_END(map, entry, end); /* to be safe */ 2955 UVM_MAP_CLIP_END(map, entry, end); /* to be safe */
2956 } else { 2956 } else {
2957 entry = NULL; 2957 entry = NULL;
2958 } 2958 }
2959 2959
2960 if (entry != NULL && 2960 if (entry != NULL &&
2961 entry->start == start && entry->end == end && 2961 entry->start == start && entry->end == end &&
2962 entry->object.uvm_obj == NULL && entry->aref.ar_amap == NULL && 2962 entry->object.uvm_obj == NULL && entry->aref.ar_amap == NULL &&
2963 !UVM_ET_ISCOPYONWRITE(entry) && !UVM_ET_ISNEEDSCOPY(entry)) { 2963 !UVM_ET_ISCOPYONWRITE(entry) && !UVM_ET_ISNEEDSCOPY(entry)) {
2964 entry->etype |= UVM_ET_SUBMAP; 2964 entry->etype |= UVM_ET_SUBMAP;
2965 entry->object.sub_map = submap; 2965 entry->object.sub_map = submap;
2966 entry->offset = 0; 2966 entry->offset = 0;
2967 uvm_map_reference(submap); 2967 uvm_map_reference(submap);
2968 error = 0; 2968 error = 0;
2969 } else { 2969 } else {
2970 error = EINVAL; 2970 error = EINVAL;
2971 } 2971 }
2972 vm_map_unlock(map); 2972 vm_map_unlock(map);
2973 2973
2974 return error; 2974 return error;
2975} 2975}
2976 2976
2977/* 2977/*
2978 * uvm_map_protect_user: change map protection on behalf of the user. 2978 * uvm_map_protect_user: change map protection on behalf of the user.
2979 * Enforces PAX settings as necessary. 2979 * Enforces PAX settings as necessary.
2980 */ 2980 */
2981int 2981int
2982uvm_map_protect_user(struct lwp *l, vaddr_t start, vaddr_t end, 2982uvm_map_protect_user(struct lwp *l, vaddr_t start, vaddr_t end,
2983 vm_prot_t new_prot) 2983 vm_prot_t new_prot)
2984{ 2984{
2985 int error; 2985 int error;
2986 2986
2987 if ((error = PAX_MPROTECT_VALIDATE(l, new_prot))) 2987 if ((error = PAX_MPROTECT_VALIDATE(l, new_prot)))
2988 return error; 2988 return error;
2989 2989
2990 return uvm_map_protect(&l->l_proc->p_vmspace->vm_map, start, end, 2990 return uvm_map_protect(&l->l_proc->p_vmspace->vm_map, start, end,
2991 new_prot, false); 2991 new_prot, false);
2992} 2992}
2993 2993
2994 2994
2995/* 2995/*
2996 * uvm_map_protect: change map protection 2996 * uvm_map_protect: change map protection
2997 * 2997 *
2998 * => set_max means set max_protection. 2998 * => set_max means set max_protection.
2999 * => map must be unlocked. 2999 * => map must be unlocked.
3000 */ 3000 */
3001 3001
3002#define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 3002#define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
3003 ~VM_PROT_WRITE : VM_PROT_ALL) 3003 ~VM_PROT_WRITE : VM_PROT_ALL)
3004 3004
3005int 3005int
3006uvm_map_protect(struct vm_map *map, vaddr_t start, vaddr_t end, 3006uvm_map_protect(struct vm_map *map, vaddr_t start, vaddr_t end,
3007 vm_prot_t new_prot, bool set_max) 3007 vm_prot_t new_prot, bool set_max)
3008{ 3008{
3009 struct vm_map_entry *current, *entry; 3009 struct vm_map_entry *current, *entry;
3010 int error = 0; 3010 int error = 0;
3011 UVMHIST_FUNC("uvm_map_protect"); UVMHIST_CALLED(maphist); 3011 UVMHIST_FUNC("uvm_map_protect"); UVMHIST_CALLED(maphist);
3012 UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_prot=%#jx)", 3012 UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_prot=%#jx)",
3013 (uintptr_t)map, start, end, new_prot); 3013 (uintptr_t)map, start, end, new_prot);
3014 3014
3015 vm_map_lock(map); 3015 vm_map_lock(map);
3016 VM_MAP_RANGE_CHECK(map, start, end); 3016 VM_MAP_RANGE_CHECK(map, start, end);
3017 if (uvm_map_lookup_entry(map, start, &entry)) { 3017 if (uvm_map_lookup_entry(map, start, &entry)) {
3018 UVM_MAP_CLIP_START(map, entry, start); 3018 UVM_MAP_CLIP_START(map, entry, start);
3019 } else { 3019 } else {
3020 entry = entry->next; 3020 entry = entry->next;
3021 } 3021 }
3022 3022
3023 /* 3023 /*
3024 * make a first pass to check for protection violations. 3024 * make a first pass to check for protection violations.
3025 */ 3025 */
3026 3026
3027 current = entry; 3027 current = entry;
3028 while ((current != &map->header) && (current->start < end)) { 3028 while ((current != &map->header) && (current->start < end)) {
3029 if (UVM_ET_ISSUBMAP(current)) { 3029 if (UVM_ET_ISSUBMAP(current)) {
3030 error = EINVAL; 3030 error = EINVAL;
3031 goto out; 3031 goto out;
3032 } 3032 }
3033 if ((new_prot & current->max_protection) != new_prot) { 3033 if ((new_prot & current->max_protection) != new_prot) {
3034 error = EACCES; 3034 error = EACCES;
3035 goto out; 3035 goto out;
3036 } 3036 }
3037 /* 3037 /*
3038 * Don't allow VM_PROT_EXECUTE to be set on entries that 3038 * Don't allow VM_PROT_EXECUTE to be set on entries that
3039 * point to vnodes that are associated with a NOEXEC file 3039 * point to vnodes that are associated with a NOEXEC file
3040 * system. 3040 * system.
3041 */ 3041 */
3042 if (UVM_ET_ISOBJ(current) && 3042 if (UVM_ET_ISOBJ(current) &&
3043 UVM_OBJ_IS_VNODE(current->object.uvm_obj)) { 3043 UVM_OBJ_IS_VNODE(current->object.uvm_obj)) {
3044 struct vnode *vp = 3044 struct vnode *vp =
3045 (struct vnode *) current->object.uvm_obj; 3045 (struct vnode *) current->object.uvm_obj;
3046 3046
3047 if ((new_prot & VM_PROT_EXECUTE) != 0 && 3047 if ((new_prot & VM_PROT_EXECUTE) != 0 &&
3048 (vp->v_mount->mnt_flag & MNT_NOEXEC) != 0) { 3048 (vp->v_mount->mnt_flag & MNT_NOEXEC) != 0) {
3049 error = EACCES; 3049 error = EACCES;
3050 goto out; 3050 goto out;
3051 } 3051 }
3052 } 3052 }
3053 3053
3054 current = current->next; 3054 current = current->next;
3055 } 3055 }
3056 3056
3057 /* go back and fix up protections (no need to clip this time). */ 3057 /* go back and fix up protections (no need to clip this time). */
3058 3058
3059 current = entry; 3059 current = entry;
3060 while ((current != &map->header) && (current->start < end)) { 3060 while ((current != &map->header) && (current->start < end)) {
3061 vm_prot_t old_prot; 3061 vm_prot_t old_prot;
3062 3062
3063 UVM_MAP_CLIP_END(map, current, end); 3063 UVM_MAP_CLIP_END(map, current, end);
3064 old_prot = current->protection; 3064 old_prot = current->protection;
3065 if (set_max) 3065 if (set_max)
3066 current->protection = 3066 current->protection =
3067 (current->max_protection = new_prot) & old_prot; 3067 (current->max_protection = new_prot) & old_prot;
3068 else 3068 else
3069 current->protection = new_prot; 3069 current->protection = new_prot;
3070 3070
3071 /* 3071 /*
3072 * update physical map if necessary. worry about copy-on-write 3072 * update physical map if necessary. worry about copy-on-write
3073 * here -- CHECK THIS XXX 3073 * here -- CHECK THIS XXX
3074 */ 3074 */
3075 3075
3076 if (current->protection != old_prot) { 3076 if (current->protection != old_prot) {
3077 /* update pmap! */ 3077 /* update pmap! */
3078#ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */ 3078#ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */
3079 uvm_map_lock_entry(current, RW_WRITER); 3079 uvm_map_lock_entry(current, RW_WRITER);
3080#else 3080#else
3081 uvm_map_lock_entry(current, RW_READER); 3081 uvm_map_lock_entry(current, RW_READER);
3082#endif 3082#endif
3083 pmap_protect(map->pmap, current->start, current->end, 3083 pmap_protect(map->pmap, current->start, current->end,
3084 current->protection & MASK(current)); 3084 current->protection & MASK(current));
3085 uvm_map_unlock_entry(current); 3085 uvm_map_unlock_entry(current);
3086 3086
3087 /* 3087 /*
3088 * If this entry points at a vnode, and the 3088 * If this entry points at a vnode, and the
3089 * protection includes VM_PROT_EXECUTE, mark 3089 * protection includes VM_PROT_EXECUTE, mark
3090 * the vnode as VEXECMAP. 3090 * the vnode as VEXECMAP.
3091 */ 3091 */
3092 if (UVM_ET_ISOBJ(current)) { 3092 if (UVM_ET_ISOBJ(current)) {
3093 struct uvm_object *uobj = 3093 struct uvm_object *uobj =
3094 current->object.uvm_obj; 3094 current->object.uvm_obj;
3095 3095
3096 if (UVM_OBJ_IS_VNODE(uobj) && 3096 if (UVM_OBJ_IS_VNODE(uobj) &&
3097 (current->protection & VM_PROT_EXECUTE)) { 3097 (current->protection & VM_PROT_EXECUTE)) {
3098 vn_markexec((struct vnode *) uobj); 3098 vn_markexec((struct vnode *) uobj);
3099 } 3099 }
3100 } 3100 }
3101 } 3101 }
3102 3102
3103 /* 3103 /*
3104 * If the map is configured to lock any future mappings, 3104 * If the map is configured to lock any future mappings,
3105 * wire this entry now if the old protection was VM_PROT_NONE 3105 * wire this entry now if the old protection was VM_PROT_NONE
3106 * and the new protection is not VM_PROT_NONE. 3106 * and the new protection is not VM_PROT_NONE.
3107 */ 3107 */
3108 3108
3109 if ((map->flags & VM_MAP_WIREFUTURE) != 0 && 3109 if ((map->flags & VM_MAP_WIREFUTURE) != 0 &&
3110 VM_MAPENT_ISWIRED(current) == 0 && 3110 VM_MAPENT_ISWIRED(current) == 0 &&
3111 old_prot == VM_PROT_NONE && 3111 old_prot == VM_PROT_NONE &&
3112 new_prot != VM_PROT_NONE) { 3112 new_prot != VM_PROT_NONE) {
3113 3113
3114 /* 3114 /*
3115 * We must call pmap_update() here because the 3115 * We must call pmap_update() here because the
3116 * pmap_protect() call above might have removed some 3116 * pmap_protect() call above might have removed some
3117 * pmap entries and uvm_map_pageable() might create 3117 * pmap entries and uvm_map_pageable() might create
3118 * some new pmap entries that rely on the prior 3118 * some new pmap entries that rely on the prior
3119 * removals being completely finished. 3119 * removals being completely finished.
3120 */ 3120 */
3121 3121
3122 pmap_update(map->pmap); 3122 pmap_update(map->pmap);
3123 3123
3124 if (uvm_map_pageable(map, current->start, 3124 if (uvm_map_pageable(map, current->start,
3125 current->end, false, 3125 current->end, false,
3126 UVM_LK_ENTER|UVM_LK_EXIT) != 0) { 3126 UVM_LK_ENTER|UVM_LK_EXIT) != 0) {
3127 3127
3128 /* 3128 /*
3129 * If locking the entry fails, remember the 3129 * If locking the entry fails, remember the
3130 * error if it's the first one. Note we 3130 * error if it's the first one. Note we
3131 * still continue setting the protection in 3131 * still continue setting the protection in
3132 * the map, but will return the error 3132 * the map, but will return the error
3133 * condition regardless. 3133 * condition regardless.
3134 * 3134 *
3135 * XXX Ignore what the actual error is, 3135 * XXX Ignore what the actual error is,
3136 * XXX just call it a resource shortage 3136 * XXX just call it a resource shortage
3137 * XXX so that it doesn't get confused 3137 * XXX so that it doesn't get confused
3138 * XXX what uvm_map_protect() itself would 3138 * XXX what uvm_map_protect() itself would
3139 * XXX normally return. 3139 * XXX normally return.
3140 */ 3140 */
3141 3141
3142 error = ENOMEM; 3142 error = ENOMEM;
3143 } 3143 }
3144 } 3144 }
3145 current = current->next; 3145 current = current->next;
3146 } 3146 }
3147 pmap_update(map->pmap); 3147 pmap_update(map->pmap);
3148 3148
3149 out: 3149 out:
3150 vm_map_unlock(map); 3150 vm_map_unlock(map);
3151 3151
3152 UVMHIST_LOG(maphist, "<- done, error=%jd",error,0,0,0); 3152 UVMHIST_LOG(maphist, "<- done, error=%jd",error,0,0,0);
3153 return error; 3153 return error;
3154} 3154}
3155 3155
3156#undef MASK 3156#undef MASK
3157 3157
3158/* 3158/*
3159 * uvm_map_inherit: set inheritance code for range of addrs in map. 3159 * uvm_map_inherit: set inheritance code for range of addrs in map.
3160 * 3160 *
3161 * => map must be unlocked 3161 * => map must be unlocked
3162 * => note that the inherit code is used during a "fork". see fork 3162 * => note that the inherit code is used during a "fork". see fork
3163 * code for details. 3163 * code for details.
3164 */ 3164 */
3165 3165
3166int 3166int
3167uvm_map_inherit(struct vm_map *map, vaddr_t start, vaddr_t end, 3167uvm_map_inherit(struct vm_map *map, vaddr_t start, vaddr_t end,
3168 vm_inherit_t new_inheritance) 3168 vm_inherit_t new_inheritance)
3169{ 3169{
3170 struct vm_map_entry *entry, *temp_entry; 3170 struct vm_map_entry *entry, *temp_entry;
3171 UVMHIST_FUNC("uvm_map_inherit"); UVMHIST_CALLED(maphist); 3171 UVMHIST_FUNC("uvm_map_inherit"); UVMHIST_CALLED(maphist);
3172 UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_inh=%#jx)", 3172 UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_inh=%#jx)",
3173 (uintptr_t)map, start, end, new_inheritance); 3173 (uintptr_t)map, start, end, new_inheritance);
3174 3174
3175 switch (new_inheritance) { 3175 switch (new_inheritance) {
3176 case MAP_INHERIT_NONE: 3176 case MAP_INHERIT_NONE:
3177 case MAP_INHERIT_COPY: 3177 case MAP_INHERIT_COPY:
3178 case MAP_INHERIT_SHARE: 3178 case MAP_INHERIT_SHARE:
3179 case MAP_INHERIT_ZERO: 3179 case MAP_INHERIT_ZERO:
3180 break; 3180 break;
3181 default: 3181 default:
3182 UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0); 3182 UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
3183 return EINVAL; 3183 return EINVAL;
3184 } 3184 }
3185 3185
3186 vm_map_lock(map); 3186 vm_map_lock(map);
3187 VM_MAP_RANGE_CHECK(map, start, end); 3187 VM_MAP_RANGE_CHECK(map, start, end);
3188 if (uvm_map_lookup_entry(map, start, &temp_entry)) { 3188 if (uvm_map_lookup_entry(map, start, &temp_entry)) {
3189 entry = temp_entry; 3189 entry = temp_entry;
3190 UVM_MAP_CLIP_START(map, entry, start); 3190 UVM_MAP_CLIP_START(map, entry, start);
3191 } else { 3191 } else {
3192 entry = temp_entry->next; 3192 entry = temp_entry->next;
3193 } 3193 }
3194 while ((entry != &map->header) && (entry->start < end)) { 3194 while ((entry != &map->header) && (entry->start < end)) {
3195 UVM_MAP_CLIP_END(map, entry, end); 3195 UVM_MAP_CLIP_END(map, entry, end);
3196 entry->inheritance = new_inheritance; 3196 entry->inheritance = new_inheritance;
3197 entry = entry->next; 3197 entry = entry->next;
3198 } 3198 }
3199 vm_map_unlock(map); 3199 vm_map_unlock(map);
3200 UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0); 3200 UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
3201 return 0; 3201 return 0;
3202} 3202}
3203 3203
3204/* 3204/*
3205 * uvm_map_advice: set advice code for range of addrs in map. 3205 * uvm_map_advice: set advice code for range of addrs in map.
3206 * 3206 *
3207 * => map must be unlocked 3207 * => map must be unlocked
3208 */ 3208 */
3209 3209
3210int 3210int
3211uvm_map_advice(struct vm_map *map, vaddr_t start, vaddr_t end, int new_advice) 3211uvm_map_advice(struct vm_map *map, vaddr_t start, vaddr_t end, int new_advice)
3212{ 3212{
3213 struct vm_map_entry *entry, *temp_entry; 3213 struct vm_map_entry *entry, *temp_entry;
3214 UVMHIST_FUNC("uvm_map_advice"); UVMHIST_CALLED(maphist); 3214 UVMHIST_FUNC("uvm_map_advice"); UVMHIST_CALLED(maphist);
3215 UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_adv=%#jx)", 3215 UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_adv=%#jx)",
3216 (uintptr_t)map, start, end, new_advice); 3216 (uintptr_t)map, start, end, new_advice);
3217 3217
3218 vm_map_lock(map); 3218 vm_map_lock(map);
3219 VM_MAP_RANGE_CHECK(map, start, end); 3219 VM_MAP_RANGE_CHECK(map, start, end);
3220 if (uvm_map_lookup_entry(map, start, &temp_entry)) { 3220 if (uvm_map_lookup_entry(map, start, &temp_entry)) {
3221 entry = temp_entry; 3221 entry = temp_entry;
3222 UVM_MAP_CLIP_START(map, entry, start); 3222 UVM_MAP_CLIP_START(map, entry, start);
3223 } else { 3223 } else {
3224 entry = temp_entry->next; 3224 entry = temp_entry->next;
3225 } 3225 }
3226 3226
3227 /* 3227 /*
3228 * XXXJRT: disallow holes? 3228 * XXXJRT: disallow holes?
3229 */ 3229 */
3230 3230
3231 while ((entry != &map->header) && (entry->start < end)) { 3231 while ((entry != &map->header) && (entry->start < end)) {
3232 UVM_MAP_CLIP_END(map, entry, end); 3232 UVM_MAP_CLIP_END(map, entry, end);
3233 3233
3234 switch (new_advice) { 3234 switch (new_advice) {
3235 case MADV_NORMAL: 3235 case MADV_NORMAL:
3236 case MADV_RANDOM: 3236 case MADV_RANDOM:
3237 case MADV_SEQUENTIAL: 3237 case MADV_SEQUENTIAL:
3238 /* nothing special here */ 3238 /* nothing special here */
3239 break; 3239 break;
3240 3240
3241 default: 3241 default:
3242 vm_map_unlock(map); 3242 vm_map_unlock(map);
3243 UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0); 3243 UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
3244 return EINVAL; 3244 return EINVAL;
3245 } 3245 }
3246 entry->advice = new_advice; 3246 entry->advice = new_advice;
3247 entry = entry->next; 3247 entry = entry->next;
3248 } 3248 }
3249 3249
3250 vm_map_unlock(map); 3250 vm_map_unlock(map);
3251 UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0); 3251 UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
3252 return 0; 3252 return 0;
3253} 3253}
3254 3254
3255/* 3255/*
3256 * uvm_map_willneed: apply MADV_WILLNEED 3256 * uvm_map_willneed: apply MADV_WILLNEED
3257 */ 3257 */
3258 3258
3259int 3259int
3260uvm_map_willneed(struct vm_map *map, vaddr_t start, vaddr_t end) 3260uvm_map_willneed(struct vm_map *map, vaddr_t start, vaddr_t end)
3261{ 3261{
3262 struct vm_map_entry *entry; 3262 struct vm_map_entry *entry;
3263 UVMHIST_FUNC("uvm_map_willneed"); UVMHIST_CALLED(maphist); 3263 UVMHIST_FUNC("uvm_map_willneed"); UVMHIST_CALLED(maphist);
3264 UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx)", 3264 UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx)",
3265 (uintptr_t)map, start, end, 0); 3265 (uintptr_t)map, start, end, 0);
3266 3266
3267 vm_map_lock_read(map); 3267 vm_map_lock_read(map);
3268 VM_MAP_RANGE_CHECK(map, start, end); 3268 VM_MAP_RANGE_CHECK(map, start, end);
3269 if (!uvm_map_lookup_entry(map, start, &entry)) { 3269 if (!uvm_map_lookup_entry(map, start, &entry)) {
3270 entry = entry->next; 3270 entry = entry->next;
3271 } 3271 }
3272 while (entry->start < end) { 3272 while (entry->start < end) {
3273 struct vm_amap * const amap = entry->aref.ar_amap; 3273 struct vm_amap * const amap = entry->aref.ar_amap;
3274 struct uvm_object * const uobj = entry->object.uvm_obj; 3274 struct uvm_object * const uobj = entry->object.uvm_obj;
3275 3275
3276 KASSERT(entry != &map->header); 3276 KASSERT(entry != &map->header);
3277 KASSERT(start < entry->end); 3277 KASSERT(start < entry->end);
3278 /* 3278 /*
3279 * For now, we handle only the easy but commonly-requested case. 3279 * For now, we handle only the easy but commonly-requested case.
3280 * ie. start prefetching of backing uobj pages. 3280 * ie. start prefetching of backing uobj pages.
3281 * 3281 *
3282 * XXX It might be useful to pmap_enter() the already-in-core 3282 * XXX It might be useful to pmap_enter() the already-in-core
3283 * pages by inventing a "weak" mode for uvm_fault() which would 3283 * pages by inventing a "weak" mode for uvm_fault() which would
3284 * only do the PGO_LOCKED pgo_get(). 3284 * only do the PGO_LOCKED pgo_get().
3285 */ 3285 */
3286 if (UVM_ET_ISOBJ(entry) && amap == NULL && uobj != NULL) { 3286 if (UVM_ET_ISOBJ(entry) && amap == NULL && uobj != NULL) {
3287 off_t offset; 3287 off_t offset;
3288 off_t size; 3288 off_t size;
3289 3289
3290 offset = entry->offset; 3290 offset = entry->offset;
3291 if (start < entry->start) { 3291 if (start < entry->start) {
3292 offset += entry->start - start; 3292 offset += entry->start - start;
3293 } 3293 }
3294 size = entry->offset + (entry->end - entry->start); 3294 size = entry->offset + (entry->end - entry->start);
3295 if (entry->end < end) { 3295 if (entry->end < end) {
3296 size -= end - entry->end; 3296 size -= end - entry->end;
3297 } 3297 }
3298 uvm_readahead(uobj, offset, size); 3298 uvm_readahead(uobj, offset, size);
3299 } 3299 }
3300 entry = entry->next; 3300 entry = entry->next;
3301 } 3301 }
3302 vm_map_unlock_read(map); 3302 vm_map_unlock_read(map);
3303 UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0); 3303 UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
3304 return 0; 3304 return 0;
3305} 3305}
3306 3306
3307/* 3307/*
3308 * uvm_map_pageable: sets the pageability of a range in a map. 3308 * uvm_map_pageable: sets the pageability of a range in a map.
3309 * 3309 *
3310 * => wires map entries. should not be used for transient page locking. 3310 * => wires map entries. should not be used for transient page locking.
3311 * for that, use uvm_fault_wire()/uvm_fault_unwire() (see uvm_vslock()). 3311 * for that, use uvm_fault_wire()/uvm_fault_unwire() (see uvm_vslock()).
3312 * => regions specified as not pageable require lock-down (wired) memory 3312 * => regions specified as not pageable require lock-down (wired) memory
3313 * and page tables. 3313 * and page tables.
3314 * => map must never be read-locked 3314 * => map must never be read-locked
3315 * => if islocked is true, map is already write-locked 3315 * => if islocked is true, map is already write-locked
3316 * => we always unlock the map, since we must downgrade to a read-lock 3316 * => we always unlock the map, since we must downgrade to a read-lock
3317 * to call uvm_fault_wire() 3317 * to call uvm_fault_wire()
3318 * => XXXCDC: check this and try and clean it up. 3318 * => XXXCDC: check this and try and clean it up.
3319 */ 3319 */
3320 3320
3321int 3321int
3322uvm_map_pageable(struct vm_map *map, vaddr_t start, vaddr_t end, 3322uvm_map_pageable(struct vm_map *map, vaddr_t start, vaddr_t end,
3323 bool new_pageable, int lockflags) 3323 bool new_pageable, int lockflags)
3324{ 3324{
3325 struct vm_map_entry *entry, *start_entry, *failed_entry; 3325 struct vm_map_entry *entry, *start_entry, *failed_entry;
3326 int rv; 3326 int rv;
3327#ifdef DIAGNOSTIC 3327#ifdef DIAGNOSTIC
3328 u_int timestamp_save; 3328 u_int timestamp_save;
3329#endif 3329#endif
3330 UVMHIST_FUNC("uvm_map_pageable"); UVMHIST_CALLED(maphist); 3330 UVMHIST_FUNC("uvm_map_pageable"); UVMHIST_CALLED(maphist);
3331 UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_pageable=%ju)", 3331 UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,new_pageable=%ju)",
3332 (uintptr_t)map, start, end, new_pageable); 3332 (uintptr_t)map, start, end, new_pageable);
3333 KASSERT(map->flags & VM_MAP_PAGEABLE); 3333 KASSERT(map->flags & VM_MAP_PAGEABLE);
3334 3334
3335 if ((lockflags & UVM_LK_ENTER) == 0) 3335 if ((lockflags & UVM_LK_ENTER) == 0)
3336 vm_map_lock(map); 3336 vm_map_lock(map);
3337 VM_MAP_RANGE_CHECK(map, start, end); 3337 VM_MAP_RANGE_CHECK(map, start, end);
3338 3338
3339 /* 3339 /*
3340 * only one pageability change may take place at one time, since 3340 * only one pageability change may take place at one time, since
3341 * uvm_fault_wire assumes it will be called only once for each 3341 * uvm_fault_wire assumes it will be called only once for each
3342 * wiring/unwiring. therefore, we have to make sure we're actually 3342 * wiring/unwiring. therefore, we have to make sure we're actually
3343 * changing the pageability for the entire region. we do so before 3343 * changing the pageability for the entire region. we do so before
3344 * making any changes. 3344 * making any changes.
3345 */ 3345 */
3346 3346
3347 if (uvm_map_lookup_entry(map, start, &start_entry) == false) { 3347 if (uvm_map_lookup_entry(map, start, &start_entry) == false) {
3348 if ((lockflags & UVM_LK_EXIT) == 0) 3348 if ((lockflags & UVM_LK_EXIT) == 0)
3349 vm_map_unlock(map); 3349 vm_map_unlock(map);
3350 3350
3351 UVMHIST_LOG(maphist,"<- done (fault)",0,0,0,0); 3351 UVMHIST_LOG(maphist,"<- done (fault)",0,0,0,0);
3352 return EFAULT; 3352 return EFAULT;
3353 } 3353 }
3354 entry = start_entry; 3354 entry = start_entry;
3355 3355
3356 if (start == end) { /* nothing required */ 3356 if (start == end) { /* nothing required */
3357 if ((lockflags & UVM_LK_EXIT) == 0) 3357 if ((lockflags & UVM_LK_EXIT) == 0)
3358 vm_map_unlock(map); 3358 vm_map_unlock(map);
3359 3359
3360 UVMHIST_LOG(maphist,"<- done (nothing)",0,0,0,0); 3360 UVMHIST_LOG(maphist,"<- done (nothing)",0,0,0,0);
3361 return 0; 3361 return 0;
3362 } 3362 }
3363 3363
3364 /* 3364 /*
3365 * handle wiring and unwiring separately. 3365 * handle wiring and unwiring separately.
3366 */ 3366 */
3367 3367
3368 if (new_pageable) { /* unwire */ 3368 if (new_pageable) { /* unwire */
3369 UVM_MAP_CLIP_START(map, entry, start); 3369 UVM_MAP_CLIP_START(map, entry, start);
3370 3370
3371 /* 3371 /*
3372 * unwiring. first ensure that the range to be unwired is 3372 * unwiring. first ensure that the range to be unwired is
3373 * really wired down and that there are no holes. 3373 * really wired down and that there are no holes.
3374 */ 3374 */
3375 3375
3376 while ((entry != &map->header) && (entry->start < end)) { 3376 while ((entry != &map->header) && (entry->start < end)) {
3377 if (entry->wired_count == 0 || 3377 if (entry->wired_count == 0 ||
3378 (entry->end < end && 3378 (entry->end < end &&
3379 (entry->next == &map->header || 3379 (entry->next == &map->header ||
3380 entry->next->start > entry->end))) { 3380 entry->next->start > entry->end))) {
3381 if ((lockflags & UVM_LK_EXIT) == 0) 3381 if ((lockflags & UVM_LK_EXIT) == 0)
3382 vm_map_unlock(map); 3382 vm_map_unlock(map);
3383 UVMHIST_LOG(maphist, "<- done (INVAL)",0,0,0,0); 3383 UVMHIST_LOG(maphist, "<- done (INVAL)",0,0,0,0);
3384 return EINVAL; 3384 return EINVAL;
3385 } 3385 }
3386 entry = entry->next; 3386 entry = entry->next;
3387 } 3387 }
3388 3388
3389 /* 3389 /*
3390 * POSIX 1003.1b - a single munlock call unlocks a region, 3390 * POSIX 1003.1b - a single munlock call unlocks a region,
3391 * regardless of the number of mlock calls made on that 3391 * regardless of the number of mlock calls made on that
3392 * region. 3392 * region.
3393 */ 3393 */
3394 3394
3395 entry = start_entry; 3395 entry = start_entry;
3396 while ((entry != &map->header) && (entry->start < end)) { 3396 while ((entry != &map->header) && (entry->start < end)) {
3397 UVM_MAP_CLIP_END(map, entry, end); 3397 UVM_MAP_CLIP_END(map, entry, end);
3398 if (VM_MAPENT_ISWIRED(entry)) 3398 if (VM_MAPENT_ISWIRED(entry))
3399 uvm_map_entry_unwire(map, entry); 3399 uvm_map_entry_unwire(map, entry);
3400 entry = entry->next; 3400 entry = entry->next;
3401 } 3401 }
3402 if ((lockflags & UVM_LK_EXIT) == 0) 3402 if ((lockflags & UVM_LK_EXIT) == 0)
3403 vm_map_unlock(map); 3403 vm_map_unlock(map);
3404 UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0); 3404 UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0);
3405 return 0; 3405 return 0;
3406 } 3406 }
3407 3407
3408 /* 3408 /*
3409 * wire case: in two passes [XXXCDC: ugly block of code here] 3409 * wire case: in two passes [XXXCDC: ugly block of code here]
3410 * 3410 *
3411 * 1: holding the write lock, we create any anonymous maps that need 3411 * 1: holding the write lock, we create any anonymous maps that need
3412 * to be created. then we clip each map entry to the region to 3412 * to be created. then we clip each map entry to the region to
3413 * be wired and increment its wiring count. 3413 * be wired and increment its wiring count.
3414 * 3414 *
3415 * 2: we downgrade to a read lock, and call uvm_fault_wire to fault 3415 * 2: we downgrade to a read lock, and call uvm_fault_wire to fault
3416 * in the pages for any newly wired area (wired_count == 1). 3416 * in the pages for any newly wired area (wired_count == 1).
3417 * 3417 *
3418 * downgrading to a read lock for uvm_fault_wire avoids a possible 3418 * downgrading to a read lock for uvm_fault_wire avoids a possible
3419 * deadlock with another thread that may have faulted on one of 3419 * deadlock with another thread that may have faulted on one of
3420 * the pages to be wired (it would mark the page busy, blocking 3420 * the pages to be wired (it would mark the page busy, blocking
3421 * us, then in turn block on the map lock that we hold). because 3421 * us, then in turn block on the map lock that we hold). because
3422 * of problems in the recursive lock package, we cannot upgrade 3422 * of problems in the recursive lock package, we cannot upgrade
3423 * to a write lock in vm_map_lookup. thus, any actions that 3423 * to a write lock in vm_map_lookup. thus, any actions that
3424 * require the write lock must be done beforehand. because we 3424 * require the write lock must be done beforehand. because we
3425 * keep the read lock on the map, the copy-on-write status of the 3425 * keep the read lock on the map, the copy-on-write status of the
3426 * entries we modify here cannot change. 3426 * entries we modify here cannot change.
3427 */ 3427 */
3428 3428
3429 while ((entry != &map->header) && (entry->start < end)) { 3429 while ((entry != &map->header) && (entry->start < end)) {
3430 if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */ 3430 if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
3431 3431
3432 /* 3432 /*
3433 * perform actions of vm_map_lookup that need the 3433 * perform actions of vm_map_lookup that need the
3434 * write lock on the map: create an anonymous map 3434 * write lock on the map: create an anonymous map
3435 * for a copy-on-write region, or an anonymous map 3435 * for a copy-on-write region, or an anonymous map
3436 * for a zero-fill region. (XXXCDC: submap case 3436 * for a zero-fill region. (XXXCDC: submap case
3437 * ok?) 3437 * ok?)
3438 */ 3438 */
3439 3439
3440 if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */ 3440 if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */
3441 if (UVM_ET_ISNEEDSCOPY(entry) && 3441 if (UVM_ET_ISNEEDSCOPY(entry) &&
3442 ((entry->max_protection & VM_PROT_WRITE) || 3442 ((entry->max_protection & VM_PROT_WRITE) ||
3443 (entry->object.uvm_obj == NULL))) { 3443 (entry->object.uvm_obj == NULL))) {
3444 amap_copy(map, entry, 0, start, end); 3444 amap_copy(map, entry, 0, start, end);
3445 /* XXXCDC: wait OK? */ 3445 /* XXXCDC: wait OK? */
3446 } 3446 }
3447 } 3447 }
3448 } 3448 }
3449 UVM_MAP_CLIP_START(map, entry, start); 3449 UVM_MAP_CLIP_START(map, entry, start);
3450 UVM_MAP_CLIP_END(map, entry, end); 3450 UVM_MAP_CLIP_END(map, entry, end);
3451 entry->wired_count++; 3451 entry->wired_count++;
3452 3452
3453 /* 3453 /*
3454 * Check for holes 3454 * Check for holes
3455 */ 3455 */
3456 3456
3457 if (entry->protection == VM_PROT_NONE || 3457 if (entry->protection == VM_PROT_NONE ||
3458 (entry->end < end && 3458 (entry->end < end &&
3459 (entry->next == &map->header || 3459 (entry->next == &map->header ||
3460 entry->next->start > entry->end))) { 3460 entry->next->start > entry->end))) {
3461 3461
3462 /* 3462 /*
3463 * found one. amap creation actions do not need to 3463 * found one. amap creation actions do not need to
3464 * be undone, but the wired counts need to be restored. 3464 * be undone, but the wired counts need to be restored.
3465 */ 3465 */
3466 3466
3467 while (entry != &map->header && entry->end > start) { 3467 while (entry != &map->header && entry->end > start) {
3468 entry->wired_count--; 3468 entry->wired_count--;
3469 entry = entry->prev; 3469 entry = entry->prev;
3470 } 3470 }
3471 if ((lockflags & UVM_LK_EXIT) == 0) 3471 if ((lockflags & UVM_LK_EXIT) == 0)
3472 vm_map_unlock(map); 3472 vm_map_unlock(map);
3473 UVMHIST_LOG(maphist,"<- done (INVALID WIRE)",0,0,0,0); 3473 UVMHIST_LOG(maphist,"<- done (INVALID WIRE)",0,0,0,0);
3474 return EINVAL; 3474 return EINVAL;
3475 } 3475 }
3476 entry = entry->next; 3476 entry = entry->next;
3477 } 3477 }
3478 3478
3479 /* 3479 /*
3480 * Pass 2. 3480 * Pass 2.
3481 */ 3481 */
3482 3482
3483#ifdef DIAGNOSTIC 3483#ifdef DIAGNOSTIC
3484 timestamp_save = map->timestamp; 3484 timestamp_save = map->timestamp;
3485#endif 3485#endif
3486 vm_map_busy(map); 3486 vm_map_busy(map);
3487 vm_map_unlock(map); 3487 vm_map_unlock(map);
3488 3488
3489 rv = 0; 3489 rv = 0;
3490 entry = start_entry; 3490 entry = start_entry;
3491 while (entry != &map->header && entry->start < end) { 3491 while (entry != &map->header && entry->start < end) {
3492 if (entry->wired_count == 1) { 3492 if (entry->wired_count == 1) {
3493 rv = uvm_fault_wire(map, entry->start, entry->end, 3493 rv = uvm_fault_wire(map, entry->start, entry->end,
3494 entry->max_protection, 1); 3494 entry->max_protection, 1);
3495 if (rv) { 3495 if (rv) {
3496 3496
3497 /* 3497 /*
3498 * wiring failed. break out of the loop. 3498 * wiring failed. break out of the loop.
3499 * we'll clean up the map below, once we 3499 * we'll clean up the map below, once we
3500 * have a write lock again. 3500 * have a write lock again.
3501 */ 3501 */
3502 3502
3503 break; 3503 break;
3504 } 3504 }
3505 } 3505 }
3506 entry = entry->next; 3506 entry = entry->next;
3507 } 3507 }
3508 3508
3509 if (rv) { /* failed? */ 3509 if (rv) { /* failed? */
3510 3510
3511 /* 3511 /*
3512 * Get back to an exclusive (write) lock. 3512 * Get back to an exclusive (write) lock.
3513 */ 3513 */
3514 3514
3515 vm_map_lock(map); 3515 vm_map_lock(map);
3516 vm_map_unbusy(map); 3516 vm_map_unbusy(map);
3517 3517
3518#ifdef DIAGNOSTIC 3518#ifdef DIAGNOSTIC
3519 if (timestamp_save + 1 != map->timestamp) 3519 if (timestamp_save + 1 != map->timestamp)
3520 panic("uvm_map_pageable: stale map"); 3520 panic("uvm_map_pageable: stale map");
3521#endif 3521#endif
3522 3522
3523 /* 3523 /*
3524 * first drop the wiring count on all the entries 3524 * first drop the wiring count on all the entries
3525 * which haven't actually been wired yet. 3525 * which haven't actually been wired yet.
3526 */ 3526 */
3527 3527
3528 failed_entry = entry; 3528 failed_entry = entry;
3529 while (entry != &map->header && entry->start < end) { 3529 while (entry != &map->header && entry->start < end) {
3530 entry->wired_count--; 3530 entry->wired_count--;
3531 entry = entry->next; 3531 entry = entry->next;
3532 } 3532 }
3533 3533
3534 /* 3534 /*
3535 * now, unwire all the entries that were successfully 3535 * now, unwire all the entries that were successfully
3536 * wired above. 3536 * wired above.
3537 */ 3537 */
3538 3538
3539 entry = start_entry; 3539 entry = start_entry;
3540 while (entry != failed_entry) { 3540 while (entry != failed_entry) {
3541 entry->wired_count--; 3541 entry->wired_count--;
3542 if (VM_MAPENT_ISWIRED(entry) == 0) 3542 if (VM_MAPENT_ISWIRED(entry) == 0)
3543 uvm_map_entry_unwire(map, entry); 3543 uvm_map_entry_unwire(map, entry);
3544 entry = entry->next; 3544 entry = entry->next;
3545 } 3545 }
3546 if ((lockflags & UVM_LK_EXIT) == 0) 3546 if ((lockflags & UVM_LK_EXIT) == 0)
3547 vm_map_unlock(map); 3547 vm_map_unlock(map);
3548 UVMHIST_LOG(maphist, "<- done (RV=%jd)", rv,0,0,0); 3548 UVMHIST_LOG(maphist, "<- done (RV=%jd)", rv,0,0,0);
3549 return (rv); 3549 return (rv);
3550 } 3550 }
3551 3551
3552 if ((lockflags & UVM_LK_EXIT) == 0) { 3552 if ((lockflags & UVM_LK_EXIT) == 0) {
3553 vm_map_unbusy(map); 3553 vm_map_unbusy(map);
3554 } else { 3554 } else {
3555 3555
3556 /* 3556 /*
3557 * Get back to an exclusive (write) lock. 3557 * Get back to an exclusive (write) lock.
3558 */ 3558 */
3559 3559
3560 vm_map_lock(map); 3560 vm_map_lock(map);
3561 vm_map_unbusy(map); 3561 vm_map_unbusy(map);
3562 } 3562 }
3563 3563
3564 UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0); 3564 UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0);
3565 return 0; 3565 return 0;
3566} 3566}
3567 3567
3568/* 3568/*
3569 * uvm_map_pageable_all: special case of uvm_map_pageable - affects 3569 * uvm_map_pageable_all: special case of uvm_map_pageable - affects
3570 * all mapped regions. 3570 * all mapped regions.
3571 * 3571 *
3572 * => map must not be locked. 3572 * => map must not be locked.
3573 * => if no flags are specified, all regions are unwired. 3573 * => if no flags are specified, all regions are unwired.
3574 * => XXXJRT: has some of the same problems as uvm_map_pageable() above. 3574 * => XXXJRT: has some of the same problems as uvm_map_pageable() above.
3575 */ 3575 */
3576 3576
3577int 3577int
3578uvm_map_pageable_all(struct vm_map *map, int flags, vsize_t limit) 3578uvm_map_pageable_all(struct vm_map *map, int flags, vsize_t limit)
3579{ 3579{
3580 struct vm_map_entry *entry, *failed_entry; 3580 struct vm_map_entry *entry, *failed_entry;
3581 vsize_t size; 3581 vsize_t size;
3582 int rv; 3582 int rv;
3583#ifdef DIAGNOSTIC 3583#ifdef DIAGNOSTIC
3584 u_int timestamp_save; 3584 u_int timestamp_save;
3585#endif 3585#endif
3586 UVMHIST_FUNC("uvm_map_pageable_all"); UVMHIST_CALLED(maphist); 3586 UVMHIST_FUNC("uvm_map_pageable_all"); UVMHIST_CALLED(maphist);
3587 UVMHIST_LOG(maphist,"(map=%#jx,flags=%#jx)", (uintptr_t)map, flags, 3587 UVMHIST_LOG(maphist,"(map=%#jx,flags=%#jx)", (uintptr_t)map, flags,
3588 0, 0); 3588 0, 0);
3589 3589
3590 KASSERT(map->flags & VM_MAP_PAGEABLE); 3590 KASSERT(map->flags & VM_MAP_PAGEABLE);
3591 3591
3592 vm_map_lock(map); 3592 vm_map_lock(map);
3593 3593
3594 /* 3594 /*
3595 * handle wiring and unwiring separately. 3595 * handle wiring and unwiring separately.
3596 */ 3596 */
3597 3597
3598 if (flags == 0) { /* unwire */ 3598 if (flags == 0) { /* unwire */
3599 3599
3600 /* 3600 /*
3601 * POSIX 1003.1b -- munlockall unlocks all regions, 3601 * POSIX 1003.1b -- munlockall unlocks all regions,
3602 * regardless of how many times mlockall has been called. 3602 * regardless of how many times mlockall has been called.
3603 */ 3603 */
3604 3604
3605 for (entry = map->header.next; entry != &map->header; 3605 for (entry = map->header.next; entry != &map->header;
3606 entry = entry->next) { 3606 entry = entry->next) {
3607 if (VM_MAPENT_ISWIRED(entry)) 3607 if (VM_MAPENT_ISWIRED(entry))
3608 uvm_map_entry_unwire(map, entry); 3608 uvm_map_entry_unwire(map, entry);
3609 } 3609 }
3610 map->flags &= ~VM_MAP_WIREFUTURE; 3610 map->flags &= ~VM_MAP_WIREFUTURE;
3611 vm_map_unlock(map); 3611 vm_map_unlock(map);
3612 UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0); 3612 UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0);
3613 return 0; 3613 return 0;
3614 } 3614 }
3615 3615
3616 if (flags & MCL_FUTURE) { 3616 if (flags & MCL_FUTURE) {
3617 3617
3618 /* 3618 /*
3619 * must wire all future mappings; remember this. 3619 * must wire all future mappings; remember this.
3620 */ 3620 */
3621 3621
3622 map->flags |= VM_MAP_WIREFUTURE; 3622 map->flags |= VM_MAP_WIREFUTURE;
3623 } 3623 }
3624 3624
3625 if ((flags & MCL_CURRENT) == 0) { 3625 if ((flags & MCL_CURRENT) == 0) {
3626 3626
3627 /* 3627 /*
3628 * no more work to do! 3628 * no more work to do!
3629 */ 3629 */
3630 3630
3631 UVMHIST_LOG(maphist,"<- done (OK no wire)",0,0,0,0); 3631 UVMHIST_LOG(maphist,"<- done (OK no wire)",0,0,0,0);
3632 vm_map_unlock(map); 3632 vm_map_unlock(map);
3633 return 0; 3633 return 0;
3634 } 3634 }
3635 3635
3636 /* 3636 /*
3637 * wire case: in three passes [XXXCDC: ugly block of code here] 3637 * wire case: in three passes [XXXCDC: ugly block of code here]
3638 * 3638 *
3639 * 1: holding the write lock, count all pages mapped by non-wired 3639 * 1: holding the write lock, count all pages mapped by non-wired
3640 * entries. if this would cause us to go over our limit, we fail. 3640 * entries. if this would cause us to go over our limit, we fail.
3641 * 3641 *
3642 * 2: still holding the write lock, we create any anonymous maps that 3642 * 2: still holding the write lock, we create any anonymous maps that
3643 * need to be created. then we increment its wiring count. 3643 * need to be created. then we increment its wiring count.
3644 * 3644 *
3645 * 3: we downgrade to a read lock, and call uvm_fault_wire to fault 3645 * 3: we downgrade to a read lock, and call uvm_fault_wire to fault
3646 * in the pages for any newly wired area (wired_count == 1). 3646 * in the pages for any newly wired area (wired_count == 1).
3647 * 3647 *
3648 * downgrading to a read lock for uvm_fault_wire avoids a possible 3648 * downgrading to a read lock for uvm_fault_wire avoids a possible
3649 * deadlock with another thread that may have faulted on one of 3649 * deadlock with another thread that may have faulted on one of
3650 * the pages to be wired (it would mark the page busy, blocking 3650 * the pages to be wired (it would mark the page busy, blocking
3651 * us, then in turn block on the map lock that we hold). because 3651 * us, then in turn block on the map lock that we hold). because
3652 * of problems in the recursive lock package, we cannot upgrade 3652 * of problems in the recursive lock package, we cannot upgrade
3653 * to a write lock in vm_map_lookup. thus, any actions that 3653 * to a write lock in vm_map_lookup. thus, any actions that
3654 * require the write lock must be done beforehand. because we 3654 * require the write lock must be done beforehand. because we
3655 * keep the read lock on the map, the copy-on-write status of the 3655 * keep the read lock on the map, the copy-on-write status of the
3656 * entries we modify here cannot change. 3656 * entries we modify here cannot change.
3657 */ 3657 */
3658 3658
3659 for (size = 0, entry = map->header.next; entry != &map->header; 3659 for (size = 0, entry = map->header.next; entry != &map->header;
3660 entry = entry->next) { 3660 entry = entry->next) {
3661 if (entry->protection != VM_PROT_NONE && 3661 if (entry->protection != VM_PROT_NONE &&
3662 VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */ 3662 VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
3663 size += entry->end - entry->start; 3663 size += entry->end - entry->start;
3664 } 3664 }
3665 } 3665 }
3666 3666
3667 if (atop(size) + uvmexp.wired > uvmexp.wiredmax) { 3667 if (atop(size) + uvmexp.wired > uvmexp.wiredmax) {
3668 vm_map_unlock(map); 3668 vm_map_unlock(map);
3669 return ENOMEM; 3669 return ENOMEM;
3670 } 3670 }
3671 3671
3672 if (limit != 0 && 3672 if (limit != 0 &&
3673 (size + ptoa(pmap_wired_count(vm_map_pmap(map))) > limit)) { 3673 (size + ptoa(pmap_wired_count(vm_map_pmap(map))) > limit)) {
3674 vm_map_unlock(map); 3674 vm_map_unlock(map);
3675 return ENOMEM; 3675 return ENOMEM;
3676 } 3676 }
3677 3677
3678 /* 3678 /*
3679 * Pass 2. 3679 * Pass 2.
3680 */ 3680 */
3681 3681
3682 for (entry = map->header.next; entry != &map->header; 3682 for (entry = map->header.next; entry != &map->header;
3683 entry = entry->next) { 3683 entry = entry->next) {
3684 if (entry->protection == VM_PROT_NONE) 3684 if (entry->protection == VM_PROT_NONE)
3685 continue; 3685 continue;
3686 if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */ 3686 if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
3687 3687
3688 /* 3688 /*
3689 * perform actions of vm_map_lookup that need the 3689 * perform actions of vm_map_lookup that need the
3690 * write lock on the map: create an anonymous map 3690 * write lock on the map: create an anonymous map
3691 * for a copy-on-write region, or an anonymous map 3691 * for a copy-on-write region, or an anonymous map
3692 * for a zero-fill region. (XXXCDC: submap case 3692 * for a zero-fill region. (XXXCDC: submap case
3693 * ok?) 3693 * ok?)
3694 */ 3694 */
3695 3695
3696 if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */ 3696 if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */
3697 if (UVM_ET_ISNEEDSCOPY(entry) && 3697 if (UVM_ET_ISNEEDSCOPY(entry) &&
3698 ((entry->max_protection & VM_PROT_WRITE) || 3698 ((entry->max_protection & VM_PROT_WRITE) ||
3699 (entry->object.uvm_obj == NULL))) { 3699 (entry->object.uvm_obj == NULL))) {
3700 amap_copy(map, entry, 0, entry->start, 3700 amap_copy(map, entry, 0, entry->start,
3701 entry->end); 3701 entry->end);
3702 /* XXXCDC: wait OK? */ 3702 /* XXXCDC: wait OK? */
3703 } 3703 }
3704 } 3704 }
3705 } 3705 }
3706 entry->wired_count++; 3706 entry->wired_count++;
3707 } 3707 }
3708 3708
3709 /* 3709 /*
3710 * Pass 3. 3710 * Pass 3.
3711 */ 3711 */
3712 3712
3713#ifdef DIAGNOSTIC 3713#ifdef DIAGNOSTIC
3714 timestamp_save = map->timestamp; 3714 timestamp_save = map->timestamp;
3715#endif 3715#endif
3716 vm_map_busy(map); 3716 vm_map_busy(map);
3717 vm_map_unlock(map); 3717 vm_map_unlock(map);
3718 3718
3719 rv = 0; 3719 rv = 0;
3720 for (entry = map->header.next; entry != &map->header; 3720 for (entry = map->header.next; entry != &map->header;
3721 entry = entry->next) { 3721 entry = entry->next) {
3722 if (entry->wired_count == 1) { 3722 if (entry->wired_count == 1) {
3723 rv = uvm_fault_wire(map, entry->start, entry->end, 3723 rv = uvm_fault_wire(map, entry->start, entry->end,
3724 entry->max_protection, 1); 3724 entry->max_protection, 1);
3725 if (rv) { 3725 if (rv) {
3726 3726
3727 /* 3727 /*
3728 * wiring failed. break out of the loop. 3728 * wiring failed. break out of the loop.
3729 * we'll clean up the map below, once we 3729 * we'll clean up the map below, once we
3730 * have a write lock again. 3730 * have a write lock again.
3731 */ 3731 */
3732 3732
3733 break; 3733 break;
3734 } 3734 }
3735 } 3735 }
3736 } 3736 }
3737 3737
3738 if (rv) { 3738 if (rv) {
3739 3739
3740 /* 3740 /*
3741 * Get back an exclusive (write) lock. 3741 * Get back an exclusive (write) lock.
3742 */ 3742 */
3743 3743
3744 vm_map_lock(map); 3744 vm_map_lock(map);
3745 vm_map_unbusy(map); 3745 vm_map_unbusy(map);
3746 3746
3747#ifdef DIAGNOSTIC 3747#ifdef DIAGNOSTIC
3748 if (timestamp_save + 1 != map->timestamp) 3748 if (timestamp_save + 1 != map->timestamp)
3749 panic("uvm_map_pageable_all: stale map"); 3749 panic("uvm_map_pageable_all: stale map");
3750#endif 3750#endif
3751 3751
3752 /* 3752 /*
3753 * first drop the wiring count on all the entries 3753 * first drop the wiring count on all the entries
3754 * which haven't actually been wired yet. 3754 * which haven't actually been wired yet.
3755 * 3755 *
3756 * Skip VM_PROT_NONE entries like we did above. 3756 * Skip VM_PROT_NONE entries like we did above.
3757 */ 3757 */
3758 3758
3759 failed_entry = entry; 3759 failed_entry = entry;
3760 for (/* nothing */; entry != &map->header; 3760 for (/* nothing */; entry != &map->header;
3761 entry = entry->next) { 3761 entry = entry->next) {
3762 if (entry->protection == VM_PROT_NONE) 3762 if (entry->protection == VM_PROT_NONE)
3763 continue; 3763 continue;
3764 entry->wired_count--; 3764 entry->wired_count--;
3765 } 3765 }
3766 3766
3767 /* 3767 /*
3768 * now, unwire all the entries that were successfully 3768 * now, unwire all the entries that were successfully
3769 * wired above. 3769 * wired above.
3770 * 3770 *
3771 * Skip VM_PROT_NONE entries like we did above. 3771 * Skip VM_PROT_NONE entries like we did above.
3772 */ 3772 */
3773 3773
3774 for (entry = map->header.next; entry != failed_entry; 3774 for (entry = map->header.next; entry != failed_entry;
3775 entry = entry->next) { 3775 entry = entry->next) {
3776 if (entry->protection == VM_PROT_NONE) 3776 if (entry->protection == VM_PROT_NONE)
3777 continue; 3777 continue;
3778 entry->wired_count--; 3778 entry->wired_count--;
3779 if (VM_MAPENT_ISWIRED(entry)) 3779 if (VM_MAPENT_ISWIRED(entry))
3780 uvm_map_entry_unwire(map, entry); 3780 uvm_map_entry_unwire(map, entry);
3781 } 3781 }
3782 vm_map_unlock(map); 3782 vm_map_unlock(map);
3783 UVMHIST_LOG(maphist,"<- done (RV=%jd)", rv,0,0,0); 3783 UVMHIST_LOG(maphist,"<- done (RV=%jd)", rv,0,0,0);
3784 return (rv); 3784 return (rv);
3785 } 3785 }
3786 3786
3787 vm_map_unbusy(map); 3787 vm_map_unbusy(map);
3788 3788
3789 UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0); 3789 UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0);
3790 return 0; 3790 return 0;
3791} 3791}
3792 3792
3793/* 3793/*
3794 * uvm_map_clean: clean out a map range 3794 * uvm_map_clean: clean out a map range
3795 * 3795 *
3796 * => valid flags: 3796 * => valid flags:
3797 * if (flags & PGO_CLEANIT): dirty pages are cleaned first 3797 * if (flags & PGO_CLEANIT): dirty pages are cleaned first
3798 * if (flags & PGO_SYNCIO): dirty pages are written synchronously 3798 * if (flags & PGO_SYNCIO): dirty pages are written synchronously
3799 * if (flags & PGO_DEACTIVATE): any cached pages are deactivated after clean 3799 * if (flags & PGO_DEACTIVATE): any cached pages are deactivated after clean
3800 * if (flags & PGO_FREE): any cached pages are freed after clean 3800 * if (flags & PGO_FREE): any cached pages are freed after clean
3801 * => returns an error if any part of the specified range isn't mapped 3801 * => returns an error if any part of the specified range isn't mapped
3802 * => never a need to flush amap layer since the anonymous memory has 3802 * => never a need to flush amap layer since the anonymous memory has
3803 * no permanent home, but may deactivate pages there 3803 * no permanent home, but may deactivate pages there
3804 * => called from sys_msync() and sys_madvise() 3804 * => called from sys_msync() and sys_madvise()
3805 * => caller must not write-lock map (read OK). 3805 * => caller must not write-lock map (read OK).
3806 * => we may sleep while cleaning if SYNCIO [with map read-locked] 3806 * => we may sleep while cleaning if SYNCIO [with map read-locked]
3807 */ 3807 */
3808 3808
3809int 3809int
3810uvm_map_clean(struct vm_map *map, vaddr_t start, vaddr_t end, int flags) 3810uvm_map_clean(struct vm_map *map, vaddr_t start, vaddr_t end, int flags)
3811{ 3811{
3812 struct vm_map_entry *current, *entry; 3812 struct vm_map_entry *current, *entry;
3813 struct uvm_object *uobj; 3813 struct uvm_object *uobj;
3814 struct vm_amap *amap; 3814 struct vm_amap *amap;
3815 struct vm_anon *anon; 3815 struct vm_anon *anon;
3816 struct vm_page *pg; 3816 struct vm_page *pg;
3817 vaddr_t offset; 3817 vaddr_t offset;
3818 vsize_t size; 3818 vsize_t size;
3819 voff_t uoff; 3819 voff_t uoff;
3820 int error, refs; 3820 int error, refs;
3821 UVMHIST_FUNC("uvm_map_clean"); UVMHIST_CALLED(maphist); 3821 UVMHIST_FUNC("uvm_map_clean"); UVMHIST_CALLED(maphist);
3822 3822
3823 UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,flags=%#jx)", 3823 UVMHIST_LOG(maphist,"(map=%#jx,start=%#jx,end=%#jx,flags=%#jx)",
3824 (uintptr_t)map, start, end, flags); 3824 (uintptr_t)map, start, end, flags);
3825 KASSERT((flags & (PGO_FREE|PGO_DEACTIVATE)) != 3825 KASSERT((flags & (PGO_FREE|PGO_DEACTIVATE)) !=
3826 (PGO_FREE|PGO_DEACTIVATE)); 3826 (PGO_FREE|PGO_DEACTIVATE));
3827 3827
3828 vm_map_lock_read(map); 3828 vm_map_lock_read(map);
3829 VM_MAP_RANGE_CHECK(map, start, end); 3829 VM_MAP_RANGE_CHECK(map, start, end);
3830 if (uvm_map_lookup_entry(map, start, &entry) == false) { 3830 if (uvm_map_lookup_entry(map, start, &entry) == false) {
3831 vm_map_unlock_read(map); 3831 vm_map_unlock_read(map);
3832 return EFAULT; 3832 return EFAULT;
3833 } 3833 }
3834 3834
3835 /* 3835 /*
3836 * Make a first pass to check for holes and wiring problems. 3836 * Make a first pass to check for holes and wiring problems.
3837 */ 3837 */
3838 3838
3839 for (current = entry; current->start < end; current = current->next) { 3839 for (current = entry; current->start < end; current = current->next) {
3840 if (UVM_ET_ISSUBMAP(current)) { 3840 if (UVM_ET_ISSUBMAP(current)) {
3841 vm_map_unlock_read(map); 3841 vm_map_unlock_read(map);
3842 return EINVAL; 3842 return EINVAL;
3843 } 3843 }
3844 if ((flags & PGO_FREE) != 0 && VM_MAPENT_ISWIRED(entry)) { 3844 if ((flags & PGO_FREE) != 0 && VM_MAPENT_ISWIRED(entry)) {
3845 vm_map_unlock_read(map); 3845 vm_map_unlock_read(map);
3846 return EBUSY; 3846 return EBUSY;
3847 } 3847 }
3848 if (end <= current->end) { 3848 if (end <= current->end) {
3849 break; 3849 break;
3850 } 3850 }
3851 if (current->end != current->next->start) { 3851 if (current->end != current->next->start) {
3852 vm_map_unlock_read(map); 3852 vm_map_unlock_read(map);
3853 return EFAULT; 3853 return EFAULT;
3854 } 3854 }
3855 } 3855 }
3856 3856
3857 error = 0; 3857 error = 0;
3858 for (current = entry; start < end; current = current->next) { 3858 for (current = entry; start < end; current = current->next) {
3859 amap = current->aref.ar_amap; /* upper layer */ 3859 amap = current->aref.ar_amap; /* upper layer */
3860 uobj = current->object.uvm_obj; /* lower layer */ 3860 uobj = current->object.uvm_obj; /* lower layer */
3861 KASSERT(start >= current->start); 3861 KASSERT(start >= current->start);
3862 3862
3863 /* 3863 /*
3864 * No amap cleaning necessary if: 3864 * No amap cleaning necessary if:
3865 * 3865 *
3866 * (1) There's no amap. 3866 * (1) There's no amap.
3867 * 3867 *
3868 * (2) We're not deactivating or freeing pages. 3868 * (2) We're not deactivating or freeing pages.
3869 */ 3869 */
3870 3870
3871 if (amap == NULL || (flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) 3871 if (amap == NULL || (flags & (PGO_DEACTIVATE|PGO_FREE)) == 0)
3872 goto flush_object; 3872 goto flush_object;
3873 3873
3874 offset = start - current->start; 3874 offset = start - current->start;
3875 size = MIN(end, current->end) - start; 3875 size = MIN(end, current->end) - start;
3876 3876
3877 amap_lock(amap, RW_WRITER); 3877 amap_lock(amap, RW_WRITER);
3878 for ( ; size != 0; size -= PAGE_SIZE, offset += PAGE_SIZE) { 3878 for ( ; size != 0; size -= PAGE_SIZE, offset += PAGE_SIZE) {
3879 anon = amap_lookup(&current->aref, offset); 3879 anon = amap_lookup(&current->aref, offset);
3880 if (anon == NULL) 3880 if (anon == NULL)
3881 continue; 3881 continue;
3882 3882
3883 KASSERT(anon->an_lock == amap->am_lock); 3883 KASSERT(anon->an_lock == amap->am_lock);
3884 pg = anon->an_page; 3884 pg = anon->an_page;
3885 if (pg == NULL) { 3885 if (pg == NULL) {
3886 continue; 3886 continue;
3887 } 3887 }
3888 if (pg->flags & PG_BUSY) { 3888 if (pg->flags & PG_BUSY) {
3889 continue; 3889 continue;
3890 } 3890 }
3891 3891
3892 switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) { 3892 switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
3893 3893
3894 /* 3894 /*
3895 * In these first 3 cases, we just deactivate the page. 3895 * In these first 3 cases, we just deactivate the page.
3896 */ 3896 */
3897 3897
3898 case PGO_CLEANIT|PGO_FREE: 3898 case PGO_CLEANIT|PGO_FREE:
3899 case PGO_CLEANIT|PGO_DEACTIVATE: 3899 case PGO_CLEANIT|PGO_DEACTIVATE:
3900 case PGO_DEACTIVATE: 3900 case PGO_DEACTIVATE:
3901 deactivate_it: 3901 deactivate_it:
3902 /* 3902 /*
3903 * skip the page if it's loaned or wired, 3903 * skip the page if it's loaned or wired,
3904 * since it shouldn't be on a paging queue 3904 * since it shouldn't be on a paging queue
3905 * at all in these cases. 3905 * at all in these cases.
3906 */ 3906 */
3907 3907
3908 if (pg->loan_count != 0 || 3908 if (pg->loan_count != 0 ||
3909 pg->wire_count != 0) { 3909 pg->wire_count != 0) {
3910 continue; 3910 continue;
3911 } 3911 }
3912 KASSERT(pg->uanon == anon); 3912 KASSERT(pg->uanon == anon);
3913 uvm_pagelock(pg); 3913 uvm_pagelock(pg);
3914 uvm_pagedeactivate(pg); 3914 uvm_pagedeactivate(pg);
3915 uvm_pageunlock(pg); 3915 uvm_pageunlock(pg);
3916 continue; 3916 continue;
3917 3917
3918 case PGO_FREE: 3918 case PGO_FREE:
3919 3919
3920 /* 3920 /*
3921 * If there are multiple references to 3921 * If there are multiple references to
3922 * the amap, just deactivate the page. 3922 * the amap, just deactivate the page.
3923 */ 3923 */
3924 3924
3925 if (amap_refs(amap) > 1) 3925 if (amap_refs(amap) > 1)
3926 goto deactivate_it; 3926 goto deactivate_it;
3927 3927
3928 /* skip the page if it's wired */ 3928 /* skip the page if it's wired */
3929 if (pg->wire_count != 0) { 3929 if (pg->wire_count != 0) {
3930 continue; 3930 continue;
3931 } 3931 }
3932 amap_unadd(&current->aref, offset); 3932 amap_unadd(&current->aref, offset);
3933 refs = --anon->an_ref; 3933 refs = --anon->an_ref;
3934 if (refs == 0) { 3934 if (refs == 0) {
3935 uvm_anfree(anon); 3935 uvm_anfree(anon);
3936 } 3936 }
3937 continue; 3937 continue;
3938 } 3938 }
3939 } 3939 }
3940 amap_unlock(amap); 3940 amap_unlock(amap);
3941 3941
3942 flush_object: 3942 flush_object:
3943 /* 3943 /*
3944 * flush pages if we've got a valid backing object. 3944 * flush pages if we've got a valid backing object.
3945 * note that we must always clean object pages before 3945 * note that we must always clean object pages before
3946 * freeing them since otherwise we could reveal stale 3946 * freeing them since otherwise we could reveal stale
3947 * data from files. 3947 * data from files.
3948 */ 3948 */
3949 3949
3950 uoff = current->offset + (start - current->start); 3950 uoff = current->offset + (start - current->start);
3951 size = MIN(end, current->end) - start; 3951 size = MIN(end, current->end) - start;
3952 if (uobj != NULL) { 3952 if (uobj != NULL) {
3953 rw_enter(uobj->vmobjlock, RW_WRITER); 3953 rw_enter(uobj->vmobjlock, RW_WRITER);
3954 if (uobj->pgops->pgo_put != NULL) 3954 if (uobj->pgops->pgo_put != NULL)
3955 error = (uobj->pgops->pgo_put)(uobj, uoff, 3955 error = (uobj->pgops->pgo_put)(uobj, uoff,
3956 uoff + size, flags | PGO_CLEANIT); 3956 uoff + size, flags | PGO_CLEANIT);
3957 else 3957 else
3958 error = 0; 3958 error = 0;
3959 } 3959 }
3960 start += size; 3960 start += size;
3961 } 3961 }
3962 vm_map_unlock_read(map); 3962 vm_map_unlock_read(map);
3963 return (error); 3963 return (error);
3964} 3964}
3965 3965
3966 3966
3967/* 3967/*
3968 * uvm_map_checkprot: check protection in map 3968 * uvm_map_checkprot: check protection in map
3969 * 3969 *
3970 * => must allow specified protection in a fully allocated region. 3970 * => must allow specified protection in a fully allocated region.
3971 * => map must be read or write locked by caller. 3971 * => map must be read or write locked by caller.
3972 */ 3972 */
3973 3973
3974bool 3974bool
3975uvm_map_checkprot(struct vm_map *map, vaddr_t start, vaddr_t end, 3975uvm_map_checkprot(struct vm_map *map, vaddr_t start, vaddr_t end,
3976 vm_prot_t protection) 3976 vm_prot_t protection)
3977{ 3977{
3978 struct vm_map_entry *entry; 3978 struct vm_map_entry *entry;
3979 struct vm_map_entry *tmp_entry; 3979 struct vm_map_entry *tmp_entry;
3980 3980
3981 if (!uvm_map_lookup_entry(map, start, &tmp_entry)) { 3981 if (!uvm_map_lookup_entry(map, start, &tmp_entry)) {
3982 return (false); 3982 return (false);
3983 } 3983 }
3984 entry = tmp_entry; 3984 entry = tmp_entry;
3985 while (start < end) { 3985 while (start < end) {
3986 if (entry == &map->header) { 3986 if (entry == &map->header) {
3987 return (false); 3987 return (false);
3988 } 3988 }
3989 3989
3990 /* 3990 /*
3991 * no holes allowed 3991 * no holes allowed
3992 */ 3992 */
3993 3993
3994 if (start < entry->start) { 3994 if (start < entry->start) {
3995 return (false); 3995 return (false);
3996 } 3996 }
3997 3997
3998 /* 3998 /*
3999 * check protection associated with entry 3999 * check protection associated with entry
4000 */ 4000 */
4001 4001
4002 if ((entry->protection & protection) != protection) { 4002 if ((entry->protection & protection) != protection) {
4003 return (false); 4003 return (false);
4004 } 4004 }
4005 start = entry->end; 4005 start = entry->end;
4006 entry = entry->next; 4006 entry = entry->next;
4007 } 4007 }
4008 return (true); 4008 return (true);
4009} 4009}
4010 4010
4011/* 4011/*
4012 * uvmspace_alloc: allocate a vmspace structure. 4012 * uvmspace_alloc: allocate a vmspace structure.
4013 * 4013 *
4014 * - structure includes vm_map and pmap 4014 * - structure includes vm_map and pmap
4015 * - XXX: no locking on this structure 4015 * - XXX: no locking on this structure
4016 * - refcnt set to 1, rest must be init'd by caller 4016 * - refcnt set to 1, rest must be init'd by caller
4017 */ 4017 */
4018struct vmspace * 4018struct vmspace *
4019uvmspace_alloc(vaddr_t vmin, vaddr_t vmax, bool topdown) 4019uvmspace_alloc(vaddr_t vmin, vaddr_t vmax, bool topdown)
4020{ 4020{
4021 struct vmspace *vm; 4021 struct vmspace *vm;
4022 UVMHIST_FUNC("uvmspace_alloc"); UVMHIST_CALLED(maphist); 4022 UVMHIST_FUNC("uvmspace_alloc"); UVMHIST_CALLED(maphist);
4023 4023
4024 vm = pool_cache_get(&uvm_vmspace_cache, PR_WAITOK); 4024 vm = pool_cache_get(&uvm_vmspace_cache, PR_WAITOK);
4025 uvmspace_init(vm, NULL, vmin, vmax, topdown); 4025 uvmspace_init(vm, NULL, vmin, vmax, topdown);
4026 UVMHIST_LOG(maphist,"<- done (vm=%#jx)", (uintptr_t)vm, 0, 0, 0); 4026 UVMHIST_LOG(maphist,"<- done (vm=%#jx)", (uintptr_t)vm, 0, 0, 0);
4027 return (vm); 4027 return (vm);
4028} 4028}
4029 4029
4030/* 4030/*
4031 * uvmspace_init: initialize a vmspace structure. 4031 * uvmspace_init: initialize a vmspace structure.
4032 * 4032 *
4033 * - XXX: no locking on this structure 4033 * - XXX: no locking on this structure
4034 * - refcnt set to 1, rest must be init'd by caller 4034 * - refcnt set to 1, rest must be init'd by caller
4035 */ 4035 */
4036void 4036void
4037uvmspace_init(struct vmspace *vm, struct pmap *pmap, vaddr_t vmin, 4037uvmspace_init(struct vmspace *vm, struct pmap *pmap, vaddr_t vmin,
4038 vaddr_t vmax, bool topdown) 4038 vaddr_t vmax, bool topdown)
4039{ 4039{
4040 UVMHIST_FUNC("uvmspace_init"); UVMHIST_CALLED(maphist); 4040 UVMHIST_FUNC("uvmspace_init"); UVMHIST_CALLED(maphist);
4041 4041
4042 UVMHIST_LOG(maphist, "(vm=%#jx, pmap=%#jx, vmin=%#jx, vmax=%#jx", 4042 UVMHIST_LOG(maphist, "(vm=%#jx, pmap=%#jx, vmin=%#jx, vmax=%#jx",
4043 (uintptr_t)vm, (uintptr_t)pmap, vmin, vmax); 4043 (uintptr_t)vm, (uintptr_t)pmap, vmin, vmax);
4044 UVMHIST_LOG(maphist, " topdown=%ju)", topdown, 0, 0, 0); 4044 UVMHIST_LOG(maphist, " topdown=%ju)", topdown, 0, 0, 0);
4045 4045
4046 memset(vm, 0, sizeof(*vm)); 4046 memset(vm, 0, sizeof(*vm));
4047 uvm_map_setup(&vm->vm_map, vmin, vmax, VM_MAP_PAGEABLE 4047 uvm_map_setup(&vm->vm_map, vmin, vmax, VM_MAP_PAGEABLE
4048 | (topdown ? VM_MAP_TOPDOWN : 0) 4048 | (topdown ? VM_MAP_TOPDOWN : 0)
4049 ); 4049 );
4050 if (pmap) 4050 if (pmap)
4051 pmap_reference(pmap); 4051 pmap_reference(pmap);
4052 else 4052 else
4053 pmap = pmap_create(); 4053 pmap = pmap_create();
4054 vm->vm_map.pmap = pmap; 4054 vm->vm_map.pmap = pmap;
4055 vm->vm_refcnt = 1; 4055 vm->vm_refcnt = 1;
4056 UVMHIST_LOG(maphist,"<- done",0,0,0,0); 4056 UVMHIST_LOG(maphist,"<- done",0,0,0,0);
4057} 4057}
4058 4058
4059/* 4059/*
4060 * uvmspace_share: share a vmspace between two processes 4060 * uvmspace_share: share a vmspace between two processes
4061 * 4061 *
4062 * - used for vfork, threads(?) 4062 * - used for vfork, threads(?)
4063 */ 4063 */
4064 4064
4065void 4065void
4066uvmspace_share(struct proc *p1, struct proc *p2) 4066uvmspace_share(struct proc *p1, struct proc *p2)
4067{ 4067{
4068 4068
4069 uvmspace_addref(p1->p_vmspace); 4069 uvmspace_addref(p1->p_vmspace);
4070 p2->p_vmspace = p1->p_vmspace; 4070 p2->p_vmspace = p1->p_vmspace;
4071} 4071}
4072 4072
4073#if 0 4073#if 0
4074 4074
4075/* 4075/*
4076 * uvmspace_unshare: ensure that process "p" has its own, unshared, vmspace 4076 * uvmspace_unshare: ensure that process "p" has its own, unshared, vmspace
4077 * 4077 *
4078 * - XXX: no locking on vmspace 4078 * - XXX: no locking on vmspace
4079 */ 4079 */
4080 4080
4081void 4081void
4082uvmspace_unshare(struct lwp *l) 4082uvmspace_unshare(struct lwp *l)
4083{ 4083{
4084 struct proc *p = l->l_proc; 4084 struct proc *p = l->l_proc;
4085 struct vmspace *nvm, *ovm = p->p_vmspace; 4085 struct vmspace *nvm, *ovm = p->p_vmspace;
4086 4086
4087 if (ovm->vm_refcnt == 1) 4087 if (ovm->vm_refcnt == 1)
4088 /* nothing to do: vmspace isn't shared in the first place */ 4088 /* nothing to do: vmspace isn't shared in the first place */
4089 return; 4089 return;
4090 4090
4091 /* make a new vmspace, still holding old one */ 4091 /* make a new vmspace, still holding old one */
4092 nvm = uvmspace_fork(ovm); 4092 nvm = uvmspace_fork(ovm);
4093 4093
4094 kpreempt_disable(); 4094 kpreempt_disable();
4095 pmap_deactivate(l); /* unbind old vmspace */ 4095 pmap_deactivate(l); /* unbind old vmspace */
4096 p->p_vmspace = nvm; 4096 p->p_vmspace = nvm;
4097 pmap_activate(l); /* switch to new vmspace */ 4097 pmap_activate(l); /* switch to new vmspace */
4098 kpreempt_enable(); 4098 kpreempt_enable();
4099 4099
4100 uvmspace_free(ovm); /* drop reference to old vmspace */ 4100 uvmspace_free(ovm); /* drop reference to old vmspace */
4101} 4101}
4102 4102
4103#endif 4103#endif
4104 4104
4105 4105
4106/* 4106/*
4107 * uvmspace_spawn: a new process has been spawned and needs a vmspace 4107 * uvmspace_spawn: a new process has been spawned and needs a vmspace
4108 */ 4108 */
4109 4109
4110void 4110void
4111uvmspace_spawn(struct lwp *l, vaddr_t start, vaddr_t end, bool topdown) 4111uvmspace_spawn(struct lwp *l, vaddr_t start, vaddr_t end, bool topdown)
4112{ 4112{
4113 struct proc *p = l->l_proc; 4113 struct proc *p = l->l_proc;
4114 struct vmspace *nvm; 4114 struct vmspace *nvm;
4115 4115
4116#ifdef __HAVE_CPU_VMSPACE_EXEC 4116#ifdef __HAVE_CPU_VMSPACE_EXEC
4117 cpu_vmspace_exec(l, start, end); 4117 cpu_vmspace_exec(l, start, end);
4118#endif 4118#endif
4119 4119
4120 nvm = uvmspace_alloc(start, end, topdown); 4120 nvm = uvmspace_alloc(start, end, topdown);
4121 kpreempt_disable(); 4121 kpreempt_disable();
4122 p->p_vmspace = nvm; 4122 p->p_vmspace = nvm;
4123 pmap_activate(l); 4123 pmap_activate(l);
4124 kpreempt_enable(); 4124 kpreempt_enable();
4125} 4125}
4126 4126
4127/* 4127/*
4128 * uvmspace_exec: the process wants to exec a new program 4128 * uvmspace_exec: the process wants to exec a new program
4129 */ 4129 */
4130 4130
4131void 4131void
4132uvmspace_exec(struct lwp *l, vaddr_t start, vaddr_t end, bool topdown) 4132uvmspace_exec(struct lwp *l, vaddr_t start, vaddr_t end, bool topdown)
4133{ 4133{
4134 struct proc *p = l->l_proc; 4134 struct proc *p = l->l_proc;
4135 struct vmspace *nvm, *ovm = p->p_vmspace; 4135 struct vmspace *nvm, *ovm = p->p_vmspace;
4136 struct vm_map *map; 4136 struct vm_map *map;
4137 int flags; 4137 int flags;
4138 4138
4139 KASSERT(ovm != NULL); 4139 KASSERT(ovm != NULL);
4140#ifdef __HAVE_CPU_VMSPACE_EXEC 4140#ifdef __HAVE_CPU_VMSPACE_EXEC
4141 cpu_vmspace_exec(l, start, end); 4141 cpu_vmspace_exec(l, start, end);
4142#endif 4142#endif
4143 4143
4144 map = &ovm->vm_map; 4144 map = &ovm->vm_map;
4145 /* 4145 /*
4146 * see if more than one process is using this vmspace... 4146 * see if more than one process is using this vmspace...
4147 */ 4147 */
4148 4148
4149 if (ovm->vm_refcnt == 1 4149 if (ovm->vm_refcnt == 1
4150 && topdown == ((ovm->vm_map.flags & VM_MAP_TOPDOWN) != 0)) { 4150 && topdown == ((ovm->vm_map.flags & VM_MAP_TOPDOWN) != 0)) {
4151 4151
4152 /* 4152 /*
4153 * if p is the only process using its vmspace then we can safely 4153 * if p is the only process using its vmspace then we can safely
4154 * recycle that vmspace for the program that is being exec'd. 4154 * recycle that vmspace for the program that is being exec'd.
4155 * But only if TOPDOWN matches the requested value for the new 4155 * But only if TOPDOWN matches the requested value for the new
4156 * vm space! 4156 * vm space!
4157 */ 4157 */
4158 4158
4159 /* 4159 /*
4160 * SYSV SHM semantics require us to kill all segments on an exec 4160 * SYSV SHM semantics require us to kill all segments on an exec
4161 */ 4161 */
4162 if (uvm_shmexit && ovm->vm_shm) 4162 if (uvm_shmexit && ovm->vm_shm)
4163 (*uvm_shmexit)(ovm); 4163 (*uvm_shmexit)(ovm);
4164 4164
4165 /* 4165 /*
4166 * POSIX 1003.1b -- "lock future mappings" is revoked 4166 * POSIX 1003.1b -- "lock future mappings" is revoked
4167 * when a process execs another program image. 4167 * when a process execs another program image.
4168 */ 4168 */
4169 4169
4170 map->flags &= ~VM_MAP_WIREFUTURE; 4170 map->flags &= ~VM_MAP_WIREFUTURE;
4171 4171
4172 /* 4172 /*
4173 * now unmap the old program. 4173 * now unmap the old program.
4174 *  4174 *
4175 * XXX set VM_MAP_DYING for the duration, so pmap_update() 4175 * XXX set VM_MAP_DYING for the duration, so pmap_update()
4176 * is not called until the pmap has been totally cleared out 4176 * is not called until the pmap has been totally cleared out
4177 * after pmap_remove_all(), or it can confuse some pmap 4177 * after pmap_remove_all(), or it can confuse some pmap
4178 * implementations. it would be nice to handle this by 4178 * implementations. it would be nice to handle this by
4179 * deferring the pmap_update() while it is known the address 4179 * deferring the pmap_update() while it is known the address
4180 * space is not visible to any user LWP other than curlwp, 4180 * space is not visible to any user LWP other than curlwp,
4181 * but there isn't an elegant way of inferring that right 4181 * but there isn't an elegant way of inferring that right
4182 * now. 4182 * now.
4183 */ 4183 */
4184 4184
4185 flags = pmap_remove_all(map->pmap) ? UVM_FLAG_VAONLY : 0; 4185 flags = pmap_remove_all(map->pmap) ? UVM_FLAG_VAONLY : 0;
4186 map->flags |= VM_MAP_DYING; 4186 map->flags |= VM_MAP_DYING;
4187 uvm_unmap1(map, vm_map_min(map), vm_map_max(map), flags); 4187 uvm_unmap1(map, vm_map_min(map), vm_map_max(map), flags);
4188 map->flags &= ~VM_MAP_DYING; 4188 map->flags &= ~VM_MAP_DYING;
4189 pmap_update(map->pmap); 4189 pmap_update(map->pmap);
4190 KASSERT(map->header.prev == &map->header); 4190 KASSERT(map->header.prev == &map->header);
4191 KASSERT(map->nentries == 0); 4191 KASSERT(map->nentries == 0);
4192 4192
4193 /* 4193 /*
4194 * resize the map 4194 * resize the map
4195 */ 4195 */
4196 4196
4197 vm_map_setmin(map, start); 4197 vm_map_setmin(map, start);
4198 vm_map_setmax(map, end); 4198 vm_map_setmax(map, end);
4199 } else { 4199 } else {
4200 4200
4201 /* 4201 /*
4202 * p's vmspace is being shared, so we can't reuse it for p since 4202 * p's vmspace is being shared, so we can't reuse it for p since
4203 * it is still being used for others. allocate a new vmspace 4203 * it is still being used for others. allocate a new vmspace
4204 * for p 4204 * for p
4205 */ 4205 */
4206 4206
4207 nvm = uvmspace_alloc(start, end, topdown); 4207 nvm = uvmspace_alloc(start, end, topdown);
4208 4208
4209 /* 4209 /*
4210 * install new vmspace and drop our ref to the old one. 4210 * install new vmspace and drop our ref to the old one.
4211 */ 4211 */
4212 4212
4213 kpreempt_disable(); 4213 kpreempt_disable();
4214 pmap_deactivate(l); 4214 pmap_deactivate(l);
4215 p->p_vmspace = nvm; 4215 p->p_vmspace = nvm;
4216 pmap_activate(l); 4216 pmap_activate(l);
4217 kpreempt_enable(); 4217 kpreempt_enable();
4218 4218
4219 uvmspace_free(ovm); 4219 uvmspace_free(ovm);
4220 } 4220 }
4221} 4221}
4222 4222
4223/* 4223/*
4224 * uvmspace_addref: add a reference to a vmspace. 4224 * uvmspace_addref: add a reference to a vmspace.
4225 */ 4225 */
4226 4226
4227void 4227void
4228uvmspace_addref(struct vmspace *vm) 4228uvmspace_addref(struct vmspace *vm)
4229{ 4229{
4230 4230
4231 KASSERT((vm->vm_map.flags & VM_MAP_DYING) == 0); 4231 KASSERT((vm->vm_map.flags & VM_MAP_DYING) == 0);
4232 KASSERT(vm->vm_refcnt > 0); 4232 KASSERT(vm->vm_refcnt > 0);
4233 atomic_inc_uint(&vm->vm_refcnt); 4233 atomic_inc_uint(&vm->vm_refcnt);
4234} 4234}
4235 4235
4236/* 4236/*
4237 * uvmspace_free: free a vmspace data structure 4237 * uvmspace_free: free a vmspace data structure
4238 */ 4238 */
4239 4239
4240void 4240void
4241uvmspace_free(struct vmspace *vm) 4241uvmspace_free(struct vmspace *vm)
4242{ 4242{
4243 struct vm_map_entry *dead_entries; 4243 struct vm_map_entry *dead_entries;
4244 struct vm_map *map = &vm->vm_map; 4244 struct vm_map *map = &vm->vm_map;
4245 int flags; 4245 int flags;
4246 4246
4247 UVMHIST_FUNC("uvmspace_free"); UVMHIST_CALLED(maphist); 4247 UVMHIST_FUNC("uvmspace_free"); UVMHIST_CALLED(maphist);
4248 4248
4249 UVMHIST_LOG(maphist,"(vm=%#jx) ref=%jd", (uintptr_t)vm, vm->vm_refcnt, 4249 UVMHIST_LOG(maphist,"(vm=%#jx) ref=%jd", (uintptr_t)vm, vm->vm_refcnt,
4250 0, 0); 4250 0, 0);
4251 if (atomic_dec_uint_nv(&vm->vm_refcnt) > 0) 4251 if (atomic_dec_uint_nv(&vm->vm_refcnt) > 0)
4252 return; 4252 return;
4253 4253
4254 /* 4254 /*
4255 * at this point, there should be no other references to the map. 4255 * at this point, there should be no other references to the map.
4256 * delete all of the mappings, then destroy the pmap. 4256 * delete all of the mappings, then destroy the pmap.
4257 */ 4257 */
4258 4258
4259 map->flags |= VM_MAP_DYING; 4259 map->flags |= VM_MAP_DYING;
4260 flags = pmap_remove_all(map->pmap) ? UVM_FLAG_VAONLY : 0; 4260 flags = pmap_remove_all(map->pmap) ? UVM_FLAG_VAONLY : 0;
4261 4261
4262 /* Get rid of any SYSV shared memory segments. */ 4262 /* Get rid of any SYSV shared memory segments. */
4263 if (uvm_shmexit && vm->vm_shm != NULL) 4263 if (uvm_shmexit && vm->vm_shm != NULL)
4264 (*uvm_shmexit)(vm); 4264 (*uvm_shmexit)(vm);
4265 4265
4266 if (map->nentries) { 4266 if (map->nentries) {
4267 uvm_unmap_remove(map, vm_map_min(map), vm_map_max(map), 4267 uvm_unmap_remove(map, vm_map_min(map), vm_map_max(map),
4268 &dead_entries, flags); 4268 &dead_entries, flags);
4269 if (dead_entries != NULL) 4269 if (dead_entries != NULL)
4270 uvm_unmap_detach(dead_entries, 0); 4270 uvm_unmap_detach(dead_entries, 0);
4271 } 4271 }
4272 KASSERT(map->nentries == 0); 4272 KASSERT(map->nentries == 0);
4273 KASSERT(map->size == 0); 4273 KASSERT(map->size == 0);
4274 4274
4275 mutex_destroy(&map->misc_lock); 4275 mutex_destroy(&map->misc_lock);
4276 rw_destroy(&map->lock); 4276 rw_destroy(&map->lock);
4277 cv_destroy(&map->cv); 4277 cv_destroy(&map->cv);
4278 pmap_destroy(map->pmap); 4278 pmap_destroy(map->pmap);
4279 pool_cache_put(&uvm_vmspace_cache, vm); 4279 pool_cache_put(&uvm_vmspace_cache, vm);
4280} 4280}
4281 4281
4282static struct vm_map_entry * 4282static struct vm_map_entry *
4283uvm_mapent_clone(struct vm_map *new_map, struct vm_map_entry *old_entry, 4283uvm_mapent_clone(struct vm_map *new_map, struct vm_map_entry *old_entry,
4284 int flags) 4284 int flags)
4285{ 4285{
4286 struct vm_map_entry *new_entry; 4286 struct vm_map_entry *new_entry;
4287 4287
4288 new_entry = uvm_mapent_alloc(new_map, 0); 4288 new_entry = uvm_mapent_alloc(new_map, 0);
4289 /* old_entry -> new_entry */ 4289 /* old_entry -> new_entry */
4290 uvm_mapent_copy(old_entry, new_entry); 4290 uvm_mapent_copy(old_entry, new_entry);
4291 4291
4292 /* new pmap has nothing wired in it */ 4292 /* new pmap has nothing wired in it */
4293 new_entry->wired_count = 0; 4293 new_entry->wired_count = 0;
4294 4294
4295 /* 4295 /*
4296 * gain reference to object backing the map (can't 4296 * gain reference to object backing the map (can't
4297 * be a submap, already checked this case). 4297 * be a submap, already checked this case).
4298 */ 4298 */
4299 4299
4300 if (new_entry->aref.ar_amap) 4300 if (new_entry->aref.ar_amap)
4301 uvm_map_reference_amap(new_entry, flags); 4301 uvm_map_reference_amap(new_entry, flags);
4302 4302
4303 if (new_entry->object.uvm_obj && 4303 if (new_entry->object.uvm_obj &&
4304 new_entry->object.uvm_obj->pgops->pgo_reference) 4304 new_entry->object.uvm_obj->pgops->pgo_reference)
4305 new_entry->object.uvm_obj->pgops->pgo_reference( 4305 new_entry->object.uvm_obj->pgops->pgo_reference(
4306 new_entry->object.uvm_obj); 4306 new_entry->object.uvm_obj);
4307 4307
4308 /* insert entry at end of new_map's entry list */ 4308 /* insert entry at end of new_map's entry list */
4309 uvm_map_entry_link(new_map, new_map->header.prev, 4309 uvm_map_entry_link(new_map, new_map->header.prev,
4310 new_entry); 4310 new_entry);
4311 4311
4312 return new_entry; 4312 return new_entry;
4313} 4313}
4314 4314
4315/* 4315/*
4316 * share the mapping: this means we want the old and 4316 * share the mapping: this means we want the old and
4317 * new entries to share amaps and backing objects. 4317 * new entries to share amaps and backing objects.
4318 */ 4318 */
4319static void 4319static void
4320uvm_mapent_forkshared(struct vm_map *new_map, struct vm_map *old_map, 4320uvm_mapent_forkshared(struct vm_map *new_map, struct vm_map *old_map,
4321 struct vm_map_entry *old_entry) 4321 struct vm_map_entry *old_entry)
4322{ 4322{
4323 /* 4323 /*
4324 * if the old_entry needs a new amap (due to prev fork) 4324 * if the old_entry needs a new amap (due to prev fork)
4325 * then we need to allocate it now so that we have 4325 * then we need to allocate it now so that we have
4326 * something we own to share with the new_entry. [in 4326 * something we own to share with the new_entry. [in
4327 * other words, we need to clear needs_copy] 4327 * other words, we need to clear needs_copy]
4328 */ 4328 */
4329 4329
4330 if (UVM_ET_ISNEEDSCOPY(old_entry)) { 4330 if (UVM_ET_ISNEEDSCOPY(old_entry)) {
4331 /* get our own amap, clears needs_copy */ 4331 /* get our own amap, clears needs_copy */
4332 amap_copy(old_map, old_entry, AMAP_COPY_NOCHUNK, 4332 amap_copy(old_map, old_entry, AMAP_COPY_NOCHUNK,
4333 0, 0); 4333 0, 0);
4334 /* XXXCDC: WAITOK??? */ 4334 /* XXXCDC: WAITOK??? */
4335 } 4335 }
4336 4336
4337 uvm_mapent_clone(new_map, old_entry, AMAP_SHARED); 4337 uvm_mapent_clone(new_map, old_entry, AMAP_SHARED);
4338} 4338}
4339 4339
4340 4340
4341static void 4341static void
4342uvm_mapent_forkcopy(struct vm_map *new_map, struct vm_map *old_map, 4342uvm_mapent_forkcopy(struct vm_map *new_map, struct vm_map *old_map,
4343 struct vm_map_entry *old_entry) 4343 struct vm_map_entry *old_entry)
4344{ 4344{
4345 struct vm_map_entry *new_entry; 4345 struct vm_map_entry *new_entry;
4346 4346
4347 /* 4347 /*
4348 * copy-on-write the mapping (using mmap's 4348 * copy-on-write the mapping (using mmap's
4349 * MAP_PRIVATE semantics) 4349 * MAP_PRIVATE semantics)
4350 * 4350 *
4351 * allocate new_entry, adjust reference counts. 4351 * allocate new_entry, adjust reference counts.
4352 * (note that new references are read-only). 4352 * (note that new references are read-only).
4353 */ 4353 */
4354 4354
4355 new_entry = uvm_mapent_clone(new_map, old_entry, 0); 4355 new_entry = uvm_mapent_clone(new_map, old_entry, 0);
4356 4356
4357 new_entry->etype |= 4357 new_entry->etype |=
4358 (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY); 4358 (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY);
4359 4359
4360 /* 4360 /*
4361 * the new entry will need an amap. it will either 4361 * the new entry will need an amap. it will either
4362 * need to be copied from the old entry or created 4362 * need to be copied from the old entry or created
4363 * from scratch (if the old entry does not have an 4363 * from scratch (if the old entry does not have an
4364 * amap). can we defer this process until later 4364 * amap). can we defer this process until later
4365 * (by setting "needs_copy") or do we need to copy 4365 * (by setting "needs_copy") or do we need to copy
4366 * the amap now? 4366 * the amap now?
4367 * 4367 *
4368 * we must copy the amap now if any of the following 4368 * we must copy the amap now if any of the following
4369 * conditions hold: 4369 * conditions hold:
4370 * 1. the old entry has an amap and that amap is 4370 * 1. the old entry has an amap and that amap is
4371 * being shared. this means that the old (parent) 4371 * being shared. this means that the old (parent)
4372 * process is sharing the amap with another 4372 * process is sharing the amap with another
4373 * process. if we do not clear needs_copy here 4373 * process. if we do not clear needs_copy here
4374 * we will end up in a situation where both the 4374 * we will end up in a situation where both the
4375 * parent and child process are refering to the 4375 * parent and child process are refering to the
4376 * same amap with "needs_copy" set. if the 4376 * same amap with "needs_copy" set. if the
4377 * parent write-faults, the fault routine will 4377 * parent write-faults, the fault routine will
4378 * clear "needs_copy" in the parent by allocating 4378 * clear "needs_copy" in the parent by allocating
4379 * a new amap. this is wrong because the 4379 * a new amap. this is wrong because the
4380 * parent is supposed to be sharing the old amap 4380 * parent is supposed to be sharing the old amap
4381 * and the new amap will break that. 4381 * and the new amap will break that.
4382 * 4382 *
4383 * 2. if the old entry has an amap and a non-zero 4383 * 2. if the old entry has an amap and a non-zero
4384 * wire count then we are going to have to call 4384 * wire count then we are going to have to call
4385 * amap_cow_now to avoid page faults in the 4385 * amap_cow_now to avoid page faults in the
4386 * parent process. since amap_cow_now requires 4386 * parent process. since amap_cow_now requires
4387 * "needs_copy" to be clear we might as well 4387 * "needs_copy" to be clear we might as well
4388 * clear it here as well. 4388 * clear it here as well.
4389 * 4389 *
4390 */ 4390 */
4391 4391
4392 if (old_entry->aref.ar_amap != NULL) { 4392 if (old_entry->aref.ar_amap != NULL) {
4393 if ((amap_flags(old_entry->aref.ar_amap) & AMAP_SHARED) != 0 || 4393 if ((amap_flags(old_entry->aref.ar_amap) & AMAP_SHARED) != 0 ||
4394 VM_MAPENT_ISWIRED(old_entry)) { 4394 VM_MAPENT_ISWIRED(old_entry)) {
4395 4395
4396 amap_copy(new_map, new_entry, 4396 amap_copy(new_map, new_entry,
4397 AMAP_COPY_NOCHUNK, 0, 0); 4397 AMAP_COPY_NOCHUNK, 0, 0);
4398 /* XXXCDC: M_WAITOK ... ok? */ 4398 /* XXXCDC: M_WAITOK ... ok? */
4399 } 4399 }
4400 } 4400 }
4401 4401
4402 /* 4402 /*
4403 * if the parent's entry is wired down, then the 4403 * if the parent's entry is wired down, then the
4404 * parent process does not want page faults on 4404 * parent process does not want page faults on
4405 * access to that memory. this means that we 4405 * access to that memory. this means that we
4406 * cannot do copy-on-write because we can't write 4406 * cannot do copy-on-write because we can't write
4407 * protect the old entry. in this case we 4407 * protect the old entry. in this case we
4408 * resolve all copy-on-write faults now, using 4408 * resolve all copy-on-write faults now, using
4409 * amap_cow_now. note that we have already 4409 * amap_cow_now. note that we have already
4410 * allocated any needed amap (above). 4410 * allocated any needed amap (above).
4411 */ 4411 */
4412 4412
4413 if (VM_MAPENT_ISWIRED(old_entry)) { 4413 if (VM_MAPENT_ISWIRED(old_entry)) {
4414 4414
4415 /* 4415 /*
4416 * resolve all copy-on-write faults now 4416 * resolve all copy-on-write faults now
4417 * (note that there is nothing to do if 4417 * (note that there is nothing to do if
4418 * the old mapping does not have an amap). 4418 * the old mapping does not have an amap).
4419 */ 4419 */
4420 if (old_entry->aref.ar_amap) 4420 if (old_entry->aref.ar_amap)
4421 amap_cow_now(new_map, new_entry); 4421 amap_cow_now(new_map, new_entry);
4422 4422
4423 } else { 4423 } else {
4424 /* 4424 /*
4425 * setup mappings to trigger copy-on-write faults 4425 * setup mappings to trigger copy-on-write faults
4426 * we must write-protect the parent if it has 4426 * we must write-protect the parent if it has
4427 * an amap and it is not already "needs_copy"... 4427 * an amap and it is not already "needs_copy"...
4428 * if it is already "needs_copy" then the parent 4428 * if it is already "needs_copy" then the parent
4429 * has already been write-protected by a previous 4429 * has already been write-protected by a previous
4430 * fork operation. 4430 * fork operation.
4431 */ 4431 */
4432 if (old_entry->aref.ar_amap && 4432 if (old_entry->aref.ar_amap &&
4433 !UVM_ET_ISNEEDSCOPY(old_entry)) { 4433 !UVM_ET_ISNEEDSCOPY(old_entry)) {
4434 if (old_entry->max_protection & VM_PROT_WRITE) { 4434 if (old_entry->max_protection & VM_PROT_WRITE) {
4435#ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */ 4435#ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */
4436 uvm_map_lock_entry(old_entry, RW_WRITER); 4436 uvm_map_lock_entry(old_entry, RW_WRITER);
4437#else 4437#else
4438 uvm_map_lock_entry(old_entry, RW_READER); 4438 uvm_map_lock_entry(old_entry, RW_READER);
4439#endif 4439#endif
4440 pmap_protect(old_map->pmap, 4440 pmap_protect(old_map->pmap,
4441 old_entry->start, old_entry->end, 4441 old_entry->start, old_entry->end,
4442 old_entry->protection & ~VM_PROT_WRITE); 4442 old_entry->protection & ~VM_PROT_WRITE);
4443 uvm_map_unlock_entry(old_entry); 4443 uvm_map_unlock_entry(old_entry);
4444 } 4444 }
4445 old_entry->etype |= UVM_ET_NEEDSCOPY; 4445 old_entry->etype |= UVM_ET_NEEDSCOPY;
4446 } 4446 }
4447 } 4447 }
4448} 4448}
4449 4449
4450/* 4450/*
4451 * zero the mapping: the new entry will be zero initialized 4451 * zero the mapping: the new entry will be zero initialized
4452 */ 4452 */
4453static void 4453static void
4454uvm_mapent_forkzero(struct vm_map *new_map, struct vm_map *old_map, 4454uvm_mapent_forkzero(struct vm_map *new_map, struct vm_map *old_map,
4455 struct vm_map_entry *old_entry) 4455 struct vm_map_entry *old_entry)
4456{ 4456{
4457 struct vm_map_entry *new_entry; 4457 struct vm_map_entry *new_entry;
4458 4458
4459 new_entry = uvm_mapent_clone(new_map, old_entry, 0); 4459 new_entry = uvm_mapent_clone(new_map, old_entry, 0);
4460 4460
4461 new_entry->etype |= 4461 new_entry->etype |=
4462 (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY); 4462 (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY);
4463 4463
4464 if (new_entry->aref.ar_amap) { 4464 if (new_entry->aref.ar_amap) {
4465 uvm_map_unreference_amap(new_entry, 0); 4465 uvm_map_unreference_amap(new_entry, 0);
4466 new_entry->aref.ar_pageoff = 0; 4466 new_entry->aref.ar_pageoff = 0;
4467 new_entry->aref.ar_amap = NULL; 4467 new_entry->aref.ar_amap = NULL;
4468 } 4468 }
4469 4469
4470 if (UVM_ET_ISOBJ(new_entry)) { 4470 if (UVM_ET_ISOBJ(new_entry)) {
4471 if (new_entry->object.uvm_obj->pgops->pgo_detach) 4471 if (new_entry->object.uvm_obj->pgops->pgo_detach)
4472 new_entry->object.uvm_obj->pgops->pgo_detach( 4472 new_entry->object.uvm_obj->pgops->pgo_detach(
4473 new_entry->object.uvm_obj); 4473 new_entry->object.uvm_obj);
4474 new_entry->object.uvm_obj = NULL; 4474 new_entry->object.uvm_obj = NULL;
4475 new_entry->etype &= ~UVM_ET_OBJ; 4475 new_entry->etype &= ~UVM_ET_OBJ;
4476 } 4476 }
4477} 4477}
4478 4478
4479/* 4479/*
4480 * F O R K - m a i n e n t r y p o i n t 4480 * F O R K - m a i n e n t r y p o i n t
4481 */ 4481 */
4482/* 4482/*
4483 * uvmspace_fork: fork a process' main map 4483 * uvmspace_fork: fork a process' main map
4484 * 4484 *
4485 * => create a new vmspace for child process from parent. 4485 * => create a new vmspace for child process from parent.
4486 * => parent's map must not be locked. 4486 * => parent's map must not be locked.
4487 */ 4487 */
4488 4488
4489struct vmspace * 4489struct vmspace *
4490uvmspace_fork(struct vmspace *vm1) 4490uvmspace_fork(struct vmspace *vm1)
4491{ 4491{
4492 struct vmspace *vm2; 4492 struct vmspace *vm2;
4493 struct vm_map *old_map = &vm1->vm_map; 4493 struct vm_map *old_map = &vm1->vm_map;
4494 struct vm_map *new_map; 4494 struct vm_map *new_map;
4495 struct vm_map_entry *old_entry; 4495 struct vm_map_entry *old_entry;
4496 UVMHIST_FUNC("uvmspace_fork"); UVMHIST_CALLED(maphist); 4496 UVMHIST_FUNC("uvmspace_fork"); UVMHIST_CALLED(maphist);
4497 4497
4498 vm_map_lock(old_map); 4498 vm_map_lock(old_map);
4499 4499
4500 vm2 = uvmspace_alloc(vm_map_min(old_map), vm_map_max(old_map), 4500 vm2 = uvmspace_alloc(vm_map_min(old_map), vm_map_max(old_map),
4501 vm1->vm_map.flags & VM_MAP_TOPDOWN); 4501 vm1->vm_map.flags & VM_MAP_TOPDOWN);
4502 memcpy(&vm2->vm_startcopy, &vm1->vm_startcopy, 4502 memcpy(&vm2->vm_startcopy, &vm1->vm_startcopy,
4503 (char *) (vm1 + 1) - (char *) &vm1->vm_startcopy); 4503 (char *) (vm1 + 1) - (char *) &vm1->vm_startcopy);
4504 new_map = &vm2->vm_map; /* XXX */ 4504 new_map = &vm2->vm_map; /* XXX */
4505 4505
4506 old_entry = old_map->header.next; 4506 old_entry = old_map->header.next;
4507 new_map->size = old_map->size; 4507 new_map->size = old_map->size;
4508 4508
4509 /* 4509 /*
4510 * go entry-by-entry 4510 * go entry-by-entry
4511 */ 4511 */
4512 4512
4513 while (old_entry != &old_map->header) { 4513 while (old_entry != &old_map->header) {
4514 4514
4515 /* 4515 /*
4516 * first, some sanity checks on the old entry 4516 * first, some sanity checks on the old entry
4517 */ 4517 */
4518 4518
4519 KASSERT(!UVM_ET_ISSUBMAP(old_entry)); 4519 KASSERT(!UVM_ET_ISSUBMAP(old_entry));
4520 KASSERT(UVM_ET_ISCOPYONWRITE(old_entry) || 4520 KASSERT(UVM_ET_ISCOPYONWRITE(old_entry) ||
4521 !UVM_ET_ISNEEDSCOPY(old_entry)); 4521 !UVM_ET_ISNEEDSCOPY(old_entry));
4522 4522
4523 switch (old_entry->inheritance) { 4523 switch (old_entry->inheritance) {
4524 case MAP_INHERIT_NONE: 4524 case MAP_INHERIT_NONE:
4525 /* 4525 /*
4526 * drop the mapping, modify size 4526 * drop the mapping, modify size
4527 */ 4527 */
4528 new_map->size -= old_entry->end - old_entry->start; 4528 new_map->size -= old_entry->end - old_entry->start;
4529 break; 4529 break;
4530 4530
4531 case MAP_INHERIT_SHARE: 4531 case MAP_INHERIT_SHARE:
4532 uvm_mapent_forkshared(new_map, old_map, old_entry); 4532 uvm_mapent_forkshared(new_map, old_map, old_entry);
4533 break; 4533 break;
4534 4534
4535 case MAP_INHERIT_COPY: 4535 case MAP_INHERIT_COPY:
4536 uvm_mapent_forkcopy(new_map, old_map, old_entry); 4536 uvm_mapent_forkcopy(new_map, old_map, old_entry);
4537 break; 4537 break;
4538 4538
4539 case MAP_INHERIT_ZERO: 4539 case MAP_INHERIT_ZERO:
4540 uvm_mapent_forkzero(new_map, old_map, old_entry); 4540 uvm_mapent_forkzero(new_map, old_map, old_entry);
4541 break; 4541 break;
4542 default: 4542 default:
4543 KASSERT(0); 4543 KASSERT(0);
4544 break; 4544 break;
4545 } 4545 }
4546 old_entry = old_entry->next; 4546 old_entry = old_entry->next;
4547 } 4547 }
4548 4548
4549 pmap_update(old_map->pmap); 4549 pmap_update(old_map->pmap);
4550 vm_map_unlock(old_map); 4550 vm_map_unlock(old_map);
4551 4551
4552 if (uvm_shmfork && vm1->vm_shm) 4552 if (uvm_shmfork && vm1->vm_shm)
4553 (*uvm_shmfork)(vm1, vm2); 4553 (*uvm_shmfork)(vm1, vm2);
4554 4554
4555#ifdef PMAP_FORK 4555#ifdef PMAP_FORK
4556 pmap_fork(vm1->vm_map.pmap, vm2->vm_map.pmap); 4556 pmap_fork(vm1->vm_map.pmap, vm2->vm_map.pmap);
4557#endif 4557#endif
4558 4558
4559 UVMHIST_LOG(maphist,"<- done",0,0,0,0); 4559 UVMHIST_LOG(maphist,"<- done",0,0,0,0);
4560 return (vm2); 4560 return (vm2);
4561} 4561}
4562 4562
4563 4563
4564/* 4564/*
4565 * uvm_mapent_trymerge: try to merge an entry with its neighbors. 4565 * uvm_mapent_trymerge: try to merge an entry with its neighbors.
4566 * 4566 *
4567 * => called with map locked. 4567 * => called with map locked.
4568 * => return non zero if successfully merged. 4568 * => return non zero if successfully merged.
4569 */ 4569 */
4570 4570
4571int 4571int
4572uvm_mapent_trymerge(struct vm_map *map, struct vm_map_entry *entry, int flags) 4572uvm_mapent_trymerge(struct vm_map *map, struct vm_map_entry *entry, int flags)
4573{ 4573{
4574 struct uvm_object *uobj; 4574 struct uvm_object *uobj;
4575 struct vm_map_entry *next; 4575 struct vm_map_entry *next;
4576 struct vm_map_entry *prev; 4576 struct vm_map_entry *prev;
4577 vsize_t size; 4577 vsize_t size;
4578 int merged = 0; 4578 int merged = 0;
4579 bool copying; 4579 bool copying;
4580 int newetype; 4580 int newetype;
4581 4581
4582 if (entry->aref.ar_amap != NULL) { 4582 if (entry->aref.ar_amap != NULL) {
4583 return 0; 4583 return 0;
4584 } 4584 }
4585 if ((entry->flags & UVM_MAP_NOMERGE) != 0) { 4585 if ((entry->flags & UVM_MAP_NOMERGE) != 0) {
4586 return 0; 4586 return 0;
4587 } 4587 }
4588 4588
4589 uobj = entry->object.uvm_obj; 4589 uobj = entry->object.uvm_obj;
4590 size = entry->end - entry->start; 4590 size = entry->end - entry->start;
4591 copying = (flags & UVM_MERGE_COPYING) != 0; 4591 copying = (flags & UVM_MERGE_COPYING) != 0;
4592 newetype = copying ? (entry->etype & ~UVM_ET_NEEDSCOPY) : entry->etype; 4592 newetype = copying ? (entry->etype & ~UVM_ET_NEEDSCOPY) : entry->etype;
4593 4593
4594 next = entry->next; 4594 next = entry->next;
4595 if (next != &map->header && 4595 if (next != &map->header &&
4596 next->start == entry->end && 4596 next->start == entry->end &&
4597 ((copying && next->aref.ar_amap != NULL && 4597 ((copying && next->aref.ar_amap != NULL &&
4598 amap_refs(next->aref.ar_amap) == 1) || 4598 amap_refs(next->aref.ar_amap) == 1) ||
4599 (!copying && next->aref.ar_amap == NULL)) && 4599 (!copying && next->aref.ar_amap == NULL)) &&
4600 UVM_ET_ISCOMPATIBLE(next, newetype, 4600 UVM_ET_ISCOMPATIBLE(next, newetype,
4601 uobj, entry->flags, entry->protection, 4601 uobj, entry->flags, entry->protection,
4602 entry->max_protection, entry->inheritance, entry->advice, 4602 entry->max_protection, entry->inheritance, entry->advice,
4603 entry->wired_count) && 4603 entry->wired_count) &&
4604 (uobj == NULL || entry->offset + size == next->offset)) { 4604 (uobj == NULL || entry->offset + size == next->offset)) {
4605 int error; 4605 int error;
4606 4606
4607 if (copying) { 4607 if (copying) {
4608 error = amap_extend(next, size, 4608 error = amap_extend(next, size,
4609 AMAP_EXTEND_NOWAIT|AMAP_EXTEND_BACKWARDS); 4609 AMAP_EXTEND_NOWAIT|AMAP_EXTEND_BACKWARDS);
4610 } else { 4610 } else {
4611 error = 0; 4611 error = 0;
4612 } 4612 }
4613 if (error == 0) { 4613 if (error == 0) {
4614 if (uobj) { 4614 if (uobj) {
4615 if (uobj->pgops->pgo_detach) { 4615 if (uobj->pgops->pgo_detach) {
4616 uobj->pgops->pgo_detach(uobj); 4616 uobj->pgops->pgo_detach(uobj);
4617 } 4617 }
4618 } 4618 }
4619 4619
4620 entry->end = next->end; 4620 entry->end = next->end;
4621 clear_hints(map, next); 4621 clear_hints(map, next);
4622 uvm_map_entry_unlink(map, next); 4622 uvm_map_entry_unlink(map, next);
4623 if (copying) { 4623 if (copying) {
4624 entry->aref = next->aref; 4624 entry->aref = next->aref;
4625 entry->etype &= ~UVM_ET_NEEDSCOPY; 4625 entry->etype &= ~UVM_ET_NEEDSCOPY;
4626 } 4626 }
4627 uvm_map_check(map, "trymerge forwardmerge"); 4627 uvm_map_check(map, "trymerge forwardmerge");
4628 uvm_mapent_free(next); 4628 uvm_mapent_free(next);
4629 merged++; 4629 merged++;
4630 } 4630 }
4631 } 4631 }
4632 4632
4633 prev = entry->prev; 4633 prev = entry->prev;
4634 if (prev != &map->header && 4634 if (prev != &map->header &&
4635 prev->end == entry->start && 4635 prev->end == entry->start &&
4636 ((copying && !merged && prev->aref.ar_amap != NULL && 4636 ((copying && !merged && prev->aref.ar_amap != NULL &&
4637 amap_refs(prev->aref.ar_amap) == 1) || 4637 amap_refs(prev->aref.ar_amap) == 1) ||
4638 (!copying && prev->aref.ar_amap == NULL)) && 4638 (!copying && prev->aref.ar_amap == NULL)) &&
4639 UVM_ET_ISCOMPATIBLE(prev, newetype, 4639 UVM_ET_ISCOMPATIBLE(prev, newetype,
4640 uobj, entry->flags, entry->protection, 4640 uobj, entry->flags, entry->protection,
4641 entry->max_protection, entry->inheritance, entry->advice, 4641 entry->max_protection, entry->inheritance, entry->advice,
4642 entry->wired_count) && 4642 entry->wired_count) &&
4643 (uobj == NULL || 4643 (uobj == NULL ||
4644 prev->offset + prev->end - prev->start == entry->offset)) { 4644 prev->offset + prev->end - prev->start == entry->offset)) {
4645 int error; 4645 int error;
4646 4646
4647 if (copying) { 4647 if (copying) {
4648 error = amap_extend(prev, size, 4648 error = amap_extend(prev, size,
4649 AMAP_EXTEND_NOWAIT|AMAP_EXTEND_FORWARDS); 4649 AMAP_EXTEND_NOWAIT|AMAP_EXTEND_FORWARDS);
4650 } else { 4650 } else {
4651 error = 0; 4651 error = 0;
4652 } 4652 }
4653 if (error == 0) { 4653 if (error == 0) {
4654 if (uobj) { 4654 if (uobj) {
4655 if (uobj->pgops->pgo_detach) { 4655 if (uobj->pgops->pgo_detach) {
4656 uobj->pgops->pgo_detach(uobj); 4656 uobj->pgops->pgo_detach(uobj);
4657 } 4657 }
4658 entry->offset = prev->offset; 4658 entry->offset = prev->offset;
4659 } 4659 }
4660 4660
4661 entry->start = prev->start; 4661 entry->start = prev->start;
4662 clear_hints(map, prev); 4662 clear_hints(map, prev);
4663 uvm_map_entry_unlink(map, prev); 4663 uvm_map_entry_unlink(map, prev);
4664 if (copying) { 4664 if (copying) {
4665 entry->aref = prev->aref; 4665 entry->aref = prev->aref;
4666 entry->etype &= ~UVM_ET_NEEDSCOPY; 4666 entry->etype &= ~UVM_ET_NEEDSCOPY;
4667 } 4667 }
4668 uvm_map_check(map, "trymerge backmerge"); 4668 uvm_map_check(map, "trymerge backmerge");
4669 uvm_mapent_free(prev); 4669 uvm_mapent_free(prev);
4670 merged++; 4670 merged++;
4671 } 4671 }
4672 } 4672 }
4673 4673
4674 return merged; 4674 return merged;
4675} 4675}
4676 4676
4677/* 4677/*
4678 * uvm_map_setup: init map 4678 * uvm_map_setup: init map
4679 * 4679 *
4680 * => map must not be in service yet. 4680 * => map must not be in service yet.
4681 */ 4681 */
4682 4682
4683void 4683void
4684uvm_map_setup(struct vm_map *map, vaddr_t vmin, vaddr_t vmax, int flags) 4684uvm_map_setup(struct vm_map *map, vaddr_t vmin, vaddr_t vmax, int flags)
4685{ 4685{
4686 4686
4687 rb_tree_init(&map->rb_tree, &uvm_map_tree_ops); 4687 rb_tree_init(&map->rb_tree, &uvm_map_tree_ops);
4688 map->header.next = map->header.prev = &map->header; 4688 map->header.next = map->header.prev = &map->header;
4689 map->nentries = 0; 4689 map->nentries = 0;
4690 map->size = 0; 4690 map->size = 0;
4691 map->ref_count = 1; 4691 map->ref_count = 1;
4692 vm_map_setmin(map, vmin); 4692 vm_map_setmin(map, vmin);
4693 vm_map_setmax(map, vmax); 4693 vm_map_setmax(map, vmax);
4694 map->flags = flags; 4694 map->flags = flags;
4695 map->first_free = &map->header; 4695 map->first_free = &map->header;
4696 map->hint = &map->header; 4696 map->hint = &map->header;
4697 map->timestamp = 0; 4697 map->timestamp = 0;
4698 map->busy = NULL; 4698 map->busy = NULL;
4699 4699
4700 rw_init(&map->lock); 4700 rw_init(&map->lock);
4701 cv_init(&map->cv, "vm_map"); 4701 cv_init(&map->cv, "vm_map");
4702 mutex_init(&map->misc_lock, MUTEX_DRIVER, IPL_NONE); 4702 mutex_init(&map->misc_lock, MUTEX_DRIVER, IPL_NONE);
4703} 4703}
4704 4704
4705/* 4705/*
4706 * U N M A P - m a i n e n t r y p o i n t 4706 * U N M A P - m a i n e n t r y p o i n t
4707 */ 4707 */
4708 4708
4709/* 4709/*
4710 * uvm_unmap1: remove mappings from a vm_map (from "start" up to "stop") 4710 * uvm_unmap1: remove mappings from a vm_map (from "start" up to "stop")
4711 * 4711 *
4712 * => caller must check alignment and size 4712 * => caller must check alignment and size
4713 * => map must be unlocked (we will lock it) 4713 * => map must be unlocked (we will lock it)
4714 * => flags is UVM_FLAG_QUANTUM or 0. 4714 * => flags is UVM_FLAG_QUANTUM or 0.
4715 */ 4715 */
4716 4716
4717void 4717void
4718uvm_unmap1(struct vm_map *map, vaddr_t start, vaddr_t end, int flags) 4718uvm_unmap1(struct vm_map *map, vaddr_t start, vaddr_t end, int flags)
4719{ 4719{
4720 struct vm_map_entry *dead_entries; 4720 struct vm_map_entry *dead_entries;
4721 UVMHIST_FUNC("uvm_unmap"); UVMHIST_CALLED(maphist); 4721 UVMHIST_FUNC("uvm_unmap"); UVMHIST_CALLED(maphist);
4722 4722
4723 KASSERTMSG(start < end, 4723 KASSERTMSG(start < end,
4724 "%s: map %p: start %#jx < end %#jx", __func__, map, 4724 "%s: map %p: start %#jx < end %#jx", __func__, map,
4725 (uintmax_t)start, (uintmax_t)end); 4725 (uintmax_t)start, (uintmax_t)end);
4726 UVMHIST_LOG(maphist, " (map=%#jx, start=%#jx, end=%#jx)", 4726 UVMHIST_LOG(maphist, " (map=%#jx, start=%#jx, end=%#jx)",
4727 (uintptr_t)map, start, end, 0); 4727 (uintptr_t)map, start, end, 0);
4728 if (map == kernel_map) { 4728 if (map == kernel_map) {
4729 LOCKDEBUG_MEM_CHECK((void *)start, end - start); 4729 LOCKDEBUG_MEM_CHECK((void *)start, end - start);
4730 } 4730 }
4731 4731
4732 /* 4732 /*
4733 * work now done by helper functions. wipe the pmap's and then 4733 * work now done by helper functions. wipe the pmap's and then
4734 * detach from the dead entries... 4734 * detach from the dead entries...
4735 */ 4735 */
4736 vm_map_lock(map); 4736 vm_map_lock(map);
4737 uvm_unmap_remove(map, start, end, &dead_entries, flags); 4737 uvm_unmap_remove(map, start, end, &dead_entries, flags);
4738 vm_map_unlock(map); 4738 vm_map_unlock(map);
4739 4739
4740 if (dead_entries != NULL) 4740 if (dead_entries != NULL)
4741 uvm_unmap_detach(dead_entries, 0); 4741 uvm_unmap_detach(dead_entries, 0);
4742 4742
4743 UVMHIST_LOG(maphist, "<- done", 0,0,0,0); 4743 UVMHIST_LOG(maphist, "<- done", 0,0,0,0);
4744} 4744}
4745 4745
4746 4746
4747/* 4747/*
4748 * uvm_map_reference: add reference to a map 4748 * uvm_map_reference: add reference to a map
4749 * 4749 *
4750 * => map need not be locked 4750 * => map need not be locked
4751 */ 4751 */
4752 4752
4753void 4753void
4754uvm_map_reference(struct vm_map *map) 4754uvm_map_reference(struct vm_map *map)
4755{ 4755{
4756 4756
4757 atomic_inc_uint(&map->ref_count); 4757 atomic_inc_uint(&map->ref_count);
4758} 4758}
4759 4759
4760void 4760void
4761uvm_map_lock_entry(struct vm_map_entry *entry, krw_t op) 4761uvm_map_lock_entry(struct vm_map_entry *entry, krw_t op)
4762{ 4762{
4763 4763
4764 if (entry->aref.ar_amap != NULL) { 4764 if (entry->aref.ar_amap != NULL) {
4765 amap_lock(entry->aref.ar_amap, op); 4765 amap_lock(entry->aref.ar_amap, op);
4766 } 4766 }
4767 if (UVM_ET_ISOBJ(entry)) { 4767 if (UVM_ET_ISOBJ(entry)) {
4768 rw_enter(entry->object.uvm_obj->vmobjlock, op); 4768 rw_enter(entry->object.uvm_obj->vmobjlock, op);
4769 } 4769 }
4770} 4770}
4771 4771
4772void 4772void
4773uvm_map_unlock_entry(struct vm_map_entry *entry) 4773uvm_map_unlock_entry(struct vm_map_entry *entry)
4774{ 4774{
4775 4775
4776 if (UVM_ET_ISOBJ(entry)) { 4776 if (UVM_ET_ISOBJ(entry)) {
4777 rw_exit(entry->object.uvm_obj->vmobjlock); 4777 rw_exit(entry->object.uvm_obj->vmobjlock);
4778 } 4778 }
4779 if (entry->aref.ar_amap != NULL) { 4779 if (entry->aref.ar_amap != NULL) {
4780 amap_unlock(entry->aref.ar_amap); 4780 amap_unlock(entry->aref.ar_amap);
4781 } 4781 }
4782} 4782}
4783 4783
4784/* 4784/*
4785 * uvm_voaddr_acquire: returns the virtual object address corresponding 4785 * uvm_voaddr_acquire: returns the virtual object address corresponding
4786 * to the specified virtual address. 4786 * to the specified virtual address.
4787 * 4787 *
4788 * => resolves COW so the true page identity is tracked. 4788 * => resolves COW so the true page identity is tracked.
4789 * 4789 *
4790 * => acquires a reference on the page's owner (uvm_object or vm_anon) 4790 * => acquires a reference on the page's owner (uvm_object or vm_anon)
4791 */ 4791 */
4792bool 4792bool
4793uvm_voaddr_acquire(struct vm_map * const map, vaddr_t const va, 4793uvm_voaddr_acquire(struct vm_map * const map, vaddr_t const va,
4794 struct uvm_voaddr * const voaddr) 4794 struct uvm_voaddr * const voaddr)
4795{ 4795{
4796 struct vm_map_entry *entry; 4796 struct vm_map_entry *entry;
4797 struct vm_anon *anon = NULL; 4797 struct vm_anon *anon = NULL;
4798 bool result = false; 4798 bool result = false;
4799 bool exclusive = false; 4799 bool exclusive = false;
4800 void (*unlock_fn)(struct vm_map *); 4800 void (*unlock_fn)(struct vm_map *);
4801 4801
4802 UVMHIST_FUNC("uvm_voaddr_acquire"); UVMHIST_CALLED(maphist); 4802 UVMHIST_FUNC("uvm_voaddr_acquire"); UVMHIST_CALLED(maphist);
4803 UVMHIST_LOG(maphist,"(map=%#jx,va=%jx)", (uintptr_t)map, va, 0, 0); 4803 UVMHIST_LOG(maphist,"(map=%#jx,va=%jx)", (uintptr_t)map, va, 0, 0);
4804 4804
4805 const vaddr_t start = trunc_page(va); 4805 const vaddr_t start = trunc_page(va);
4806 const vaddr_t end = round_page(va+1); 4806 const vaddr_t end = round_page(va+1);
4807 4807
4808 lookup_again: 4808 lookup_again:
4809 if (__predict_false(exclusive)) { 4809 if (__predict_false(exclusive)) {
4810 vm_map_lock(map); 4810 vm_map_lock(map);
4811 unlock_fn = vm_map_unlock; 4811 unlock_fn = vm_map_unlock;
4812 } else { 4812 } else {
4813 vm_map_lock_read(map); 4813 vm_map_lock_read(map);
4814 unlock_fn = vm_map_unlock_read; 4814 unlock_fn = vm_map_unlock_read;
4815 } 4815 }
4816 4816
4817 if (__predict_false(!uvm_map_lookup_entry(map, start, &entry))) { 4817 if (__predict_false(!uvm_map_lookup_entry(map, start, &entry))) {
4818 unlock_fn(map); 4818 unlock_fn(map);
4819 UVMHIST_LOG(maphist,"<- done (no entry)",0,0,0,0); 4819 UVMHIST_LOG(maphist,"<- done (no entry)",0,0,0,0);
4820 return false; 4820 return false;
4821 } 4821 }
4822 4822
4823 if (__predict_false(entry->protection == VM_PROT_NONE)) { 4823 if (__predict_false(entry->protection == VM_PROT_NONE)) {
4824 unlock_fn(map); 4824 unlock_fn(map);
4825 UVMHIST_LOG(maphist,"<- done (PROT_NONE)",0,0,0,0); 4825 UVMHIST_LOG(maphist,"<- done (PROT_NONE)",0,0,0,0);
4826 return false; 4826 return false;
4827 } 4827 }
4828 4828
4829 /* 4829 /*
4830 * We have a fast path for the common case of "no COW resolution 4830 * We have a fast path for the common case of "no COW resolution
4831 * needed" whereby we have taken a read lock on the map and if 4831 * needed" whereby we have taken a read lock on the map and if
4832 * we don't encounter any need to create a vm_anon then great! 4832 * we don't encounter any need to create a vm_anon then great!
4833 * But if we do, we loop around again, instead taking an exclusive 4833 * But if we do, we loop around again, instead taking an exclusive
4834 * lock so that we can perform the fault. 4834 * lock so that we can perform the fault.
4835 * 4835 *
4836 * In the event that we have to resolve the fault, we do nearly the 4836 * In the event that we have to resolve the fault, we do nearly the
4837 * same work as uvm_map_pageable() does: 4837 * same work as uvm_map_pageable() does:
4838 * 4838 *
4839 * 1: holding the write lock, we create any anonymous maps that need 4839 * 1: holding the write lock, we create any anonymous maps that need
4840 * to be created. however, we do NOT need to clip the map entries 4840 * to be created. however, we do NOT need to clip the map entries
4841 * in this case. 4841 * in this case.
4842 * 4842 *
4843 * 2: we downgrade to a read lock, and call uvm_fault_wire to fault 4843 * 2: we downgrade to a read lock, and call uvm_fault_wire to fault
4844 * in the page (assuming the entry is not already wired). this 4844 * in the page (assuming the entry is not already wired). this
4845 * is done because we need the vm_anon to be present. 4845 * is done because we need the vm_anon to be present.
4846 */ 4846 */
4847 if (__predict_true(!VM_MAPENT_ISWIRED(entry))) { 4847 if (__predict_true(!VM_MAPENT_ISWIRED(entry))) {
4848 4848
4849 bool need_fault = false; 4849 bool need_fault = false;
4850 4850
4851 /* 4851 /*
4852 * perform the action of vm_map_lookup that need the 4852 * perform the action of vm_map_lookup that need the
4853 * write lock on the map: create an anonymous map for 4853 * write lock on the map: create an anonymous map for
4854 * a copy-on-write region, or an anonymous map for 4854 * a copy-on-write region, or an anonymous map for
4855 * a zero-fill region. 4855 * a zero-fill region.
4856 */ 4856 */
4857 if (__predict_false(UVM_ET_ISSUBMAP(entry))) { 4857 if (__predict_false(UVM_ET_ISSUBMAP(entry))) {
4858 unlock_fn(map); 4858 unlock_fn(map);
4859 UVMHIST_LOG(maphist,"<- done (submap)",0,0,0,0); 4859 UVMHIST_LOG(maphist,"<- done (submap)",0,0,0,0);
4860 return false; 4860 return false;
4861 } 4861 }
4862 if (__predict_false(UVM_ET_ISNEEDSCOPY(entry) && 4862 if (__predict_false(UVM_ET_ISNEEDSCOPY(entry) &&
4863 ((entry->max_protection & VM_PROT_WRITE) || 4863 ((entry->max_protection & VM_PROT_WRITE) ||
4864 (entry->object.uvm_obj == NULL)))) { 4864 (entry->object.uvm_obj == NULL)))) {
4865 if (!exclusive) { 4865 if (!exclusive) {
4866 /* need to take the slow path */ 4866 /* need to take the slow path */
4867 KASSERT(unlock_fn == vm_map_unlock_read); 4867 KASSERT(unlock_fn == vm_map_unlock_read);
4868 vm_map_unlock_read(map); 4868 vm_map_unlock_read(map);
4869 exclusive = true; 4869 exclusive = true;
4870 goto lookup_again; 4870 goto lookup_again;
4871 } 4871 }
4872 need_fault = true; 4872 need_fault = true;
4873 amap_copy(map, entry, 0, start, end); 4873 amap_copy(map, entry, 0, start, end);
4874 /* XXXCDC: wait OK? */ 4874 /* XXXCDC: wait OK? */
4875 } 4875 }
4876 4876
4877 /* 4877 /*
4878 * do a quick check to see if the fault has already 4878 * do a quick check to see if the fault has already
4879 * been resolved to the upper layer. 4879 * been resolved to the upper layer.
4880 */ 4880 */
4881 if (__predict_true(entry->aref.ar_amap != NULL && 4881 if (__predict_true(entry->aref.ar_amap != NULL &&
4882 need_fault == false)) { 4882 need_fault == false)) {
4883 amap_lock(entry->aref.ar_amap, RW_WRITER); 4883 amap_lock(entry->aref.ar_amap, RW_WRITER);
4884 anon = amap_lookup(&entry->aref, start - entry->start); 4884 anon = amap_lookup(&entry->aref, start - entry->start);
4885 if (__predict_true(anon != NULL)) { 4885 if (__predict_true(anon != NULL)) {
4886 /* amap unlocked below */ 4886 /* amap unlocked below */
4887 goto found_anon; 4887 goto found_anon;
4888 } 4888 }
4889 amap_unlock(entry->aref.ar_amap); 4889 amap_unlock(entry->aref.ar_amap);
4890 need_fault = true; 4890 need_fault = true;
4891 } 4891 }
4892 4892
4893 /* 4893 /*
4894 * we predict this test as false because if we reach 4894 * we predict this test as false because if we reach
4895 * this point, then we are likely dealing with a 4895 * this point, then we are likely dealing with a
4896 * shared memory region backed by a uvm_object, in 4896 * shared memory region backed by a uvm_object, in
4897 * which case a fault to create the vm_anon is not 4897 * which case a fault to create the vm_anon is not
4898 * necessary. 4898 * necessary.
4899 */ 4899 */
4900 if (__predict_false(need_fault)) { 4900 if (__predict_false(need_fault)) {
4901 if (exclusive) { 4901 if (exclusive) {
4902 vm_map_busy(map); 4902 vm_map_busy(map);
4903 vm_map_unlock(map); 4903 vm_map_unlock(map);
4904 unlock_fn = vm_map_unbusy; 4904 unlock_fn = vm_map_unbusy;
4905 } 4905 }
4906 4906
4907 if (uvm_fault_wire(map, start, end, 4907 if (uvm_fault_wire(map, start, end,
4908 entry->max_protection, 1)) { 4908 entry->max_protection, 1)) {
4909 /* wiring failed */ 4909 /* wiring failed */
4910 unlock_fn(map); 4910 unlock_fn(map);
4911 UVMHIST_LOG(maphist,"<- done (wire failed)", 4911 UVMHIST_LOG(maphist,"<- done (wire failed)",
4912 0,0,0,0); 4912 0,0,0,0);
4913 return false; 4913 return false;
4914 } 4914 }
4915 4915
4916 /* 4916 /*
4917 * now that we have resolved the fault, we can unwire 4917 * now that we have resolved the fault, we can unwire
4918 * the page. 4918 * the page.
4919 */ 4919 */
4920 if (exclusive) { 4920 if (exclusive) {
4921 vm_map_lock(map); 4921 vm_map_lock(map);
4922 vm_map_unbusy(map); 4922 vm_map_unbusy(map);
4923 unlock_fn = vm_map_unlock; 4923 unlock_fn = vm_map_unlock;
4924 } 4924 }
4925 4925
4926 uvm_fault_unwire_locked(map, start, end); 4926 uvm_fault_unwire_locked(map, start, end);
4927 } 4927 }
4928 } 4928 }
4929 4929
4930 /* check the upper layer */ 4930 /* check the upper layer */
4931 if (entry->aref.ar_amap) { 4931 if (entry->aref.ar_amap) {
4932 amap_lock(entry->aref.ar_amap, RW_WRITER); 4932 amap_lock(entry->aref.ar_amap, RW_WRITER);
4933 anon = amap_lookup(&entry->aref, start - entry->start); 4933 anon = amap_lookup(&entry->aref, start - entry->start);
4934 if (anon) { 4934 if (anon) {
4935 found_anon: KASSERT(anon->an_lock == entry->aref.ar_amap->am_lock); 4935 found_anon: KASSERT(anon->an_lock == entry->aref.ar_amap->am_lock);
4936 anon->an_ref++; 4936 anon->an_ref++;
4937 KASSERT(anon->an_ref != 0); 4937 KASSERT(anon->an_ref != 0);
4938 voaddr->type = UVM_VOADDR_TYPE_ANON; 4938 voaddr->type = UVM_VOADDR_TYPE_ANON;
4939 voaddr->anon = anon; 4939 voaddr->anon = anon;
4940 voaddr->offset = va & PAGE_MASK; 4940 voaddr->offset = va & PAGE_MASK;
4941 result = true; 4941 result = true;
4942 } 4942 }
4943 amap_unlock(entry->aref.ar_amap); 4943 amap_unlock(entry->aref.ar_amap);
4944 } 4944 }
4945 4945
4946 /* check the lower layer */ 4946 /* check the lower layer */
4947 if (!result && UVM_ET_ISOBJ(entry)) { 4947 if (!result && UVM_ET_ISOBJ(entry)) {
4948 struct uvm_object *uobj = entry->object.uvm_obj; 4948 struct uvm_object *uobj = entry->object.uvm_obj;
4949 4949
4950 KASSERT(uobj != NULL); 4950 KASSERT(uobj != NULL);
4951 (*uobj->pgops->pgo_reference)(uobj); 4951 (*uobj->pgops->pgo_reference)(uobj);
4952 voaddr->type = UVM_VOADDR_TYPE_OBJECT; 4952 voaddr->type = UVM_VOADDR_TYPE_OBJECT;
4953 voaddr->uobj = uobj; 4953 voaddr->uobj = uobj;
4954 voaddr->offset = entry->offset + (va - entry->start); 4954 voaddr->offset = entry->offset + (va - entry->start);
4955 result = true; 4955 result = true;
4956 } 4956 }
4957 4957
4958 unlock_fn(map); 4958 unlock_fn(map);
4959 4959
4960 if (result) { 4960 if (result) {
4961 UVMHIST_LOG(maphist, 4961 UVMHIST_LOG(maphist,
4962 "<- done OK (type=%jd,owner=#%jx,offset=%jx)", 4962 "<- done OK (type=%jd,owner=#%jx,offset=%jx)",
4963 voaddr->type, voaddr->uobj, voaddr->offset, 0); 4963 voaddr->type, voaddr->uobj, voaddr->offset, 0);
4964 } else { 4964 } else {
4965 UVMHIST_LOG(maphist,"<- done (failed)",0,0,0,0); 4965 UVMHIST_LOG(maphist,"<- done (failed)",0,0,0,0);
4966 } 4966 }
4967 4967
4968 return result; 4968 return result;
4969} 4969}
4970 4970
4971/* 4971/*
4972 * uvm_voaddr_release: release the references held by the 4972 * uvm_voaddr_release: release the references held by the
4973 * vitual object address. 4973 * vitual object address.
4974 */ 4974 */
4975void 4975void
4976uvm_voaddr_release(struct uvm_voaddr * const voaddr) 4976uvm_voaddr_release(struct uvm_voaddr * const voaddr)
4977{ 4977{
4978 4978
4979 switch (voaddr->type) { 4979 switch (voaddr->type) {
4980 case UVM_VOADDR_TYPE_OBJECT: { 4980 case UVM_VOADDR_TYPE_OBJECT: {
4981 struct uvm_object * const uobj = voaddr->uobj; 4981 struct uvm_object * const uobj = voaddr->uobj;
4982 4982
4983 KASSERT(uobj != NULL); 4983 KASSERT(uobj != NULL);
4984 KASSERT(uobj->pgops->pgo_detach != NULL); 4984 KASSERT(uobj->pgops->pgo_detach != NULL);
4985 (*uobj->pgops->pgo_detach)(uobj); 4985 (*uobj->pgops->pgo_detach)(uobj);
4986 break; 4986 break;
4987 } 4987 }
4988 case UVM_VOADDR_TYPE_ANON: { 4988 case UVM_VOADDR_TYPE_ANON: {
4989 struct vm_anon * const anon = voaddr->anon; 4989 struct vm_anon * const anon = voaddr->anon;
4990 4990
4991 KASSERT(anon != NULL); 4991 KASSERT(anon != NULL);
4992 rw_enter(anon->an_lock, RW_WRITER); 4992 rw_enter(anon->an_lock, RW_WRITER);
4993 KASSERT(anon->an_ref > 0); 4993 KASSERT(anon->an_ref > 0);
4994 anon->an_ref--; 4994 anon->an_ref--;
4995 if (anon->an_ref == 0) { 4995 if (anon->an_ref == 0) {
4996 uvm_anon_release(anon); 4996 uvm_anon_release(anon);
4997 } else { 4997 } else {
4998 rw_exit(anon->an_lock); 4998 rw_exit(anon->an_lock);
4999 } 4999 }
5000 break; 5000 break;
5001 } 5001 }
5002 default: 5002 default:
5003 panic("uvm_voaddr_release: bad type"); 5003 panic("uvm_voaddr_release: bad type");
5004 } 5004 }
5005 memset(voaddr, 0, sizeof(*voaddr)); 5005 memset(voaddr, 0, sizeof(*voaddr));
5006} 5006}
5007 5007
5008/* 5008/*
5009 * uvm_voaddr_compare: compare two uvm_voaddr objects. 5009 * uvm_voaddr_compare: compare two uvm_voaddr objects.
5010 * 5010 *
5011 * => memcmp() semantics 5011 * => memcmp() semantics
5012 */ 5012 */
5013int 5013int
5014uvm_voaddr_compare(const struct uvm_voaddr * const voaddr1, 5014uvm_voaddr_compare(const struct uvm_voaddr * const voaddr1,
5015 const struct uvm_voaddr * const voaddr2) 5015 const struct uvm_voaddr * const voaddr2)
5016{ 5016{
5017 5017
5018 KASSERT(voaddr1->type == UVM_VOADDR_TYPE_OBJECT || 5018 KASSERT(voaddr1->type == UVM_VOADDR_TYPE_OBJECT ||
5019 voaddr1->type == UVM_VOADDR_TYPE_ANON); 5019 voaddr1->type == UVM_VOADDR_TYPE_ANON);
5020 5020
5021 KASSERT(voaddr2->type == UVM_VOADDR_TYPE_OBJECT || 5021 KASSERT(voaddr2->type == UVM_VOADDR_TYPE_OBJECT ||
5022 voaddr2->type == UVM_VOADDR_TYPE_ANON); 5022 voaddr2->type == UVM_VOADDR_TYPE_ANON);
5023  5023
5024 if (voaddr1->type < voaddr2->type) 5024 if (voaddr1->type < voaddr2->type)
5025 return -1; 5025 return -1;
5026 if (voaddr1->type > voaddr2->type) 5026 if (voaddr1->type > voaddr2->type)
5027 return 1; 5027 return 1;
5028  5028
5029 /* These fields are unioned together. */ 5029 /* These fields are unioned together. */
5030 CTASSERT(offsetof(struct uvm_voaddr, uobj) == 5030 CTASSERT(offsetof(struct uvm_voaddr, uobj) ==
5031 offsetof(struct uvm_voaddr, anon)); 5031 offsetof(struct uvm_voaddr, anon));
5032 const uintptr_t addr1 = (uintptr_t)voaddr1->uobj; 5032 const uintptr_t addr1 = (uintptr_t)voaddr1->uobj;
5033 const uintptr_t addr2 = (uintptr_t)voaddr2->uobj; 5033 const uintptr_t addr2 = (uintptr_t)voaddr2->uobj;
5034 5034
5035 if (addr1 < addr2) 5035 if (addr1 < addr2)
5036 return -1; 5036 return -1;
5037 if (addr1 > addr2) 5037 if (addr1 > addr2)
5038 return 1; 5038 return 1;
5039  5039
5040 if (voaddr1->offset < voaddr2->offset) 5040 if (voaddr1->offset < voaddr2->offset)
5041 return -1; 5041 return -1;
5042 if (voaddr1->offset > voaddr2->offset) 5042 if (voaddr1->offset > voaddr2->offset)
5043 return 1; 5043 return 1;
5044  5044
5045 return 0; 5045 return 0;
5046} 5046}
5047 5047
5048#if defined(DDB) || defined(DEBUGPRINT) 5048#if defined(DDB) || defined(DEBUGPRINT)
5049 5049
5050/* 5050/*
5051 * uvm_map_printit: actually prints the map 5051 * uvm_map_printit: actually prints the map
5052 */ 5052 */
5053 5053
5054void 5054void
5055uvm_map_printit(struct vm_map *map, bool full, 5055uvm_map_printit(struct vm_map *map, bool full,
5056 void (*pr)(const char *, ...)) 5056 void (*pr)(const char *, ...))
5057{ 5057{
5058 struct vm_map_entry *entry; 5058 struct vm_map_entry *entry;
5059 5059
5060 (*pr)("MAP %p: [%#lx->%#lx]\n", map, vm_map_min(map), 5060 (*pr)("MAP %p: [%#lx->%#lx]\n", map, vm_map_min(map),
5061 vm_map_max(map)); 5061 vm_map_max(map));
5062 (*pr)("\t#ent=%d, sz=%d, ref=%d, version=%d, flags=%#x\n", 5062 (*pr)("\t#ent=%d, sz=%d, ref=%d, version=%d, flags=%#x\n",
5063 map->nentries, map->size, map->ref_count, map->timestamp, 5063 map->nentries, map->size, map->ref_count, map->timestamp,
5064 map->flags); 5064 map->flags);
5065 (*pr)("\tpmap=%p(resident=%ld, wired=%ld)\n", map->pmap, 5065 (*pr)("\tpmap=%p(resident=%ld, wired=%ld)\n", map->pmap,
5066 pmap_resident_count(map->pmap), pmap_wired_count(map->pmap)); 5066 pmap_resident_count(map->pmap), pmap_wired_count(map->pmap));
5067 if (!full) 5067 if (!full)
5068 return; 5068 return;
5069 for (entry = map->header.next; entry != &map->header; 5069 for (entry = map->header.next; entry != &map->header;
5070 entry = entry->next) { 5070 entry = entry->next) {
5071 (*pr)(" - %p: %#lx->%#lx: obj=%p/%#llx, amap=%p/%d\n", 5071 (*pr)(" - %p: %#lx->%#lx: obj=%p/%#llx, amap=%p/%d\n",
5072 entry, entry->start, entry->end, entry->object.uvm_obj, 5072 entry, entry->start, entry->end, entry->object.uvm_obj,
5073 (long long)entry->offset, entry->aref.ar_amap, 5073 (long long)entry->offset, entry->aref.ar_amap,
5074 entry->aref.ar_pageoff); 5074 entry->aref.ar_pageoff);
5075 (*pr)( 5075 (*pr)(
5076 "\tsubmap=%c, cow=%c, nc=%c, prot(max)=%d/%d, inh=%d, " 5076 "\tsubmap=%c, cow=%c, nc=%c, prot(max)=%d/%d, inh=%d, "
5077 "wc=%d, adv=%d\n", 5077 "wc=%d, adv=%d\n",
5078 (entry->etype & UVM_ET_SUBMAP) ? 'T' : 'F', 5078 (entry->etype & UVM_ET_SUBMAP) ? 'T' : 'F',
5079 (entry->etype & UVM_ET_COPYONWRITE) ? 'T' : 'F', 5079 (entry->etype & UVM_ET_COPYONWRITE) ? 'T' : 'F',
5080 (entry->etype & UVM_ET_NEEDSCOPY) ? 'T' : 'F', 5080 (entry->etype & UVM_ET_NEEDSCOPY) ? 'T' : 'F',
5081 entry->protection, entry->max_protection, 5081 entry->protection, entry->max_protection,
5082 entry->inheritance, entry->wired_count, entry->advice); 5082 entry->inheritance, entry->wired_count, entry->advice);
5083 } 5083 }
5084} 5084}
5085 5085
5086void 5086void
5087uvm_whatis(uintptr_t addr, void (*pr)(const char *, ...)) 5087uvm_whatis(uintptr_t addr, void (*pr)(const char *, ...))
5088{ 5088{
5089 struct vm_map *map; 5089 struct vm_map *map;
5090 5090
5091 for (map = kernel_map;;) { 5091 for (map = kernel_map;;) {
5092 struct vm_map_entry *entry; 5092 struct vm_map_entry *entry;
5093 5093
5094 if (!uvm_map_lookup_entry_bytree(map, (vaddr_t)addr, &entry)) { 5094 if (!uvm_map_lookup_entry_bytree(map, (vaddr_t)addr, &entry)) {
5095 break; 5095 break;
5096 } 5096 }
5097 (*pr)("%p is %p+%zu from VMMAP %p\n", 5097 (*pr)("%p is %p+%zu from VMMAP %p\n",
5098 (void *)addr, (void *)entry->start, 5098 (void *)addr, (void *)entry->start,
5099 (size_t)(addr - (uintptr_t)entry->start), map); 5099 (size_t)(addr - (uintptr_t)entry->start), map);
5100 if (!UVM_ET_ISSUBMAP(entry)) { 5100 if (!UVM_ET_ISSUBMAP(entry)) {
5101 break; 5101 break;
5102 } 5102 }
5103 map = entry->object.sub_map; 5103 map = entry->object.sub_map;
5104 } 5104 }
5105} 5105}
5106 5106
5107#endif /* DDB || DEBUGPRINT */ 5107#endif /* DDB || DEBUGPRINT */
5108 5108
5109#ifndef __USER_VA0_IS_SAFE 5109#ifndef __USER_VA0_IS_SAFE
5110static int 5110static int
5111sysctl_user_va0_disable(SYSCTLFN_ARGS) 5111sysctl_user_va0_disable(SYSCTLFN_ARGS)
5112{ 5112{
5113 struct sysctlnode node; 5113 struct sysctlnode node;
5114 int t, error; 5114 int t, error;
5115 5115
5116 node = *rnode; 5116 node = *rnode;
5117 node.sysctl_data = &t; 5117 node.sysctl_data = &t;
5118 t = user_va0_disable; 5118 t = user_va0_disable;
5119 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 5119 error = sysctl_lookup(SYSCTLFN_CALL(&node));
5120 if (error || newp == NULL) 5120 if (error || newp == NULL)
5121 return (error); 5121 return (error);
5122 5122
5123 if (!t && user_va0_disable && 5123 if (!t && user_va0_disable &&
5124 kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_MAP_VA_ZERO, 0, 5124 kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_MAP_VA_ZERO, 0,
5125 NULL, NULL, NULL)) 5125 NULL, NULL, NULL))
5126 return EPERM; 5126 return EPERM;
5127 5127
5128 user_va0_disable = !!t; 5128 user_va0_disable = !!t;
5129 return 0; 5129 return 0;
5130} 5130}
5131#endif 5131#endif
5132 5132
5133static int 5133static int
5134fill_vmentry(struct lwp *l, struct proc *p, struct kinfo_vmentry *kve, 5134fill_vmentry(struct lwp *l, struct proc *p, struct kinfo_vmentry *kve,
5135 struct vm_map *m, struct vm_map_entry *e) 5135 struct vm_map *m, struct vm_map_entry *e)
5136{ 5136{
5137#ifndef _RUMPKERNEL 5137#ifndef _RUMPKERNEL
5138 int error; 5138 int error;
5139 5139
5140 memset(kve, 0, sizeof(*kve)); 5140 memset(kve, 0, sizeof(*kve));
5141 KASSERT(e != NULL); 5141 KASSERT(e != NULL);
5142 if (UVM_ET_ISOBJ(e)) { 5142 if (UVM_ET_ISOBJ(e)) {
5143 struct uvm_object *uobj = e->object.uvm_obj; 5143 struct uvm_object *uobj = e->object.uvm_obj;
5144 KASSERT(uobj != NULL); 5144 KASSERT(uobj != NULL);
5145 kve->kve_ref_count = uobj->uo_refs; 5145 kve->kve_ref_count = uobj->uo_refs;
5146 kve->kve_count = uobj->uo_npages; 5146 kve->kve_count = uobj->uo_npages;
5147 if (UVM_OBJ_IS_VNODE(uobj)) { 5147 if (UVM_OBJ_IS_VNODE(uobj)) {
5148 struct vattr va; 5148 struct vattr va;
5149 struct vnode *vp = (struct vnode *)uobj; 5149 struct vnode *vp = (struct vnode *)uobj;
5150 vn_lock(vp, LK_SHARED | LK_RETRY); 5150 vn_lock(vp, LK_SHARED | LK_RETRY);
5151 error = VOP_GETATTR(vp, &va, l->l_cred); 5151 error = VOP_GETATTR(vp, &va, l->l_cred);
5152 VOP_UNLOCK(vp); 5152 VOP_UNLOCK(vp);
5153 kve->kve_type = KVME_TYPE_VNODE; 5153 kve->kve_type = KVME_TYPE_VNODE;
5154 if (error == 0) { 5154 if (error == 0) {
5155 kve->kve_vn_size = vp->v_size; 5155 kve->kve_vn_size = vp->v_size;
5156 kve->kve_vn_type = (int)vp->v_type; 5156 kve->kve_vn_type = (int)vp->v_type;
5157 kve->kve_vn_mode = va.va_mode; 5157 kve->kve_vn_mode = va.va_mode;
5158 kve->kve_vn_rdev = va.va_rdev; 5158 kve->kve_vn_rdev = va.va_rdev;
5159 kve->kve_vn_fileid = va.va_fileid; 5159 kve->kve_vn_fileid = va.va_fileid;
5160 kve->kve_vn_fsid = va.va_fsid; 5160 kve->kve_vn_fsid = va.va_fsid;
5161 error = vnode_to_path(kve->kve_path, 5161 error = vnode_to_path(kve->kve_path,
5162 sizeof(kve->kve_path) / 2, vp, l, p); 5162 sizeof(kve->kve_path) / 2, vp, l, p);
5163#ifdef DIAGNOSTIC 5163#ifdef DIAGNOSTIC
5164 if (error) 5164 if (error)
5165 printf("%s: vp %p error %d\n", __func__, 5165 printf("%s: vp %p error %d\n", __func__,
5166 vp, error); 5166 vp, error);
5167#endif 5167#endif
5168 } 5168 }
5169 } else if (UVM_OBJ_IS_KERN_OBJECT(uobj)) { 5169 } else if (UVM_OBJ_IS_KERN_OBJECT(uobj)) {
5170 kve->kve_type = KVME_TYPE_KERN; 5170 kve->kve_type = KVME_TYPE_KERN;
5171 } else if (UVM_OBJ_IS_DEVICE(uobj)) { 5171 } else if (UVM_OBJ_IS_DEVICE(uobj)) {
5172 kve->kve_type = KVME_TYPE_DEVICE; 5172 kve->kve_type = KVME_TYPE_DEVICE;
5173 } else if (UVM_OBJ_IS_AOBJ(uobj)) { 5173 } else if (UVM_OBJ_IS_AOBJ(uobj)) {
5174 kve->kve_type = KVME_TYPE_ANON; 5174 kve->kve_type = KVME_TYPE_ANON;
5175 } else { 5175 } else {
5176 kve->kve_type = KVME_TYPE_OBJECT; 5176 kve->kve_type = KVME_TYPE_OBJECT;
5177 } 5177 }
5178 } else if (UVM_ET_ISSUBMAP(e)) { 5178 } else if (UVM_ET_ISSUBMAP(e)) {
5179 struct vm_map *map = e->object.sub_map; 5179 struct vm_map *map = e->object.sub_map;
5180 KASSERT(map != NULL); 5180 KASSERT(map != NULL);
5181 kve->kve_ref_count = map->ref_count; 5181 kve->kve_ref_count = map->ref_count;
5182 kve->kve_count = map->nentries; 5182 kve->kve_count = map->nentries;
5183 kve->kve_type = KVME_TYPE_SUBMAP; 5183 kve->kve_type = KVME_TYPE_SUBMAP;
5184 } else 5184 } else
5185 kve->kve_type = KVME_TYPE_UNKNOWN; 5185 kve->kve_type = KVME_TYPE_UNKNOWN;
5186 5186
5187 kve->kve_start = e->start; 5187 kve->kve_start = e->start;
5188 kve->kve_end = e->end; 5188 kve->kve_end = e->end;
5189 kve->kve_offset = e->offset; 5189 kve->kve_offset = e->offset;
5190 kve->kve_wired_count = e->wired_count; 5190 kve->kve_wired_count = e->wired_count;
5191 kve->kve_inheritance = e->inheritance; 5191 kve->kve_inheritance = e->inheritance;
5192 kve->kve_attributes = 0; /* unused */ 5192 kve->kve_attributes = 0; /* unused */
5193 kve->kve_advice = e->advice; 5193 kve->kve_advice = e->advice;
5194#define PROT(p) (((p) & VM_PROT_READ) ? KVME_PROT_READ : 0) | \ 5194#define PROT(p) (((p) & VM_PROT_READ) ? KVME_PROT_READ : 0) | \
5195 (((p) & VM_PROT_WRITE) ? KVME_PROT_WRITE : 0) | \ 5195 (((p) & VM_PROT_WRITE) ? KVME_PROT_WRITE : 0) | \
5196 (((p) & VM_PROT_EXECUTE) ? KVME_PROT_EXEC : 0) 5196 (((p) & VM_PROT_EXECUTE) ? KVME_PROT_EXEC : 0)
5197 kve->kve_protection = PROT(e->protection); 5197 kve->kve_protection = PROT(e->protection);
5198 kve->kve_max_protection = PROT(e->max_protection); 5198 kve->kve_max_protection = PROT(e->max_protection);
5199 kve->kve_flags |= (e->etype & UVM_ET_COPYONWRITE) 5199 kve->kve_flags |= (e->etype & UVM_ET_COPYONWRITE)
5200 ? KVME_FLAG_COW : 0; 5200 ? KVME_FLAG_COW : 0;
5201 kve->kve_flags |= (e->etype & UVM_ET_NEEDSCOPY) 5201 kve->kve_flags |= (e->etype & UVM_ET_NEEDSCOPY)
5202 ? KVME_FLAG_NEEDS_COPY : 0; 5202 ? KVME_FLAG_NEEDS_COPY : 0;
5203 kve->kve_flags |= (m->flags & VM_MAP_TOPDOWN) 5203 kve->kve_flags |= (m->flags & VM_MAP_TOPDOWN)
5204 ? KVME_FLAG_GROWS_DOWN : KVME_FLAG_GROWS_UP; 5204 ? KVME_FLAG_GROWS_DOWN : KVME_FLAG_GROWS_UP;
5205 kve->kve_flags |= (m->flags & VM_MAP_PAGEABLE) 5205 kve->kve_flags |= (m->flags & VM_MAP_PAGEABLE)
5206 ? KVME_FLAG_PAGEABLE : 0; 5206 ? KVME_FLAG_PAGEABLE : 0;
5207#endif 5207#endif
5208 return 0; 5208 return 0;
5209} 5209}
5210 5210
5211static int 5211static int
5212fill_vmentries(struct lwp *l, pid_t pid, u_int elem_size, void *oldp, 5212fill_vmentries(struct lwp *l, pid_t pid, u_int elem_size, void *oldp,
5213 size_t *oldlenp) 5213 size_t *oldlenp)
5214{ 5214{
5215 int error; 5215 int error;
5216 struct proc *p; 5216 struct proc *p;
5217 struct kinfo_vmentry *vme; 5217 struct kinfo_vmentry *vme;
5218 struct vmspace *vm; 5218 struct vmspace *vm;
5219 struct vm_map *map; 5219 struct vm_map *map;
5220 struct vm_map_entry *entry; 5220 struct vm_map_entry *entry;
5221 char *dp; 5221 char *dp;
5222 size_t count, vmesize; 5222 size_t count, vmesize;
5223 5223
5224 if (elem_size == 0 || elem_size > 2 * sizeof(*vme)) 5224 if (elem_size == 0 || elem_size > 2 * sizeof(*vme))
5225 return EINVAL; 5225 return EINVAL;
5226 5226
5227 if (oldp) { 5227 if (oldp) {
5228 if (*oldlenp > 10UL * 1024UL * 1024UL) 5228 if (*oldlenp > 10UL * 1024UL * 1024UL)
5229 return E2BIG; 5229 return E2BIG;
5230 count = *oldlenp / elem_size; 5230 count = *oldlenp / elem_size;
5231 if (count == 0) 5231 if (count == 0)
5232 return ENOMEM; 5232 return ENOMEM;
5233 vmesize = count * sizeof(*vme); 5233 vmesize = count * sizeof(*vme);
5234 } else 5234 } else
5235 vmesize = 0; 5235 vmesize = 0;
5236 5236
5237 if ((error = proc_find_locked(l, &p, pid)) != 0) 5237 if ((error = proc_find_locked(l, &p, pid)) != 0)
5238 return error; 5238 return error;
5239 5239
5240 vme = NULL; 5240 vme = NULL;
5241 count = 0; 5241 count = 0;
5242 5242
5243 if ((error = proc_vmspace_getref(p, &vm)) != 0) 5243 if ((error = proc_vmspace_getref(p, &vm)) != 0)
5244 goto out; 5244 goto out;
5245 5245
5246 map = &vm->vm_map; 5246 map = &vm->vm_map;
5247 vm_map_lock_read(map); 5247 vm_map_lock_read(map);
5248 5248
5249 dp = oldp; 5249 dp = oldp;
5250 if (oldp) 5250 if (oldp)
5251 vme = kmem_alloc(vmesize, KM_SLEEP); 5251 vme = kmem_alloc(vmesize, KM_SLEEP);
5252 for (entry = map->header.next; entry != &map->header; 5252 for (entry = map->header.next; entry != &map->header;
5253 entry = entry->next) { 5253 entry = entry->next) {
5254 if (oldp && (dp - (char *)oldp) < vmesize) { 5254 if (oldp && (dp - (char *)oldp) < vmesize) {
5255 error = fill_vmentry(l, p, &vme[count], map, entry); 5255 error = fill_vmentry(l, p, &vme[count], map, entry);
5256 if (error) 5256 if (error)
5257 goto out; 5257 goto out;
5258 dp += elem_size; 5258 dp += elem_size;
5259 } 5259 }
5260 count++; 5260 count++;
5261 } 5261 }
5262 vm_map_unlock_read(map); 5262 vm_map_unlock_read(map);
5263 uvmspace_free(vm); 5263 uvmspace_free(vm);
5264 5264
5265out: 5265out:
5266 if (pid != -1) 5266 if (pid != -1)
5267 mutex_exit(p->p_lock); 5267 mutex_exit(p->p_lock);
5268 if (error == 0) { 5268 if (error == 0) {
5269 const u_int esize = uimin(sizeof(*vme), elem_size); 5269 const u_int esize = uimin(sizeof(*vme), elem_size);
5270 dp = oldp; 5270 dp = oldp;
5271 for (size_t i = 0; i < count; i++) { 5271 for (size_t i = 0; i < count; i++) {
5272 if (oldp && (dp - (char *)oldp) < vmesize) { 5272 if (oldp && (dp - (char *)oldp) < vmesize) {
5273 error = sysctl_copyout(l, &vme[i], dp, esize); 5273 error = sysctl_copyout(l, &vme[i], dp, esize);
5274 if (error) 5274 if (error)
5275 break; 5275 break;
5276 dp += elem_size; 5276 dp += elem_size;
5277 } else 5277 } else
5278 break; 5278 break;
5279 } 5279 }
5280 count *= elem_size; 5280 count *= elem_size;
5281 if (oldp != NULL && *oldlenp < count) 5281 if (oldp != NULL && *oldlenp < count)
5282 error = ENOSPC; 5282 error = ENOSPC;
5283 *oldlenp = count; 5283 *oldlenp = count;
5284 } 5284 }
5285 if (vme) 5285 if (vme)
5286 kmem_free(vme, vmesize); 5286 kmem_free(vme, vmesize);
5287 return error; 5287 return error;
5288} 5288}
5289 5289
5290static int 5290static int
5291sysctl_vmproc(SYSCTLFN_ARGS) 5291sysctl_vmproc(SYSCTLFN_ARGS)
5292{ 5292{
5293 int error; 5293 int error;
5294 5294
5295 if (namelen == 1 && name[0] == CTL_QUERY) 5295 if (namelen == 1 && name[0] == CTL_QUERY)
5296 return (sysctl_query(SYSCTLFN_CALL(rnode))); 5296 return (sysctl_query(SYSCTLFN_CALL(rnode)));
5297 5297
5298 if (namelen == 0) 5298 if (namelen == 0)
5299 return EINVAL; 5299 return EINVAL;
5300 5300
5301 switch (name[0]) { 5301 switch (name[0]) {
5302 case VM_PROC_MAP: 5302 case VM_PROC_MAP:
5303 if (namelen != 3) 5303 if (namelen != 3)
5304 return EINVAL; 5304 return EINVAL;
5305 sysctl_unlock(); 5305 sysctl_unlock();
5306 error = fill_vmentries(l, name[1], name[2], oldp, oldlenp); 5306 error = fill_vmentries(l, name[1], name[2], oldp, oldlenp);
5307 sysctl_relock(); 5307 sysctl_relock();
5308 return error; 5308 return error;
5309 default: 5309 default:
5310 return EINVAL; 5310 return EINVAL;
5311 } 5311 }
5312} 5312}
5313 5313
5314SYSCTL_SETUP(sysctl_uvmmap_setup, "sysctl uvmmap setup") 5314SYSCTL_SETUP(sysctl_uvmmap_setup, "sysctl uvmmap setup")
5315{ 5315{
5316 5316
5317 sysctl_createv(clog, 0, NULL, NULL, 5317 sysctl_createv(clog, 0, NULL, NULL,
5318 CTLFLAG_PERMANENT, 5318 CTLFLAG_PERMANENT,
5319 CTLTYPE_STRUCT, "proc", 5319 CTLTYPE_STRUCT, "proc",
5320 SYSCTL_DESCR("Process vm information"), 5320 SYSCTL_DESCR("Process vm information"),
5321 sysctl_vmproc, 0, NULL, 0, 5321 sysctl_vmproc, 0, NULL, 0,
5322 CTL_VM, VM_PROC, CTL_EOL); 5322 CTL_VM, VM_PROC, CTL_EOL);
5323#ifndef __USER_VA0_IS_SAFE 5323#ifndef __USER_VA0_IS_SAFE
5324 sysctl_createv(clog, 0, NULL, NULL, 5324 sysctl_createv(clog, 0, NULL, NULL,
5325 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 5325 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
5326 CTLTYPE_INT, "user_va0_disable", 5326 CTLTYPE_INT, "user_va0_disable",
5327 SYSCTL_DESCR("Disable VA 0"), 5327 SYSCTL_DESCR("Disable VA 0"),
5328 sysctl_user_va0_disable, 0, &user_va0_disable, 0, 5328 sysctl_user_va0_disable, 0, &user_va0_disable, 0,
5329 CTL_VM, CTL_CREATE, CTL_EOL); 5329 CTL_VM, CTL_CREATE, CTL_EOL);
5330#endif 5330#endif
5331} 5331}