| @@ -1,825 +1,827 @@ | | | @@ -1,825 +1,827 @@ |
1 | /* $NetBSD: rump.c,v 1.336 2019/12/03 05:07:49 riastradh Exp $ */ | | 1 | /* $NetBSD: rump.c,v 1.337 2019/12/07 14:55:58 riastradh Exp $ */ |
2 | | | 2 | |
3 | /* | | 3 | /* |
4 | * Copyright (c) 2007-2011 Antti Kantee. All Rights Reserved. | | 4 | * Copyright (c) 2007-2011 Antti Kantee. All Rights Reserved. |
5 | * | | 5 | * |
6 | * Redistribution and use in source and binary forms, with or without | | 6 | * Redistribution and use in source and binary forms, with or without |
7 | * modification, are permitted provided that the following conditions | | 7 | * modification, are permitted provided that the following conditions |
8 | * are met: | | 8 | * are met: |
9 | * 1. Redistributions of source code must retain the above copyright | | 9 | * 1. Redistributions of source code must retain the above copyright |
10 | * notice, this list of conditions and the following disclaimer. | | 10 | * notice, this list of conditions and the following disclaimer. |
11 | * 2. Redistributions in binary form must reproduce the above copyright | | 11 | * 2. Redistributions in binary form must reproduce the above copyright |
12 | * notice, this list of conditions and the following disclaimer in the | | 12 | * notice, this list of conditions and the following disclaimer in the |
13 | * documentation and/or other materials provided with the distribution. | | 13 | * documentation and/or other materials provided with the distribution. |
14 | * | | 14 | * |
15 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS | | 15 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS |
16 | * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED | | 16 | * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
17 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE | | 17 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
18 | * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE | | 18 | * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE |
19 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | | 19 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
20 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR | | 20 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | | 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
22 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | | 22 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
23 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | | 23 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
24 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | | 24 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
25 | * SUCH DAMAGE. | | 25 | * SUCH DAMAGE. |
26 | */ | | 26 | */ |
27 | | | 27 | |
28 | #include <sys/cdefs.h> | | 28 | #include <sys/cdefs.h> |
29 | __KERNEL_RCSID(0, "$NetBSD: rump.c,v 1.336 2019/12/03 05:07:49 riastradh Exp $"); | | 29 | __KERNEL_RCSID(0, "$NetBSD: rump.c,v 1.337 2019/12/07 14:55:58 riastradh Exp $"); |
30 | | | 30 | |
31 | #include <sys/systm.h> | | 31 | #include <sys/systm.h> |
32 | #define ELFSIZE ARCH_ELFSIZE | | 32 | #define ELFSIZE ARCH_ELFSIZE |
33 | | | 33 | |
34 | #include <sys/param.h> | | 34 | #include <sys/param.h> |
35 | #include <sys/atomic.h> | | 35 | #include <sys/atomic.h> |
36 | #include <sys/buf.h> | | 36 | #include <sys/buf.h> |
37 | #include <sys/callout.h> | | 37 | #include <sys/callout.h> |
38 | #include <sys/conf.h> | | 38 | #include <sys/conf.h> |
39 | #include <sys/cpu.h> | | 39 | #include <sys/cpu.h> |
40 | #include <sys/device.h> | | 40 | #include <sys/device.h> |
41 | #include <sys/evcnt.h> | | 41 | #include <sys/evcnt.h> |
42 | #include <sys/event.h> | | 42 | #include <sys/event.h> |
43 | #include <sys/exec_elf.h> | | 43 | #include <sys/exec_elf.h> |
44 | #include <sys/filedesc.h> | | 44 | #include <sys/filedesc.h> |
45 | #include <sys/iostat.h> | | 45 | #include <sys/iostat.h> |
46 | #include <sys/kauth.h> | | 46 | #include <sys/kauth.h> |
47 | #include <sys/kcpuset.h> | | 47 | #include <sys/kcpuset.h> |
48 | #include <sys/kernel.h> | | 48 | #include <sys/kernel.h> |
49 | #include <sys/kmem.h> | | 49 | #include <sys/kmem.h> |
50 | #include <sys/kprintf.h> | | 50 | #include <sys/kprintf.h> |
51 | #include <sys/kthread.h> | | 51 | #include <sys/kthread.h> |
52 | #include <sys/ksyms.h> | | 52 | #include <sys/ksyms.h> |
53 | #include <sys/msgbuf.h> | | 53 | #include <sys/msgbuf.h> |
54 | #include <sys/module.h> | | 54 | #include <sys/module.h> |
55 | #include <sys/namei.h> | | 55 | #include <sys/namei.h> |
56 | #include <sys/once.h> | | 56 | #include <sys/once.h> |
57 | #include <sys/percpu.h> | | 57 | #include <sys/percpu.h> |
58 | #include <sys/pipe.h> | | 58 | #include <sys/pipe.h> |
59 | #include <sys/pool.h> | | 59 | #include <sys/pool.h> |
60 | #include <sys/queue.h> | | 60 | #include <sys/queue.h> |
61 | #include <sys/reboot.h> | | 61 | #include <sys/reboot.h> |
62 | #include <sys/resourcevar.h> | | 62 | #include <sys/resourcevar.h> |
63 | #include <sys/select.h> | | 63 | #include <sys/select.h> |
64 | #include <sys/sysctl.h> | | 64 | #include <sys/sysctl.h> |
65 | #include <sys/syscall.h> | | 65 | #include <sys/syscall.h> |
66 | #include <sys/syscallvar.h> | | 66 | #include <sys/syscallvar.h> |
67 | #include <sys/threadpool.h> | | 67 | #include <sys/threadpool.h> |
68 | #include <sys/timetc.h> | | 68 | #include <sys/timetc.h> |
69 | #include <sys/tty.h> | | 69 | #include <sys/tty.h> |
70 | #include <sys/uidinfo.h> | | 70 | #include <sys/uidinfo.h> |
71 | #include <sys/vmem.h> | | 71 | #include <sys/vmem.h> |
72 | #include <sys/xcall.h> | | 72 | #include <sys/xcall.h> |
73 | #include <sys/cprng.h> | | 73 | #include <sys/cprng.h> |
74 | #include <sys/rnd.h> | | 74 | #include <sys/rnd.h> |
75 | #include <sys/ktrace.h> | | 75 | #include <sys/ktrace.h> |
| | | 76 | #include <sys/pserialize.h> |
76 | #include <sys/psref.h> | | 77 | #include <sys/psref.h> |
77 | | | 78 | |
78 | #include <rump-sys/kern.h> | | 79 | #include <rump-sys/kern.h> |
79 | #include <rump-sys/dev.h> | | 80 | #include <rump-sys/dev.h> |
80 | #include <rump-sys/net.h> | | 81 | #include <rump-sys/net.h> |
81 | #include <rump-sys/vfs.h> | | 82 | #include <rump-sys/vfs.h> |
82 | | | 83 | |
83 | #include <rump/rumpuser.h> | | 84 | #include <rump/rumpuser.h> |
84 | | | 85 | |
85 | #include <secmodel/suser/suser.h> | | 86 | #include <secmodel/suser/suser.h> |
86 | | | 87 | |
87 | #include <prop/proplib.h> | | 88 | #include <prop/proplib.h> |
88 | | | 89 | |
89 | #include <uvm/uvm_extern.h> | | 90 | #include <uvm/uvm_extern.h> |
90 | #include <uvm/uvm_readahead.h> | | 91 | #include <uvm/uvm_readahead.h> |
91 | | | 92 | |
92 | char machine[] = MACHINE; | | 93 | char machine[] = MACHINE; |
93 | char machine_arch[] = MACHINE_ARCH; | | 94 | char machine_arch[] = MACHINE_ARCH; |
94 | | | 95 | |
95 | struct proc *initproc; | | 96 | struct proc *initproc; |
96 | | | 97 | |
97 | struct device rump_rootdev = { | | 98 | struct device rump_rootdev = { |
98 | .dv_class = DV_VIRTUAL | | 99 | .dv_class = DV_VIRTUAL |
99 | }; | | 100 | }; |
100 | | | 101 | |
101 | #ifdef RUMP_WITHOUT_THREADS | | 102 | #ifdef RUMP_WITHOUT_THREADS |
102 | int rump_threads = 0; | | 103 | int rump_threads = 0; |
103 | #else | | 104 | #else |
104 | int rump_threads = 1; | | 105 | int rump_threads = 1; |
105 | #endif | | 106 | #endif |
106 | | | 107 | |
107 | static void rump_component_addlocal(void); | | 108 | static void rump_component_addlocal(void); |
108 | static struct lwp *bootlwp; | | 109 | static struct lwp *bootlwp; |
109 | | | 110 | |
110 | /* 16k should be enough for std rump needs */ | | 111 | /* 16k should be enough for std rump needs */ |
111 | static char rump_msgbuf[16*1024] __aligned(256); | | 112 | static char rump_msgbuf[16*1024] __aligned(256); |
112 | | | 113 | |
113 | bool rump_ttycomponent = false; | | 114 | bool rump_ttycomponent = false; |
114 | | | 115 | |
115 | pool_cache_t pnbuf_cache; | | 116 | pool_cache_t pnbuf_cache; |
116 | | | 117 | |
117 | static void | | 118 | static void |
118 | rump_aiodone_worker(struct work *wk, void *dummy) | | 119 | rump_aiodone_worker(struct work *wk, void *dummy) |
119 | { | | 120 | { |
120 | struct buf *bp = (struct buf *)wk; | | 121 | struct buf *bp = (struct buf *)wk; |
121 | | | 122 | |
122 | KASSERT(&bp->b_work == wk); | | 123 | KASSERT(&bp->b_work == wk); |
123 | bp->b_iodone(bp); | | 124 | bp->b_iodone(bp); |
124 | } | | 125 | } |
125 | | | 126 | |
126 | static int rump_inited; | | 127 | static int rump_inited; |
127 | | | 128 | |
128 | void (*rump_vfs_drainbufs)(int) = (void *)nullop; | | 129 | void (*rump_vfs_drainbufs)(int) = (void *)nullop; |
129 | int (*rump_vfs_makeonedevnode)(dev_t, const char *, | | 130 | int (*rump_vfs_makeonedevnode)(dev_t, const char *, |
130 | devmajor_t, devminor_t) = (void *)nullop; | | 131 | devmajor_t, devminor_t) = (void *)nullop; |
131 | int (*rump_vfs_makedevnodes)(dev_t, const char *, char, | | 132 | int (*rump_vfs_makedevnodes)(dev_t, const char *, char, |
132 | devmajor_t, devminor_t, int) = (void *)nullop; | | 133 | devmajor_t, devminor_t, int) = (void *)nullop; |
133 | int (*rump_vfs_makesymlink)(const char *, const char *) = (void *)nullop; | | 134 | int (*rump_vfs_makesymlink)(const char *, const char *) = (void *)nullop; |
134 | | | 135 | |
135 | rump_proc_vfs_init_fn rump_proc_vfs_init = (void *)nullop; | | 136 | rump_proc_vfs_init_fn rump_proc_vfs_init = (void *)nullop; |
136 | rump_proc_vfs_release_fn rump_proc_vfs_release = (void *)nullop; | | 137 | rump_proc_vfs_release_fn rump_proc_vfs_release = (void *)nullop; |
137 | | | 138 | |
138 | static void add_linkedin_modules(const struct modinfo *const *, size_t); | | 139 | static void add_linkedin_modules(const struct modinfo *const *, size_t); |
139 | | | 140 | |
140 | static pid_t rspo_wrap_getpid(void) { | | 141 | static pid_t rspo_wrap_getpid(void) { |
141 | return rump_sysproxy_hyp_getpid(); | | 142 | return rump_sysproxy_hyp_getpid(); |
142 | } | | 143 | } |
143 | static int rspo_wrap_syscall(int num, void *arg, long *retval) { | | 144 | static int rspo_wrap_syscall(int num, void *arg, long *retval) { |
144 | return rump_sysproxy_hyp_syscall(num, arg, retval); | | 145 | return rump_sysproxy_hyp_syscall(num, arg, retval); |
145 | } | | 146 | } |
146 | static int rspo_wrap_rfork(void *priv, int flag, const char *comm) { | | 147 | static int rspo_wrap_rfork(void *priv, int flag, const char *comm) { |
147 | return rump_sysproxy_hyp_rfork(priv, flag, comm); | | 148 | return rump_sysproxy_hyp_rfork(priv, flag, comm); |
148 | } | | 149 | } |
149 | static void rspo_wrap_lwpexit(void) { | | 150 | static void rspo_wrap_lwpexit(void) { |
150 | rump_sysproxy_hyp_lwpexit(); | | 151 | rump_sysproxy_hyp_lwpexit(); |
151 | } | | 152 | } |
152 | static void rspo_wrap_execnotify(const char *comm) { | | 153 | static void rspo_wrap_execnotify(const char *comm) { |
153 | rump_sysproxy_hyp_execnotify(comm); | | 154 | rump_sysproxy_hyp_execnotify(comm); |
154 | } | | 155 | } |
155 | static const struct rumpuser_hyperup hyp = { | | 156 | static const struct rumpuser_hyperup hyp = { |
156 | .hyp_schedule = rump_schedule, | | 157 | .hyp_schedule = rump_schedule, |
157 | .hyp_unschedule = rump_unschedule, | | 158 | .hyp_unschedule = rump_unschedule, |
158 | .hyp_backend_unschedule = rump_user_unschedule, | | 159 | .hyp_backend_unschedule = rump_user_unschedule, |
159 | .hyp_backend_schedule = rump_user_schedule, | | 160 | .hyp_backend_schedule = rump_user_schedule, |
160 | .hyp_lwproc_switch = rump_lwproc_switch, | | 161 | .hyp_lwproc_switch = rump_lwproc_switch, |
161 | .hyp_lwproc_release = rump_lwproc_releaselwp, | | 162 | .hyp_lwproc_release = rump_lwproc_releaselwp, |
162 | .hyp_lwproc_newlwp = rump_lwproc_newlwp, | | 163 | .hyp_lwproc_newlwp = rump_lwproc_newlwp, |
163 | .hyp_lwproc_curlwp = rump_lwproc_curlwp, | | 164 | .hyp_lwproc_curlwp = rump_lwproc_curlwp, |
164 | | | 165 | |
165 | .hyp_getpid = rspo_wrap_getpid, | | 166 | .hyp_getpid = rspo_wrap_getpid, |
166 | .hyp_syscall = rspo_wrap_syscall, | | 167 | .hyp_syscall = rspo_wrap_syscall, |
167 | .hyp_lwproc_rfork = rspo_wrap_rfork, | | 168 | .hyp_lwproc_rfork = rspo_wrap_rfork, |
168 | .hyp_lwpexit = rspo_wrap_lwpexit, | | 169 | .hyp_lwpexit = rspo_wrap_lwpexit, |
169 | .hyp_execnotify = rspo_wrap_execnotify, | | 170 | .hyp_execnotify = rspo_wrap_execnotify, |
170 | }; | | 171 | }; |
171 | struct rump_sysproxy_ops rump_sysproxy_ops = { | | 172 | struct rump_sysproxy_ops rump_sysproxy_ops = { |
172 | .rspo_copyin = (void *)enxio, | | 173 | .rspo_copyin = (void *)enxio, |
173 | .rspo_copyinstr = (void *)enxio, | | 174 | .rspo_copyinstr = (void *)enxio, |
174 | .rspo_copyout = (void *)enxio, | | 175 | .rspo_copyout = (void *)enxio, |
175 | .rspo_copyoutstr = (void *)enxio, | | 176 | .rspo_copyoutstr = (void *)enxio, |
176 | .rspo_anonmmap = (void *)enxio, | | 177 | .rspo_anonmmap = (void *)enxio, |
177 | .rspo_raise = (void *)enxio, | | 178 | .rspo_raise = (void *)enxio, |
178 | .rspo_fini = (void *)enxio, | | 179 | .rspo_fini = (void *)enxio, |
179 | .rspo_hyp_getpid = (void *)enxio, | | 180 | .rspo_hyp_getpid = (void *)enxio, |
180 | .rspo_hyp_syscall = (void *)enxio, | | 181 | .rspo_hyp_syscall = (void *)enxio, |
181 | .rspo_hyp_rfork = (void *)enxio, | | 182 | .rspo_hyp_rfork = (void *)enxio, |
182 | .rspo_hyp_lwpexit = (void *)enxio, | | 183 | .rspo_hyp_lwpexit = (void *)enxio, |
183 | .rspo_hyp_execnotify = (void *)enxio, | | 184 | .rspo_hyp_execnotify = (void *)enxio, |
184 | }; | | 185 | }; |
185 | | | 186 | |
186 | int | | 187 | int |
187 | rump_daemonize_begin(void) | | 188 | rump_daemonize_begin(void) |
188 | { | | 189 | { |
189 | | | 190 | |
190 | if (rump_inited) | | 191 | if (rump_inited) |
191 | return EALREADY; | | 192 | return EALREADY; |
192 | | | 193 | |
193 | return rumpuser_daemonize_begin(); | | 194 | return rumpuser_daemonize_begin(); |
194 | } | | 195 | } |
195 | | | 196 | |
196 | int | | 197 | int |
197 | rump_daemonize_done(int error) | | 198 | rump_daemonize_done(int error) |
198 | { | | 199 | { |
199 | | | 200 | |
200 | return rumpuser_daemonize_done(error); | | 201 | return rumpuser_daemonize_done(error); |
201 | } | | 202 | } |
202 | | | 203 | |
203 | #ifdef RUMP_USE_CTOR | | 204 | #ifdef RUMP_USE_CTOR |
204 | | | 205 | |
205 | /* sysctl bootstrap handling */ | | 206 | /* sysctl bootstrap handling */ |
206 | struct sysctl_boot_chain sysctl_boot_chain \ | | 207 | struct sysctl_boot_chain sysctl_boot_chain \ |
207 | = LIST_HEAD_INITIALIZER(sysctl_boot_chain); | | 208 | = LIST_HEAD_INITIALIZER(sysctl_boot_chain); |
208 | __link_set_add_text(sysctl_funcs,voidop); /* ensure linkset is non-empty */ | | 209 | __link_set_add_text(sysctl_funcs,voidop); /* ensure linkset is non-empty */ |
209 | | | 210 | |
210 | #else /* RUMP_USE_CTOR */ | | 211 | #else /* RUMP_USE_CTOR */ |
211 | | | 212 | |
212 | RUMP_COMPONENT(RUMP_COMPONENT_POSTINIT) | | 213 | RUMP_COMPONENT(RUMP_COMPONENT_POSTINIT) |
213 | { | | 214 | { |
214 | __link_set_decl(rump_components, struct rump_component); | | 215 | __link_set_decl(rump_components, struct rump_component); |
215 | | | 216 | |
216 | /* | | 217 | /* |
217 | * Trick compiler into generating references so that statically | | 218 | * Trick compiler into generating references so that statically |
218 | * linked rump kernels are generated with the link set symbols. | | 219 | * linked rump kernels are generated with the link set symbols. |
219 | */ | | 220 | */ |
220 | asm("" :: "r"(__start_link_set_rump_components)); | | 221 | asm("" :: "r"(__start_link_set_rump_components)); |
221 | asm("" :: "r"(__stop_link_set_rump_components)); | | 222 | asm("" :: "r"(__stop_link_set_rump_components)); |
222 | } | | 223 | } |
223 | | | 224 | |
224 | #endif /* RUMP_USE_CTOR */ | | 225 | #endif /* RUMP_USE_CTOR */ |
225 | | | 226 | |
226 | int | | 227 | int |
227 | rump_init(void) | | 228 | rump_init(void) |
228 | { | | 229 | { |
229 | char buf[256]; | | 230 | char buf[256]; |
230 | struct timespec ts; | | 231 | struct timespec ts; |
231 | int64_t sec; | | 232 | int64_t sec; |
232 | long nsec; | | 233 | long nsec; |
233 | struct lwp *l, *initlwp; | | 234 | struct lwp *l, *initlwp; |
234 | int i, numcpu; | | 235 | int i, numcpu; |
235 | | | 236 | |
236 | /* not reentrant */ | | 237 | /* not reentrant */ |
237 | if (rump_inited) | | 238 | if (rump_inited) |
238 | return 0; | | 239 | return 0; |
239 | else if (rump_inited == -1) | | 240 | else if (rump_inited == -1) |
240 | panic("rump_init: host process restart required"); | | 241 | panic("rump_init: host process restart required"); |
241 | else | | 242 | else |
242 | rump_inited = 1; | | 243 | rump_inited = 1; |
243 | | | 244 | |
244 | /* initialize hypervisor */ | | 245 | /* initialize hypervisor */ |
245 | if (rumpuser_init(RUMPUSER_VERSION, &hyp) != 0) { | | 246 | if (rumpuser_init(RUMPUSER_VERSION, &hyp) != 0) { |
246 | rumpuser_dprintf("rumpuser init failed\n"); | | 247 | rumpuser_dprintf("rumpuser init failed\n"); |
247 | return EINVAL; | | 248 | return EINVAL; |
248 | } | | 249 | } |
249 | | | 250 | |
250 | /* init minimal lwp/cpu context */ | | 251 | /* init minimal lwp/cpu context */ |
251 | rump_lwproc_init(); | | 252 | rump_lwproc_init(); |
252 | l = &lwp0; | | 253 | l = &lwp0; |
253 | l->l_cpu = l->l_target_cpu = &rump_bootcpu; | | 254 | l->l_cpu = l->l_target_cpu = &rump_bootcpu; |
254 | rump_lwproc_curlwp_set(l); | | 255 | rump_lwproc_curlwp_set(l); |
255 | | | 256 | |
256 | /* retrieve env vars which affect the early stage of bootstrap */ | | 257 | /* retrieve env vars which affect the early stage of bootstrap */ |
257 | if (rumpuser_getparam("RUMP_THREADS", buf, sizeof(buf)) == 0) { | | 258 | if (rumpuser_getparam("RUMP_THREADS", buf, sizeof(buf)) == 0) { |
258 | rump_threads = *buf != '0'; | | 259 | rump_threads = *buf != '0'; |
259 | } | | 260 | } |
260 | if (rumpuser_getparam("RUMP_VERBOSE", buf, sizeof(buf)) == 0) { | | 261 | if (rumpuser_getparam("RUMP_VERBOSE", buf, sizeof(buf)) == 0) { |
261 | if (*buf != '0') | | 262 | if (*buf != '0') |
262 | boothowto = AB_VERBOSE; | | 263 | boothowto = AB_VERBOSE; |
263 | } | | 264 | } |
264 | | | 265 | |
265 | if (rumpuser_getparam(RUMPUSER_PARAM_NCPU, buf, sizeof(buf)) != 0) | | 266 | if (rumpuser_getparam(RUMPUSER_PARAM_NCPU, buf, sizeof(buf)) != 0) |
266 | panic("mandatory hypervisor configuration (NCPU) missing"); | | 267 | panic("mandatory hypervisor configuration (NCPU) missing"); |
267 | numcpu = strtoll(buf, NULL, 10); | | 268 | numcpu = strtoll(buf, NULL, 10); |
268 | if (numcpu < 1) { | | 269 | if (numcpu < 1) { |
269 | panic("rump kernels are not lightweight enough for \"%d\" CPUs", | | 270 | panic("rump kernels are not lightweight enough for \"%d\" CPUs", |
270 | numcpu); | | 271 | numcpu); |
271 | } | | 272 | } |
272 | | | 273 | |
273 | rump_thread_init(); | | 274 | rump_thread_init(); |
274 | rump_cpus_bootstrap(&numcpu); | | 275 | rump_cpus_bootstrap(&numcpu); |
275 | | | 276 | |
276 | rumpuser_clock_gettime(RUMPUSER_CLOCK_RELWALL, &sec, &nsec); | | 277 | rumpuser_clock_gettime(RUMPUSER_CLOCK_RELWALL, &sec, &nsec); |
277 | boottime.tv_sec = sec; | | 278 | boottime.tv_sec = sec; |
278 | boottime.tv_nsec = nsec; | | 279 | boottime.tv_nsec = nsec; |
279 | | | 280 | |
280 | initmsgbuf(rump_msgbuf, sizeof(rump_msgbuf)); | | 281 | initmsgbuf(rump_msgbuf, sizeof(rump_msgbuf)); |
281 | aprint_verbose("%s%s", copyright, version); | | 282 | aprint_verbose("%s%s", copyright, version); |
282 | | | 283 | |
283 | rump_intr_init(numcpu); | | 284 | rump_intr_init(numcpu); |
284 | | | 285 | |
285 | rump_tsleep_init(); | | 286 | rump_tsleep_init(); |
286 | | | 287 | |
287 | rumpuser_mutex_init(&rump_giantlock, RUMPUSER_MTX_SPIN); | | 288 | rumpuser_mutex_init(&rump_giantlock, RUMPUSER_MTX_SPIN); |
288 | ksyms_init(); | | 289 | ksyms_init(); |
289 | uvm_init(); | | 290 | uvm_init(); |
290 | evcnt_init(); | | 291 | evcnt_init(); |
291 | | | 292 | |
292 | kcpuset_sysinit(); | | 293 | kcpuset_sysinit(); |
293 | once_init(); | | 294 | once_init(); |
294 | kernconfig_lock_init(); | | 295 | kernconfig_lock_init(); |
295 | prop_kern_init(); | | 296 | prop_kern_init(); |
296 | | | 297 | |
297 | kmem_init(); | | 298 | kmem_init(); |
298 | | | 299 | |
299 | uvm_ra_init(); | | 300 | uvm_ra_init(); |
300 | uao_init(); | | 301 | uao_init(); |
301 | | | 302 | |
302 | mutex_obj_init(); | | 303 | mutex_obj_init(); |
303 | rw_obj_init(); | | 304 | rw_obj_init(); |
304 | callout_startup(); | | 305 | callout_startup(); |
305 | | | 306 | |
306 | kprintf_init(); | | 307 | kprintf_init(); |
307 | percpu_init(); | | 308 | percpu_init(); |
| | | 309 | pserialize_init(); |
308 | | | 310 | |
309 | kauth_init(); | | 311 | kauth_init(); |
310 | | | 312 | |
311 | secmodel_init(); | | 313 | secmodel_init(); |
312 | sysctl_init(); | | 314 | sysctl_init(); |
313 | /* | | 315 | /* |
314 | * The above call to sysctl_init() only initializes sysctl nodes | | 316 | * The above call to sysctl_init() only initializes sysctl nodes |
315 | * from link sets. Initialize sysctls in case we used ctors. | | 317 | * from link sets. Initialize sysctls in case we used ctors. |
316 | */ | | 318 | */ |
317 | #ifdef RUMP_USE_CTOR | | 319 | #ifdef RUMP_USE_CTOR |
318 | { | | 320 | { |
319 | struct sysctl_setup_chain *ssc; | | 321 | struct sysctl_setup_chain *ssc; |
320 | | | 322 | |
321 | while ((ssc = LIST_FIRST(&sysctl_boot_chain)) != NULL) { | | 323 | while ((ssc = LIST_FIRST(&sysctl_boot_chain)) != NULL) { |
322 | LIST_REMOVE(ssc, ssc_entries); | | 324 | LIST_REMOVE(ssc, ssc_entries); |
323 | ssc->ssc_func(NULL); | | 325 | ssc->ssc_func(NULL); |
324 | } | | 326 | } |
325 | } | | 327 | } |
326 | #endif /* RUMP_USE_CTOR */ | | 328 | #endif /* RUMP_USE_CTOR */ |
327 | | | 329 | |
328 | rnd_init(); | | 330 | rnd_init(); |
329 | cprng_init(); | | 331 | cprng_init(); |
330 | kern_cprng = cprng_strong_create("kernel", IPL_VM, | | 332 | kern_cprng = cprng_strong_create("kernel", IPL_VM, |
331 | CPRNG_INIT_ANY|CPRNG_REKEY_ANY); | | 333 | CPRNG_INIT_ANY|CPRNG_REKEY_ANY); |
332 | | | 334 | |
333 | rump_hyperentropy_init(); | | 335 | rump_hyperentropy_init(); |
334 | | | 336 | |
335 | procinit(); | | 337 | procinit(); |
336 | proc0_init(); | | 338 | proc0_init(); |
337 | uid_init(); | | 339 | uid_init(); |
338 | chgproccnt(0, 1); | | 340 | chgproccnt(0, 1); |
339 | | | 341 | |
340 | l->l_proc = &proc0; | | 342 | l->l_proc = &proc0; |
341 | lwp_update_creds(l); | | 343 | lwp_update_creds(l); |
342 | | | 344 | |
343 | lwpinit_specificdata(); | | 345 | lwpinit_specificdata(); |
344 | lwp_initspecific(&lwp0); | | 346 | lwp_initspecific(&lwp0); |
345 | | | 347 | |
346 | /* Must be called after lwpinit_specificdata */ | | 348 | /* Must be called after lwpinit_specificdata */ |
347 | psref_init(); | | 349 | psref_init(); |
348 | | | 350 | |
349 | threadpools_init(); | | 351 | threadpools_init(); |
350 | | | 352 | |
351 | loginit(); | | 353 | loginit(); |
352 | | | 354 | |
353 | rump_biglock_init(); | | 355 | rump_biglock_init(); |
354 | | | 356 | |
355 | rump_scheduler_init(numcpu); | | 357 | rump_scheduler_init(numcpu); |
356 | /* revert temporary context and schedule a semireal context */ | | 358 | /* revert temporary context and schedule a semireal context */ |
357 | rump_lwproc_curlwp_clear(l); | | 359 | rump_lwproc_curlwp_clear(l); |
358 | initproc = &proc0; /* borrow proc0 before we get initproc started */ | | 360 | initproc = &proc0; /* borrow proc0 before we get initproc started */ |
359 | rump_schedule(); | | 361 | rump_schedule(); |
360 | bootlwp = curlwp; | | 362 | bootlwp = curlwp; |
361 | | | 363 | |
362 | inittimecounter(); | | 364 | inittimecounter(); |
363 | ntp_init(); | | 365 | ntp_init(); |
364 | | | 366 | |
365 | #ifdef KTRACE | | 367 | #ifdef KTRACE |
366 | ktrinit(); | | 368 | ktrinit(); |
367 | #endif | | 369 | #endif |
368 | | | 370 | |
369 | ts = boottime; | | 371 | ts = boottime; |
370 | tc_setclock(&ts); | | 372 | tc_setclock(&ts); |
371 | | | 373 | |
372 | extern krwlock_t exec_lock; | | 374 | extern krwlock_t exec_lock; |
373 | rw_init(&exec_lock); | | 375 | rw_init(&exec_lock); |
374 | | | 376 | |
375 | /* we are mostly go. do per-cpu subsystem init */ | | 377 | /* we are mostly go. do per-cpu subsystem init */ |
376 | for (i = 0; i < numcpu; i++) { | | 378 | for (i = 0; i < numcpu; i++) { |
377 | struct cpu_info *ci = cpu_lookup(i); | | 379 | struct cpu_info *ci = cpu_lookup(i); |
378 | | | 380 | |
379 | /* attach non-bootstrap CPUs */ | | 381 | /* attach non-bootstrap CPUs */ |
380 | if (i > 0) { | | 382 | if (i > 0) { |
381 | rump_cpu_attach(ci); | | 383 | rump_cpu_attach(ci); |
382 | ncpu++; | | 384 | ncpu++; |
383 | } | | 385 | } |
384 | | | 386 | |
385 | callout_init_cpu(ci); | | 387 | callout_init_cpu(ci); |
386 | softint_init(ci); | | 388 | softint_init(ci); |
387 | xc_init_cpu(ci); | | 389 | xc_init_cpu(ci); |
388 | pool_cache_cpu_init(ci); | | 390 | pool_cache_cpu_init(ci); |
389 | selsysinit(ci); | | 391 | selsysinit(ci); |
390 | percpu_init_cpu(ci); | | 392 | percpu_init_cpu(ci); |
391 | | | 393 | |
392 | TAILQ_INIT(&ci->ci_data.cpu_ld_locks); | | 394 | TAILQ_INIT(&ci->ci_data.cpu_ld_locks); |
393 | __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock); | | 395 | __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock); |
394 | | | 396 | |
395 | aprint_verbose("cpu%d at thinair0: rump virtual cpu\n", i); | | 397 | aprint_verbose("cpu%d at thinair0: rump virtual cpu\n", i); |
396 | } | | 398 | } |
397 | ncpuonline = ncpu; | | 399 | ncpuonline = ncpu; |
398 | | | 400 | |
399 | /* Once all CPUs are detected, initialize the per-CPU cprng_fast. */ | | 401 | /* Once all CPUs are detected, initialize the per-CPU cprng_fast. */ |
400 | cprng_fast_init(); | | 402 | cprng_fast_init(); |
401 | | | 403 | |
402 | mp_online = true; | | 404 | mp_online = true; |
403 | | | 405 | |
404 | /* CPUs are up. allow kernel threads to run */ | | 406 | /* CPUs are up. allow kernel threads to run */ |
405 | rump_thread_allow(NULL); | | 407 | rump_thread_allow(NULL); |
406 | | | 408 | |
407 | rnd_init_softint(); | | 409 | rnd_init_softint(); |
408 | | | 410 | |
409 | kqueue_init(); | | 411 | kqueue_init(); |
410 | iostat_init(); | | 412 | iostat_init(); |
411 | fd_sys_init(); | | 413 | fd_sys_init(); |
412 | module_init(); | | 414 | module_init(); |
413 | devsw_init(); | | 415 | devsw_init(); |
414 | pipe_init(); | | 416 | pipe_init(); |
415 | resource_init(); | | 417 | resource_init(); |
416 | procinit_sysctl(); | | 418 | procinit_sysctl(); |
417 | time_init(); | | 419 | time_init(); |
418 | time_init2(); | | 420 | time_init2(); |
419 | | | 421 | |
420 | /* start page baroness */ | | 422 | /* start page baroness */ |
421 | if (rump_threads) { | | 423 | if (rump_threads) { |
422 | if (kthread_create(PRI_PGDAEMON, KTHREAD_MPSAFE, NULL, | | 424 | if (kthread_create(PRI_PGDAEMON, KTHREAD_MPSAFE, NULL, |
423 | uvm_pageout, NULL, &uvm.pagedaemon_lwp, "pdaemon") != 0) | | 425 | uvm_pageout, NULL, &uvm.pagedaemon_lwp, "pdaemon") != 0) |
424 | panic("pagedaemon create failed"); | | 426 | panic("pagedaemon create failed"); |
425 | } else | | 427 | } else |
426 | uvm.pagedaemon_lwp = NULL; /* doesn't match curlwp */ | | 428 | uvm.pagedaemon_lwp = NULL; /* doesn't match curlwp */ |
427 | | | 429 | |
428 | /* process dso's */ | | 430 | /* process dso's */ |
429 | rumpuser_dl_bootstrap(add_linkedin_modules, | | 431 | rumpuser_dl_bootstrap(add_linkedin_modules, |
430 | rump_kernelfsym_load, rump_component_load); | | 432 | rump_kernelfsym_load, rump_component_load); |
431 | | | 433 | |
432 | rump_component_addlocal(); | | 434 | rump_component_addlocal(); |
433 | rump_component_init(RUMP_COMPONENT_KERN); | | 435 | rump_component_init(RUMP_COMPONENT_KERN); |
434 | | | 436 | |
435 | /* initialize factions, if present */ | | 437 | /* initialize factions, if present */ |
436 | rump_component_init(RUMP__FACTION_VFS); | | 438 | rump_component_init(RUMP__FACTION_VFS); |
437 | /* pnbuf_cache is used even without vfs */ | | 439 | /* pnbuf_cache is used even without vfs */ |
438 | if (rump_component_count(RUMP__FACTION_VFS) == 0) { | | 440 | if (rump_component_count(RUMP__FACTION_VFS) == 0) { |
439 | pnbuf_cache = pool_cache_init(MAXPATHLEN, 0, 0, 0, "pnbufpl", | | 441 | pnbuf_cache = pool_cache_init(MAXPATHLEN, 0, 0, 0, "pnbufpl", |
440 | NULL, IPL_NONE, NULL, NULL, NULL); | | 442 | NULL, IPL_NONE, NULL, NULL, NULL); |
441 | } | | 443 | } |
442 | rump_component_init(RUMP__FACTION_NET); | | 444 | rump_component_init(RUMP__FACTION_NET); |
443 | rump_component_init(RUMP__FACTION_DEV); | | 445 | rump_component_init(RUMP__FACTION_DEV); |
444 | KASSERT(rump_component_count(RUMP__FACTION_VFS) <= 1 | | 446 | KASSERT(rump_component_count(RUMP__FACTION_VFS) <= 1 |
445 | && rump_component_count(RUMP__FACTION_NET) <= 1 | | 447 | && rump_component_count(RUMP__FACTION_NET) <= 1 |
446 | && rump_component_count(RUMP__FACTION_DEV) <= 1); | | 448 | && rump_component_count(RUMP__FACTION_DEV) <= 1); |
447 | | | 449 | |
448 | rump_component_init(RUMP_COMPONENT_KERN_VFS); | | 450 | rump_component_init(RUMP_COMPONENT_KERN_VFS); |
449 | | | 451 | |
450 | /* | | 452 | /* |
451 | * if we initialized the tty component above, the tyttymtx is | | 453 | * if we initialized the tty component above, the tyttymtx is |
452 | * now initialized. otherwise, we need to initialize it. | | 454 | * now initialized. otherwise, we need to initialize it. |
453 | */ | | 455 | */ |
454 | if (!rump_ttycomponent) | | 456 | if (!rump_ttycomponent) |
455 | mutex_init(&tty_lock, MUTEX_DEFAULT, IPL_VM); | | 457 | mutex_init(&tty_lock, MUTEX_DEFAULT, IPL_VM); |
456 | | | 458 | |
457 | cold = 0; | | 459 | cold = 0; |
458 | | | 460 | |
459 | /* aieeeedondest */ | | 461 | /* aieeeedondest */ |
460 | if (rump_threads) { | | 462 | if (rump_threads) { |
461 | if (workqueue_create(&uvm.aiodone_queue, "aiodoned", | | 463 | if (workqueue_create(&uvm.aiodone_queue, "aiodoned", |
462 | rump_aiodone_worker, NULL, 0, 0, WQ_MPSAFE)) | | 464 | rump_aiodone_worker, NULL, 0, 0, WQ_MPSAFE)) |
463 | panic("aiodoned"); | | 465 | panic("aiodoned"); |
464 | } | | 466 | } |
465 | | | 467 | |
466 | sysctl_finalize(); | | 468 | sysctl_finalize(); |
467 | | | 469 | |
468 | module_init_class(MODULE_CLASS_ANY); | | 470 | module_init_class(MODULE_CLASS_ANY); |
469 | | | 471 | |
470 | if (rumpuser_getparam(RUMPUSER_PARAM_HOSTNAME, | | 472 | if (rumpuser_getparam(RUMPUSER_PARAM_HOSTNAME, |
471 | hostname, MAXHOSTNAMELEN) != 0) { | | 473 | hostname, MAXHOSTNAMELEN) != 0) { |
472 | panic("mandatory hypervisor configuration (HOSTNAME) missing"); | | 474 | panic("mandatory hypervisor configuration (HOSTNAME) missing"); |
473 | } | | 475 | } |
474 | hostnamelen = strlen(hostname); | | 476 | hostnamelen = strlen(hostname); |
475 | | | 477 | |
476 | sigemptyset(&sigcantmask); | | 478 | sigemptyset(&sigcantmask); |
477 | | | 479 | |
478 | if (rump_threads) | | 480 | if (rump_threads) |
479 | vmem_rehash_start(); | | 481 | vmem_rehash_start(); |
480 | | | 482 | |
481 | /* | | 483 | /* |
482 | * Create init (proc 1), used to attach implicit threads in rump. | | 484 | * Create init (proc 1), used to attach implicit threads in rump. |
483 | * (note: must be done after vfsinit to get cwdi) | | 485 | * (note: must be done after vfsinit to get cwdi) |
484 | */ | | 486 | */ |
485 | initlwp = rump__lwproc_alloclwp(NULL); | | 487 | initlwp = rump__lwproc_alloclwp(NULL); |
486 | mutex_enter(proc_lock); | | 488 | mutex_enter(proc_lock); |
487 | initproc = proc_find_raw(1); | | 489 | initproc = proc_find_raw(1); |
488 | mutex_exit(proc_lock); | | 490 | mutex_exit(proc_lock); |
489 | if (initproc == NULL) | | 491 | if (initproc == NULL) |
490 | panic("where in the world is initproc?"); | | 492 | panic("where in the world is initproc?"); |
491 | strlcpy(initproc->p_comm, "rumplocal", sizeof(initproc->p_comm)); | | 493 | strlcpy(initproc->p_comm, "rumplocal", sizeof(initproc->p_comm)); |
492 | | | 494 | |
493 | rump_component_init(RUMP_COMPONENT_POSTINIT); | | 495 | rump_component_init(RUMP_COMPONENT_POSTINIT); |
494 | | | 496 | |
495 | /* load syscalls */ | | 497 | /* load syscalls */ |
496 | rump_component_init(RUMP_COMPONENT_SYSCALL); | | 498 | rump_component_init(RUMP_COMPONENT_SYSCALL); |
497 | | | 499 | |
498 | /* component inits done */ | | 500 | /* component inits done */ |
499 | bootlwp = NULL; | | 501 | bootlwp = NULL; |
500 | | | 502 | |
501 | /* open 0/1/2 for init */ | | 503 | /* open 0/1/2 for init */ |
502 | KASSERT(rump_lwproc_curlwp() == NULL); | | 504 | KASSERT(rump_lwproc_curlwp() == NULL); |
503 | rump_lwproc_switch(initlwp); | | 505 | rump_lwproc_switch(initlwp); |
504 | rump_consdev_init(); | | 506 | rump_consdev_init(); |
505 | rump_lwproc_switch(NULL); | | 507 | rump_lwproc_switch(NULL); |
506 | | | 508 | |
507 | /* release cpu */ | | 509 | /* release cpu */ |
508 | rump_unschedule(); | | 510 | rump_unschedule(); |
509 | | | 511 | |
510 | return 0; | | 512 | return 0; |
511 | } | | 513 | } |
512 | /* historic compat */ | | 514 | /* historic compat */ |
513 | __strong_alias(rump__init,rump_init); | | 515 | __strong_alias(rump__init,rump_init); |
514 | | | 516 | |
515 | static int compcounter[RUMP_COMPONENT_MAX]; | | 517 | static int compcounter[RUMP_COMPONENT_MAX]; |
516 | static int compinited[RUMP_COMPONENT_MAX]; | | 518 | static int compinited[RUMP_COMPONENT_MAX]; |
517 | | | 519 | |
518 | /* | | 520 | /* |
519 | * Yea, this is O(n^2), but we're only looking at a handful of components. | | 521 | * Yea, this is O(n^2), but we're only looking at a handful of components. |
520 | * Components are always initialized from the thread that called rump_init(). | | 522 | * Components are always initialized from the thread that called rump_init(). |
521 | */ | | 523 | */ |
522 | static LIST_HEAD(, rump_component) rchead = LIST_HEAD_INITIALIZER(rchead); | | 524 | static LIST_HEAD(, rump_component) rchead = LIST_HEAD_INITIALIZER(rchead); |
523 | | | 525 | |
524 | #ifdef RUMP_USE_CTOR | | 526 | #ifdef RUMP_USE_CTOR |
525 | struct modinfo_boot_chain modinfo_boot_chain \ | | 527 | struct modinfo_boot_chain modinfo_boot_chain \ |
526 | = LIST_HEAD_INITIALIZER(modinfo_boot_chain); | | 528 | = LIST_HEAD_INITIALIZER(modinfo_boot_chain); |
527 | | | 529 | |
528 | static void | | 530 | static void |
529 | rump_component_addlocal(void) | | 531 | rump_component_addlocal(void) |
530 | { | | 532 | { |
531 | struct modinfo_chain *mc; | | 533 | struct modinfo_chain *mc; |
532 | | | 534 | |
533 | while ((mc = LIST_FIRST(&modinfo_boot_chain)) != NULL) { | | 535 | while ((mc = LIST_FIRST(&modinfo_boot_chain)) != NULL) { |
534 | LIST_REMOVE(mc, mc_entries); | | 536 | LIST_REMOVE(mc, mc_entries); |
535 | module_builtin_add(&mc->mc_info, 1, false); | | 537 | module_builtin_add(&mc->mc_info, 1, false); |
536 | } | | 538 | } |
537 | } | | 539 | } |
538 | | | 540 | |
539 | #else /* RUMP_USE_CTOR */ | | 541 | #else /* RUMP_USE_CTOR */ |
540 | | | 542 | |
541 | static void | | 543 | static void |
542 | rump_component_addlocal(void) | | 544 | rump_component_addlocal(void) |
543 | { | | 545 | { |
544 | __link_set_decl(rump_components, struct rump_component); | | 546 | __link_set_decl(rump_components, struct rump_component); |
545 | struct rump_component *const *rc; | | 547 | struct rump_component *const *rc; |
546 | | | 548 | |
547 | __link_set_foreach(rc, rump_components) { | | 549 | __link_set_foreach(rc, rump_components) { |
548 | rump_component_load(*rc); | | 550 | rump_component_load(*rc); |
549 | } | | 551 | } |
550 | } | | 552 | } |
551 | #endif /* RUMP_USE_CTOR */ | | 553 | #endif /* RUMP_USE_CTOR */ |
552 | | | 554 | |
553 | void | | 555 | void |
554 | rump_component_load(const struct rump_component *rc_const) | | 556 | rump_component_load(const struct rump_component *rc_const) |
555 | { | | 557 | { |
556 | struct rump_component *rc, *rc_iter; | | 558 | struct rump_component *rc, *rc_iter; |
557 | | | 559 | |
558 | /* time for rump component loading and unloading has passed */ | | 560 | /* time for rump component loading and unloading has passed */ |
559 | if (!cold) | | 561 | if (!cold) |
560 | return; | | 562 | return; |
561 | | | 563 | |
562 | /* | | 564 | /* |
563 | * XXX: this is ok since the "const" was removed from the | | 565 | * XXX: this is ok since the "const" was removed from the |
564 | * definition of RUMP_COMPONENT(). | | 566 | * definition of RUMP_COMPONENT(). |
565 | * | | 567 | * |
566 | * However, to preserve the hypercall interface, the const | | 568 | * However, to preserve the hypercall interface, the const |
567 | * remains here. This can be fixed in the next hypercall revision. | | 569 | * remains here. This can be fixed in the next hypercall revision. |
568 | */ | | 570 | */ |
569 | rc = __UNCONST(rc_const); | | 571 | rc = __UNCONST(rc_const); |
570 | | | 572 | |
571 | KASSERT(!rump_inited || curlwp == bootlwp); | | 573 | KASSERT(!rump_inited || curlwp == bootlwp); |
572 | | | 574 | |
573 | LIST_FOREACH(rc_iter, &rchead, rc_entries) { | | 575 | LIST_FOREACH(rc_iter, &rchead, rc_entries) { |
574 | if (rc_iter == rc) | | 576 | if (rc_iter == rc) |
575 | return; | | 577 | return; |
576 | } | | 578 | } |
577 | | | 579 | |
578 | LIST_INSERT_HEAD(&rchead, rc, rc_entries); | | 580 | LIST_INSERT_HEAD(&rchead, rc, rc_entries); |
579 | KASSERT(rc->rc_type < RUMP_COMPONENT_MAX); | | 581 | KASSERT(rc->rc_type < RUMP_COMPONENT_MAX); |
580 | compcounter[rc->rc_type]++; | | 582 | compcounter[rc->rc_type]++; |
581 | } | | 583 | } |
582 | | | 584 | |
583 | void | | 585 | void |
584 | rump_component_unload(struct rump_component *rc) | | 586 | rump_component_unload(struct rump_component *rc) |
585 | { | | 587 | { |
586 | | | 588 | |
587 | /* | | 589 | /* |
588 | * Checking for cold is enough because rump_init() both | | 590 | * Checking for cold is enough because rump_init() both |
589 | * flips it and handles component loading. | | 591 | * flips it and handles component loading. |
590 | */ | | 592 | */ |
591 | if (!cold) | | 593 | if (!cold) |
592 | return; | | 594 | return; |
593 | | | 595 | |
594 | LIST_REMOVE(rc, rc_entries); | | 596 | LIST_REMOVE(rc, rc_entries); |
595 | } | | 597 | } |
596 | | | 598 | |
597 | int | | 599 | int |
598 | rump_component_count(enum rump_component_type type) | | 600 | rump_component_count(enum rump_component_type type) |
599 | { | | 601 | { |
600 | | | 602 | |
601 | KASSERT(curlwp == bootlwp); | | 603 | KASSERT(curlwp == bootlwp); |
602 | KASSERT(type < RUMP_COMPONENT_MAX); | | 604 | KASSERT(type < RUMP_COMPONENT_MAX); |
603 | return compcounter[type]; | | 605 | return compcounter[type]; |
604 | } | | 606 | } |
605 | | | 607 | |
606 | void | | 608 | void |
607 | rump_component_init(enum rump_component_type type) | | 609 | rump_component_init(enum rump_component_type type) |
608 | { | | 610 | { |
609 | const struct rump_component *rc, *rc_safe; | | 611 | const struct rump_component *rc, *rc_safe; |
610 | | | 612 | |
611 | KASSERT(curlwp == bootlwp); | | 613 | KASSERT(curlwp == bootlwp); |
612 | KASSERT(!compinited[type]); | | 614 | KASSERT(!compinited[type]); |
613 | LIST_FOREACH_SAFE(rc, &rchead, rc_entries, rc_safe) { | | 615 | LIST_FOREACH_SAFE(rc, &rchead, rc_entries, rc_safe) { |
614 | if (rc->rc_type == type) { | | 616 | if (rc->rc_type == type) { |
615 | rc->rc_init(); | | 617 | rc->rc_init(); |
616 | LIST_REMOVE(rc, rc_entries); | | 618 | LIST_REMOVE(rc, rc_entries); |
617 | } | | 619 | } |
618 | } | | 620 | } |
619 | compinited[type] = 1; | | 621 | compinited[type] = 1; |
620 | } | | 622 | } |
621 | | | 623 | |
622 | /* | | 624 | /* |
623 | * Initialize a module which has already been loaded and linked | | 625 | * Initialize a module which has already been loaded and linked |
624 | * with dlopen(). This is fundamentally the same as a builtin module. | | 626 | * with dlopen(). This is fundamentally the same as a builtin module. |
625 | * | | 627 | * |
626 | * XXX: this interface does not really work in the RUMP_USE_CTOR case, | | 628 | * XXX: this interface does not really work in the RUMP_USE_CTOR case, |
627 | * but I'm not sure it's anything to cry about. In feeling blue, | | 629 | * but I'm not sure it's anything to cry about. In feeling blue, |
628 | * things could somehow be handled via modinfo_boot_chain. | | 630 | * things could somehow be handled via modinfo_boot_chain. |
629 | */ | | 631 | */ |
630 | int | | 632 | int |
631 | rump_module_init(const struct modinfo * const *mip, size_t nmodinfo) | | 633 | rump_module_init(const struct modinfo * const *mip, size_t nmodinfo) |
632 | { | | 634 | { |
633 | | | 635 | |
634 | return module_builtin_add(mip, nmodinfo, true); | | 636 | return module_builtin_add(mip, nmodinfo, true); |
635 | } | | 637 | } |
636 | | | 638 | |
637 | /* | | 639 | /* |
638 | * Finish module (flawless victory, fatality!). | | 640 | * Finish module (flawless victory, fatality!). |
639 | */ | | 641 | */ |
640 | int | | 642 | int |
641 | rump_module_fini(const struct modinfo *mi) | | 643 | rump_module_fini(const struct modinfo *mi) |
642 | { | | 644 | { |
643 | | | 645 | |
644 | return module_builtin_remove(mi, true); | | 646 | return module_builtin_remove(mi, true); |
645 | } | | 647 | } |
646 | | | 648 | |
647 | /* | | 649 | /* |
648 | * Add loaded and linked module to the builtin list. It will | | 650 | * Add loaded and linked module to the builtin list. It will |
649 | * later be initialized with module_init_class(). | | 651 | * later be initialized with module_init_class(). |
650 | */ | | 652 | */ |
651 | | | 653 | |
652 | static void | | 654 | static void |
653 | add_linkedin_modules(const struct modinfo * const *mip, size_t nmodinfo) | | 655 | add_linkedin_modules(const struct modinfo * const *mip, size_t nmodinfo) |
654 | { | | 656 | { |
655 | | | 657 | |
656 | module_builtin_add(mip, nmodinfo, false); | | 658 | module_builtin_add(mip, nmodinfo, false); |
657 | } | | 659 | } |
658 | | | 660 | |
659 | int | | 661 | int |
660 | rump_kernelfsym_load(void *symtab, uint64_t symsize, | | 662 | rump_kernelfsym_load(void *symtab, uint64_t symsize, |
661 | char *strtab, uint64_t strsize) | | 663 | char *strtab, uint64_t strsize) |
662 | { | | 664 | { |
663 | static int inited = 0; | | 665 | static int inited = 0; |
664 | Elf64_Ehdr ehdr; | | 666 | Elf64_Ehdr ehdr; |
665 | | | 667 | |
666 | if (inited) | | 668 | if (inited) |
667 | return EBUSY; | | 669 | return EBUSY; |
668 | inited = 1; | | 670 | inited = 1; |
669 | | | 671 | |
670 | /* | | 672 | /* |
671 | * Use 64bit header since it's bigger. Shouldn't make a | | 673 | * Use 64bit header since it's bigger. Shouldn't make a |
672 | * difference, since we're passing in all zeroes anyway. | | 674 | * difference, since we're passing in all zeroes anyway. |
673 | */ | | 675 | */ |
674 | memset(&ehdr, 0, sizeof(ehdr)); | | 676 | memset(&ehdr, 0, sizeof(ehdr)); |
675 | ksyms_addsyms_explicit(&ehdr, symtab, symsize, strtab, strsize); | | 677 | ksyms_addsyms_explicit(&ehdr, symtab, symsize, strtab, strsize); |
676 | | | 678 | |
677 | return 0; | | 679 | return 0; |
678 | } | | 680 | } |
679 | | | 681 | |
680 | int | | 682 | int |
681 | rump_boot_gethowto() | | 683 | rump_boot_gethowto() |
682 | { | | 684 | { |
683 | | | 685 | |
684 | return boothowto; | | 686 | return boothowto; |
685 | } | | 687 | } |
686 | | | 688 | |
687 | void | | 689 | void |
688 | rump_boot_sethowto(int howto) | | 690 | rump_boot_sethowto(int howto) |
689 | { | | 691 | { |
690 | | | 692 | |
691 | boothowto = howto; | | 693 | boothowto = howto; |
692 | } | | 694 | } |
693 | | | 695 | |
694 | int | | 696 | int |
695 | rump_getversion(void) | | 697 | rump_getversion(void) |
696 | { | | 698 | { |
697 | | | 699 | |
698 | return __NetBSD_Version__; | | 700 | return __NetBSD_Version__; |
699 | } | | 701 | } |
700 | /* compat */ | | 702 | /* compat */ |
701 | __strong_alias(rump_pub_getversion,rump_getversion); | | 703 | __strong_alias(rump_pub_getversion,rump_getversion); |
702 | | | 704 | |
703 | /* | | 705 | /* |
704 | * Note: may be called unscheduled. Not fully safe since no locking | | 706 | * Note: may be called unscheduled. Not fully safe since no locking |
705 | * of allevents (currently that's not even available). | | 707 | * of allevents (currently that's not even available). |
706 | */ | | 708 | */ |
707 | void | | 709 | void |
708 | rump_printevcnts() | | 710 | rump_printevcnts() |
709 | { | | 711 | { |
710 | struct evcnt *ev; | | 712 | struct evcnt *ev; |
711 | | | 713 | |
712 | TAILQ_FOREACH(ev, &allevents, ev_list) | | 714 | TAILQ_FOREACH(ev, &allevents, ev_list) |
713 | rumpuser_dprintf("%s / %s: %" PRIu64 "\n", | | 715 | rumpuser_dprintf("%s / %s: %" PRIu64 "\n", |
714 | ev->ev_group, ev->ev_name, ev->ev_count); | | 716 | ev->ev_group, ev->ev_name, ev->ev_count); |
715 | } | | 717 | } |
716 | | | 718 | |
717 | /* | | 719 | /* |
718 | * If you use this interface ... well ... all bets are off. | | 720 | * If you use this interface ... well ... all bets are off. |
719 | * The original purpose is for the p2k fs server library to be | | 721 | * The original purpose is for the p2k fs server library to be |
720 | * able to use the same pid/lid for VOPs as the host kernel. | | 722 | * able to use the same pid/lid for VOPs as the host kernel. |
721 | */ | | 723 | */ |
722 | void | | 724 | void |
723 | rump_allbetsareoff_setid(pid_t pid, int lid) | | 725 | rump_allbetsareoff_setid(pid_t pid, int lid) |
724 | { | | 726 | { |
725 | struct lwp *l = curlwp; | | 727 | struct lwp *l = curlwp; |
726 | struct proc *p = l->l_proc; | | 728 | struct proc *p = l->l_proc; |
727 | | | 729 | |
728 | l->l_lid = lid; | | 730 | l->l_lid = lid; |
729 | p->p_pid = pid; | | 731 | p->p_pid = pid; |
730 | } | | 732 | } |
731 | | | 733 | |
732 | static void | | 734 | static void |
733 | ipiemu(void *a1, void *a2) | | 735 | ipiemu(void *a1, void *a2) |
734 | { | | 736 | { |
735 | | | 737 | |
736 | xc__highpri_intr(NULL); | | 738 | xc__highpri_intr(NULL); |
737 | } | | 739 | } |
738 | | | 740 | |
739 | void | | 741 | void |
740 | rump_xc_highpri(struct cpu_info *ci) | | 742 | rump_xc_highpri(struct cpu_info *ci) |
741 | { | | 743 | { |
742 | | | 744 | |
743 | if (ci) | | 745 | if (ci) |
744 | xc_unicast(0, ipiemu, NULL, NULL, ci); | | 746 | xc_unicast(0, ipiemu, NULL, NULL, ci); |
745 | else | | 747 | else |
746 | xc_broadcast(0, ipiemu, NULL, NULL); | | 748 | xc_broadcast(0, ipiemu, NULL, NULL); |
747 | } | | 749 | } |
748 | | | 750 | |
749 | int | | 751 | int |
750 | rump_syscall(int num, void *data, size_t dlen, register_t *retval) | | 752 | rump_syscall(int num, void *data, size_t dlen, register_t *retval) |
751 | { | | 753 | { |
752 | struct proc *p; | | 754 | struct proc *p; |
753 | struct emul *e; | | 755 | struct emul *e; |
754 | struct sysent *callp; | | 756 | struct sysent *callp; |
755 | const int *etrans = NULL; | | 757 | const int *etrans = NULL; |
756 | int rv; | | 758 | int rv; |
757 | | | 759 | |
758 | rump_schedule(); | | 760 | rump_schedule(); |
759 | p = curproc; | | 761 | p = curproc; |
760 | e = p->p_emul; | | 762 | e = p->p_emul; |
761 | #ifndef __HAVE_MINIMAL_EMUL | | 763 | #ifndef __HAVE_MINIMAL_EMUL |
762 | KASSERT(num > 0 && num < e->e_nsysent); | | 764 | KASSERT(num > 0 && num < e->e_nsysent); |
763 | #endif | | 765 | #endif |
764 | callp = e->e_sysent + num; | | 766 | callp = e->e_sysent + num; |
765 | | | 767 | |
766 | rv = sy_invoke(callp, curlwp, data, retval, num); | | 768 | rv = sy_invoke(callp, curlwp, data, retval, num); |
767 | | | 769 | |
768 | /* | | 770 | /* |
769 | * I hope that (!__HAVE_MINIMAL_EMUL || __HAVE_SYSCALL_INTERN) is | | 771 | * I hope that (!__HAVE_MINIMAL_EMUL || __HAVE_SYSCALL_INTERN) is |
770 | * an invariant ... | | 772 | * an invariant ... |
771 | */ | | 773 | */ |
772 | #if !defined(__HAVE_MINIMAL_EMUL) | | 774 | #if !defined(__HAVE_MINIMAL_EMUL) |
773 | etrans = e->e_errno; | | 775 | etrans = e->e_errno; |
774 | #elif defined(__HAVE_SYSCALL_INTERN) | | 776 | #elif defined(__HAVE_SYSCALL_INTERN) |
775 | etrans = p->p_emuldata; | | 777 | etrans = p->p_emuldata; |
776 | #endif | | 778 | #endif |
777 | | | 779 | |
778 | if (etrans) { | | 780 | if (etrans) { |
779 | rv = etrans[rv]; | | 781 | rv = etrans[rv]; |
780 | /* | | 782 | /* |
781 | * XXX: small hack since Linux etrans vectors on some | | 783 | * XXX: small hack since Linux etrans vectors on some |
782 | * archs contain negative errnos, but rump_syscalls | | 784 | * archs contain negative errnos, but rump_syscalls |
783 | * uses the -1 + errno ABI. Note that these | | 785 | * uses the -1 + errno ABI. Note that these |
784 | * negative values are always the result of translation, | | 786 | * negative values are always the result of translation, |
785 | * otherwise the above translation method would not | | 787 | * otherwise the above translation method would not |
786 | * work very well. | | 788 | * work very well. |
787 | */ | | 789 | */ |
788 | if (rv < 0) | | 790 | if (rv < 0) |
789 | rv = -rv; | | 791 | rv = -rv; |
790 | } | | 792 | } |
791 | rump_unschedule(); | | 793 | rump_unschedule(); |
792 | | | 794 | |
793 | return rv; | | 795 | return rv; |
794 | } | | 796 | } |
795 | | | 797 | |
796 | void | | 798 | void |
797 | rump_syscall_boot_establish(const struct rump_onesyscall *calls, size_t ncall) | | 799 | rump_syscall_boot_establish(const struct rump_onesyscall *calls, size_t ncall) |
798 | { | | 800 | { |
799 | struct sysent *callp; | | 801 | struct sysent *callp; |
800 | size_t i; | | 802 | size_t i; |
801 | | | 803 | |
802 | for (i = 0; i < ncall; i++) { | | 804 | for (i = 0; i < ncall; i++) { |
803 | callp = rump_sysent + calls[i].ros_num; | | 805 | callp = rump_sysent + calls[i].ros_num; |
804 | KASSERT(bootlwp != NULL | | 806 | KASSERT(bootlwp != NULL |
805 | && callp->sy_call == (sy_call_t *)(void *)enosys); | | 807 | && callp->sy_call == (sy_call_t *)(void *)enosys); |
806 | callp->sy_call = calls[i].ros_handler; | | 808 | callp->sy_call = calls[i].ros_handler; |
807 | } | | 809 | } |
808 | } | | 810 | } |
809 | | | 811 | |
810 | struct rump_boot_etfs *ebstart; | | 812 | struct rump_boot_etfs *ebstart; |
811 | void | | 813 | void |
812 | rump_boot_etfs_register(struct rump_boot_etfs *eb) | | 814 | rump_boot_etfs_register(struct rump_boot_etfs *eb) |
813 | { | | 815 | { |
814 | | | 816 | |
815 | /* | | 817 | /* |
816 | * Could use atomics, but, since caller would need to synchronize | | 818 | * Could use atomics, but, since caller would need to synchronize |
817 | * against calling rump_init() anyway, easier to just specify the | | 819 | * against calling rump_init() anyway, easier to just specify the |
818 | * interface as "caller serializes". This solve-by-specification | | 820 | * interface as "caller serializes". This solve-by-specification |
819 | * approach avoids the grey area of using atomics before rump_init() | | 821 | * approach avoids the grey area of using atomics before rump_init() |
820 | * runs. | | 822 | * runs. |
821 | */ | | 823 | */ |
822 | eb->_eb_next = ebstart; | | 824 | eb->_eb_next = ebstart; |
823 | eb->eb_status = -1; | | 825 | eb->eb_status = -1; |
824 | ebstart = eb; | | 826 | ebstart = eb; |
825 | } | | 827 | } |