| @@ -1,825 +1,825 @@ | | | @@ -1,825 +1,825 @@ |
1 | /* $NetBSD: rump.c,v 1.344 2020/03/23 14:49:50 pgoyette Exp $ */ | | 1 | /* $NetBSD: rump.c,v 1.345 2020/04/23 00:34:29 joerg Exp $ */ |
2 | | | 2 | |
3 | /* | | 3 | /* |
4 | * Copyright (c) 2007-2011 Antti Kantee. All Rights Reserved. | | 4 | * Copyright (c) 2007-2011 Antti Kantee. All Rights Reserved. |
5 | * | | 5 | * |
6 | * Redistribution and use in source and binary forms, with or without | | 6 | * Redistribution and use in source and binary forms, with or without |
7 | * modification, are permitted provided that the following conditions | | 7 | * modification, are permitted provided that the following conditions |
8 | * are met: | | 8 | * are met: |
9 | * 1. Redistributions of source code must retain the above copyright | | 9 | * 1. Redistributions of source code must retain the above copyright |
10 | * notice, this list of conditions and the following disclaimer. | | 10 | * notice, this list of conditions and the following disclaimer. |
11 | * 2. Redistributions in binary form must reproduce the above copyright | | 11 | * 2. Redistributions in binary form must reproduce the above copyright |
12 | * notice, this list of conditions and the following disclaimer in the | | 12 | * notice, this list of conditions and the following disclaimer in the |
13 | * documentation and/or other materials provided with the distribution. | | 13 | * documentation and/or other materials provided with the distribution. |
14 | * | | 14 | * |
15 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS | | 15 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS |
16 | * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED | | 16 | * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
17 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE | | 17 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
18 | * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE | | 18 | * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE |
19 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | | 19 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
20 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR | | 20 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | | 21 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
22 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | | 22 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
23 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | | 23 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
24 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | | 24 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
25 | * SUCH DAMAGE. | | 25 | * SUCH DAMAGE. |
26 | */ | | 26 | */ |
27 | | | 27 | |
28 | #include <sys/cdefs.h> | | 28 | #include <sys/cdefs.h> |
29 | __KERNEL_RCSID(0, "$NetBSD: rump.c,v 1.344 2020/03/23 14:49:50 pgoyette Exp $"); | | 29 | __KERNEL_RCSID(0, "$NetBSD: rump.c,v 1.345 2020/04/23 00:34:29 joerg Exp $"); |
30 | | | 30 | |
31 | #include <sys/systm.h> | | 31 | #include <sys/systm.h> |
32 | #define ELFSIZE ARCH_ELFSIZE | | 32 | #define ELFSIZE ARCH_ELFSIZE |
33 | | | 33 | |
34 | #include <sys/param.h> | | 34 | #include <sys/param.h> |
35 | #include <sys/atomic.h> | | 35 | #include <sys/atomic.h> |
36 | #include <sys/buf.h> | | 36 | #include <sys/buf.h> |
37 | #include <sys/callout.h> | | 37 | #include <sys/callout.h> |
38 | #include <sys/conf.h> | | 38 | #include <sys/conf.h> |
39 | #include <sys/cpu.h> | | 39 | #include <sys/cpu.h> |
40 | #include <sys/device.h> | | 40 | #include <sys/device.h> |
41 | #include <sys/evcnt.h> | | 41 | #include <sys/evcnt.h> |
42 | #include <sys/event.h> | | 42 | #include <sys/event.h> |
43 | #include <sys/exec_elf.h> | | 43 | #include <sys/exec_elf.h> |
44 | #include <sys/filedesc.h> | | 44 | #include <sys/filedesc.h> |
45 | #include <sys/iostat.h> | | 45 | #include <sys/iostat.h> |
46 | #include <sys/kauth.h> | | 46 | #include <sys/kauth.h> |
47 | #include <sys/kcpuset.h> | | 47 | #include <sys/kcpuset.h> |
48 | #include <sys/kernel.h> | | 48 | #include <sys/kernel.h> |
49 | #include <sys/kmem.h> | | 49 | #include <sys/kmem.h> |
50 | #include <sys/kprintf.h> | | 50 | #include <sys/kprintf.h> |
51 | #include <sys/kthread.h> | | 51 | #include <sys/kthread.h> |
52 | #include <sys/ksyms.h> | | 52 | #include <sys/ksyms.h> |
53 | #include <sys/msgbuf.h> | | 53 | #include <sys/msgbuf.h> |
54 | #include <sys/module.h> | | 54 | #include <sys/module.h> |
55 | #include <sys/module_hook.h> | | 55 | #include <sys/module_hook.h> |
56 | #include <sys/namei.h> | | 56 | #include <sys/namei.h> |
57 | #include <sys/once.h> | | 57 | #include <sys/once.h> |
58 | #include <sys/percpu.h> | | 58 | #include <sys/percpu.h> |
59 | #include <sys/pipe.h> | | 59 | #include <sys/pipe.h> |
60 | #include <sys/pool.h> | | 60 | #include <sys/pool.h> |
61 | #include <sys/queue.h> | | 61 | #include <sys/queue.h> |
62 | #include <sys/reboot.h> | | 62 | #include <sys/reboot.h> |
63 | #include <sys/resourcevar.h> | | 63 | #include <sys/resourcevar.h> |
64 | #include <sys/select.h> | | 64 | #include <sys/select.h> |
65 | #include <sys/sysctl.h> | | 65 | #include <sys/sysctl.h> |
66 | #include <sys/syscall.h> | | 66 | #include <sys/syscall.h> |
67 | #include <sys/syscallvar.h> | | 67 | #include <sys/syscallvar.h> |
68 | #include <sys/threadpool.h> | | 68 | #include <sys/threadpool.h> |
69 | #include <sys/timetc.h> | | 69 | #include <sys/timetc.h> |
70 | #include <sys/tty.h> | | 70 | #include <sys/tty.h> |
71 | #include <sys/uidinfo.h> | | 71 | #include <sys/uidinfo.h> |
72 | #include <sys/vmem.h> | | 72 | #include <sys/vmem.h> |
73 | #include <sys/xcall.h> | | 73 | #include <sys/xcall.h> |
74 | #include <sys/cprng.h> | | 74 | #include <sys/cprng.h> |
75 | #include <sys/rnd.h> | | 75 | #include <sys/rnd.h> |
76 | #include <sys/ktrace.h> | | 76 | #include <sys/ktrace.h> |
77 | #include <sys/pserialize.h> | | 77 | #include <sys/pserialize.h> |
78 | #include <sys/psref.h> | | 78 | #include <sys/psref.h> |
79 | | | 79 | |
80 | #include <rump-sys/kern.h> | | 80 | #include <rump-sys/kern.h> |
81 | #include <rump-sys/dev.h> | | 81 | #include <rump-sys/dev.h> |
82 | #include <rump-sys/net.h> | | 82 | #include <rump-sys/net.h> |
83 | #include <rump-sys/vfs.h> | | 83 | #include <rump-sys/vfs.h> |
84 | | | 84 | |
85 | #include <rump/rumpuser.h> | | 85 | #include <rump/rumpuser.h> |
86 | | | 86 | |
87 | #include <secmodel/suser/suser.h> | | 87 | #include <secmodel/suser/suser.h> |
88 | | | 88 | |
89 | #include <prop/proplib.h> | | 89 | #include <prop/proplib.h> |
90 | | | 90 | |
91 | #include <uvm/uvm_extern.h> | | 91 | #include <uvm/uvm_extern.h> |
92 | #include <uvm/uvm_readahead.h> | | 92 | #include <uvm/uvm_readahead.h> |
93 | | | 93 | |
94 | char machine[] = MACHINE; | | 94 | char machine[] = MACHINE; |
95 | char machine_arch[] = MACHINE_ARCH; | | 95 | char machine_arch[] = MACHINE_ARCH; |
96 | | | 96 | |
97 | struct proc *initproc; | | 97 | struct proc *initproc; |
98 | | | 98 | |
99 | struct device rump_rootdev = { | | 99 | struct device rump_rootdev = { |
100 | .dv_class = DV_VIRTUAL | | 100 | .dv_class = DV_VIRTUAL |
101 | }; | | 101 | }; |
102 | | | 102 | |
103 | #ifdef RUMP_WITHOUT_THREADS | | 103 | #ifdef RUMP_WITHOUT_THREADS |
104 | int rump_threads = 0; | | 104 | int rump_threads = 0; |
105 | #else | | 105 | #else |
106 | int rump_threads = 1; | | 106 | int rump_threads = 1; |
107 | #endif | | 107 | #endif |
108 | | | 108 | |
109 | static void rump_component_addlocal(void); | | 109 | static void rump_component_addlocal(void); |
110 | static struct lwp *bootlwp; | | 110 | static struct lwp *bootlwp; |
111 | | | 111 | |
112 | /* 16k should be enough for std rump needs */ | | 112 | /* 16k should be enough for std rump needs */ |
113 | static char rump_msgbuf[16*1024] __aligned(256); | | 113 | static char rump_msgbuf[16*1024] __aligned(256); |
114 | | | 114 | |
115 | bool rump_ttycomponent = false; | | 115 | bool rump_ttycomponent = false; |
116 | | | 116 | |
117 | pool_cache_t pnbuf_cache; | | 117 | extern pool_cache_t pnbuf_cache; |
118 | | | 118 | |
119 | static int rump_inited; | | 119 | static int rump_inited; |
120 | | | 120 | |
121 | void (*rump_vfs_drainbufs)(int) = (void *)nullop; | | 121 | void (*rump_vfs_drainbufs)(int) = (void *)nullop; |
122 | int (*rump_vfs_makeonedevnode)(dev_t, const char *, | | 122 | int (*rump_vfs_makeonedevnode)(dev_t, const char *, |
123 | devmajor_t, devminor_t) = (void *)nullop; | | 123 | devmajor_t, devminor_t) = (void *)nullop; |
124 | int (*rump_vfs_makedevnodes)(dev_t, const char *, char, | | 124 | int (*rump_vfs_makedevnodes)(dev_t, const char *, char, |
125 | devmajor_t, devminor_t, int) = (void *)nullop; | | 125 | devmajor_t, devminor_t, int) = (void *)nullop; |
126 | int (*rump_vfs_makesymlink)(const char *, const char *) = (void *)nullop; | | 126 | int (*rump_vfs_makesymlink)(const char *, const char *) = (void *)nullop; |
127 | | | 127 | |
128 | rump_proc_vfs_init_fn rump_proc_vfs_init = (void *)nullop; | | 128 | rump_proc_vfs_init_fn rump_proc_vfs_init = (void *)nullop; |
129 | rump_proc_vfs_release_fn rump_proc_vfs_release = (void *)nullop; | | 129 | rump_proc_vfs_release_fn rump_proc_vfs_release = (void *)nullop; |
130 | | | 130 | |
131 | static void add_linkedin_modules(const struct modinfo *const *, size_t); | | 131 | static void add_linkedin_modules(const struct modinfo *const *, size_t); |
132 | static void add_static_evcnt(struct evcnt *); | | 132 | static void add_static_evcnt(struct evcnt *); |
133 | | | 133 | |
134 | static pid_t rspo_wrap_getpid(void) { | | 134 | static pid_t rspo_wrap_getpid(void) { |
135 | return rump_sysproxy_hyp_getpid(); | | 135 | return rump_sysproxy_hyp_getpid(); |
136 | } | | 136 | } |
137 | static int rspo_wrap_syscall(int num, void *arg, long *retval) { | | 137 | static int rspo_wrap_syscall(int num, void *arg, long *retval) { |
138 | return rump_sysproxy_hyp_syscall(num, arg, retval); | | 138 | return rump_sysproxy_hyp_syscall(num, arg, retval); |
139 | } | | 139 | } |
140 | static int rspo_wrap_rfork(void *priv, int flag, const char *comm) { | | 140 | static int rspo_wrap_rfork(void *priv, int flag, const char *comm) { |
141 | return rump_sysproxy_hyp_rfork(priv, flag, comm); | | 141 | return rump_sysproxy_hyp_rfork(priv, flag, comm); |
142 | } | | 142 | } |
143 | static void rspo_wrap_lwpexit(void) { | | 143 | static void rspo_wrap_lwpexit(void) { |
144 | rump_sysproxy_hyp_lwpexit(); | | 144 | rump_sysproxy_hyp_lwpexit(); |
145 | } | | 145 | } |
146 | static void rspo_wrap_execnotify(const char *comm) { | | 146 | static void rspo_wrap_execnotify(const char *comm) { |
147 | rump_sysproxy_hyp_execnotify(comm); | | 147 | rump_sysproxy_hyp_execnotify(comm); |
148 | } | | 148 | } |
149 | static const struct rumpuser_hyperup hyp = { | | 149 | static const struct rumpuser_hyperup hyp = { |
150 | .hyp_schedule = rump_schedule, | | 150 | .hyp_schedule = rump_schedule, |
151 | .hyp_unschedule = rump_unschedule, | | 151 | .hyp_unschedule = rump_unschedule, |
152 | .hyp_backend_unschedule = rump_user_unschedule, | | 152 | .hyp_backend_unschedule = rump_user_unschedule, |
153 | .hyp_backend_schedule = rump_user_schedule, | | 153 | .hyp_backend_schedule = rump_user_schedule, |
154 | .hyp_lwproc_switch = rump_lwproc_switch, | | 154 | .hyp_lwproc_switch = rump_lwproc_switch, |
155 | .hyp_lwproc_release = rump_lwproc_releaselwp, | | 155 | .hyp_lwproc_release = rump_lwproc_releaselwp, |
156 | .hyp_lwproc_newlwp = rump_lwproc_newlwp, | | 156 | .hyp_lwproc_newlwp = rump_lwproc_newlwp, |
157 | .hyp_lwproc_curlwp = rump_lwproc_curlwp, | | 157 | .hyp_lwproc_curlwp = rump_lwproc_curlwp, |
158 | | | 158 | |
159 | .hyp_getpid = rspo_wrap_getpid, | | 159 | .hyp_getpid = rspo_wrap_getpid, |
160 | .hyp_syscall = rspo_wrap_syscall, | | 160 | .hyp_syscall = rspo_wrap_syscall, |
161 | .hyp_lwproc_rfork = rspo_wrap_rfork, | | 161 | .hyp_lwproc_rfork = rspo_wrap_rfork, |
162 | .hyp_lwpexit = rspo_wrap_lwpexit, | | 162 | .hyp_lwpexit = rspo_wrap_lwpexit, |
163 | .hyp_execnotify = rspo_wrap_execnotify, | | 163 | .hyp_execnotify = rspo_wrap_execnotify, |
164 | }; | | 164 | }; |
165 | struct rump_sysproxy_ops rump_sysproxy_ops = { | | 165 | struct rump_sysproxy_ops rump_sysproxy_ops = { |
166 | .rspo_copyin = (void *)enxio, | | 166 | .rspo_copyin = (void *)enxio, |
167 | .rspo_copyinstr = (void *)enxio, | | 167 | .rspo_copyinstr = (void *)enxio, |
168 | .rspo_copyout = (void *)enxio, | | 168 | .rspo_copyout = (void *)enxio, |
169 | .rspo_copyoutstr = (void *)enxio, | | 169 | .rspo_copyoutstr = (void *)enxio, |
170 | .rspo_anonmmap = (void *)enxio, | | 170 | .rspo_anonmmap = (void *)enxio, |
171 | .rspo_raise = (void *)enxio, | | 171 | .rspo_raise = (void *)enxio, |
172 | .rspo_fini = (void *)enxio, | | 172 | .rspo_fini = (void *)enxio, |
173 | .rspo_hyp_getpid = (void *)enxio, | | 173 | .rspo_hyp_getpid = (void *)enxio, |
174 | .rspo_hyp_syscall = (void *)enxio, | | 174 | .rspo_hyp_syscall = (void *)enxio, |
175 | .rspo_hyp_rfork = (void *)enxio, | | 175 | .rspo_hyp_rfork = (void *)enxio, |
176 | .rspo_hyp_lwpexit = (void *)enxio, | | 176 | .rspo_hyp_lwpexit = (void *)enxio, |
177 | .rspo_hyp_execnotify = (void *)enxio, | | 177 | .rspo_hyp_execnotify = (void *)enxio, |
178 | }; | | 178 | }; |
179 | | | 179 | |
180 | int | | 180 | int |
181 | rump_daemonize_begin(void) | | 181 | rump_daemonize_begin(void) |
182 | { | | 182 | { |
183 | | | 183 | |
184 | if (rump_inited) | | 184 | if (rump_inited) |
185 | return EALREADY; | | 185 | return EALREADY; |
186 | | | 186 | |
187 | return rumpuser_daemonize_begin(); | | 187 | return rumpuser_daemonize_begin(); |
188 | } | | 188 | } |
189 | | | 189 | |
190 | int | | 190 | int |
191 | rump_daemonize_done(int error) | | 191 | rump_daemonize_done(int error) |
192 | { | | 192 | { |
193 | | | 193 | |
194 | return rumpuser_daemonize_done(error); | | 194 | return rumpuser_daemonize_done(error); |
195 | } | | 195 | } |
196 | | | 196 | |
197 | #ifdef RUMP_USE_CTOR | | 197 | #ifdef RUMP_USE_CTOR |
198 | | | 198 | |
199 | /* sysctl bootstrap handling */ | | 199 | /* sysctl bootstrap handling */ |
200 | struct sysctl_boot_chain sysctl_boot_chain \ | | 200 | struct sysctl_boot_chain sysctl_boot_chain \ |
201 | = LIST_HEAD_INITIALIZER(sysctl_boot_chain); | | 201 | = LIST_HEAD_INITIALIZER(sysctl_boot_chain); |
202 | __link_set_add_text(sysctl_funcs,voidop); /* ensure linkset is non-empty */ | | 202 | __link_set_add_text(sysctl_funcs,voidop); /* ensure linkset is non-empty */ |
203 | | | 203 | |
204 | #else /* RUMP_USE_CTOR */ | | 204 | #else /* RUMP_USE_CTOR */ |
205 | | | 205 | |
206 | RUMP_COMPONENT(RUMP_COMPONENT_POSTINIT) | | 206 | RUMP_COMPONENT(RUMP_COMPONENT_POSTINIT) |
207 | { | | 207 | { |
208 | __link_set_decl(rump_components, struct rump_component); | | 208 | __link_set_decl(rump_components, struct rump_component); |
209 | | | 209 | |
210 | /* | | 210 | /* |
211 | * Trick compiler into generating references so that statically | | 211 | * Trick compiler into generating references so that statically |
212 | * linked rump kernels are generated with the link set symbols. | | 212 | * linked rump kernels are generated with the link set symbols. |
213 | */ | | 213 | */ |
214 | asm("" :: "r"(__start_link_set_rump_components)); | | 214 | asm("" :: "r"(__start_link_set_rump_components)); |
215 | asm("" :: "r"(__stop_link_set_rump_components)); | | 215 | asm("" :: "r"(__stop_link_set_rump_components)); |
216 | } | | 216 | } |
217 | | | 217 | |
218 | #endif /* RUMP_USE_CTOR */ | | 218 | #endif /* RUMP_USE_CTOR */ |
219 | | | 219 | |
220 | int | | 220 | int |
221 | rump_init(void) | | 221 | rump_init(void) |
222 | { | | 222 | { |
223 | char buf[256]; | | 223 | char buf[256]; |
224 | struct timespec bts; | | 224 | struct timespec bts; |
225 | int64_t sec; | | 225 | int64_t sec; |
226 | long nsec; | | 226 | long nsec; |
227 | struct lwp *l, *initlwp; | | 227 | struct lwp *l, *initlwp; |
228 | int i, numcpu; | | 228 | int i, numcpu; |
229 | | | 229 | |
230 | /* not reentrant */ | | 230 | /* not reentrant */ |
231 | if (rump_inited) | | 231 | if (rump_inited) |
232 | return 0; | | 232 | return 0; |
233 | else if (rump_inited == -1) | | 233 | else if (rump_inited == -1) |
234 | panic("rump_init: host process restart required"); | | 234 | panic("rump_init: host process restart required"); |
235 | else | | 235 | else |
236 | rump_inited = 1; | | 236 | rump_inited = 1; |
237 | | | 237 | |
238 | /* initialize hypervisor */ | | 238 | /* initialize hypervisor */ |
239 | if (rumpuser_init(RUMPUSER_VERSION, &hyp) != 0) { | | 239 | if (rumpuser_init(RUMPUSER_VERSION, &hyp) != 0) { |
240 | rumpuser_dprintf("rumpuser init failed\n"); | | 240 | rumpuser_dprintf("rumpuser init failed\n"); |
241 | return EINVAL; | | 241 | return EINVAL; |
242 | } | | 242 | } |
243 | | | 243 | |
244 | /* init minimal lwp/cpu context */ | | 244 | /* init minimal lwp/cpu context */ |
245 | rump_lwproc_init(); | | 245 | rump_lwproc_init(); |
246 | l = &lwp0; | | 246 | l = &lwp0; |
247 | l->l_cpu = l->l_target_cpu = &rump_bootcpu; | | 247 | l->l_cpu = l->l_target_cpu = &rump_bootcpu; |
248 | rump_lwproc_curlwp_set(l); | | 248 | rump_lwproc_curlwp_set(l); |
249 | | | 249 | |
250 | /* retrieve env vars which affect the early stage of bootstrap */ | | 250 | /* retrieve env vars which affect the early stage of bootstrap */ |
251 | if (rumpuser_getparam("RUMP_THREADS", buf, sizeof(buf)) == 0) { | | 251 | if (rumpuser_getparam("RUMP_THREADS", buf, sizeof(buf)) == 0) { |
252 | rump_threads = *buf != '0'; | | 252 | rump_threads = *buf != '0'; |
253 | } | | 253 | } |
254 | if (rumpuser_getparam("RUMP_VERBOSE", buf, sizeof(buf)) == 0) { | | 254 | if (rumpuser_getparam("RUMP_VERBOSE", buf, sizeof(buf)) == 0) { |
255 | if (*buf != '0') | | 255 | if (*buf != '0') |
256 | boothowto = AB_VERBOSE; | | 256 | boothowto = AB_VERBOSE; |
257 | } | | 257 | } |
258 | | | 258 | |
259 | if (rumpuser_getparam(RUMPUSER_PARAM_NCPU, buf, sizeof(buf)) != 0) | | 259 | if (rumpuser_getparam(RUMPUSER_PARAM_NCPU, buf, sizeof(buf)) != 0) |
260 | panic("mandatory hypervisor configuration (NCPU) missing"); | | 260 | panic("mandatory hypervisor configuration (NCPU) missing"); |
261 | numcpu = strtoll(buf, NULL, 10); | | 261 | numcpu = strtoll(buf, NULL, 10); |
262 | if (numcpu < 1) { | | 262 | if (numcpu < 1) { |
263 | panic("rump kernels are not lightweight enough for \"%d\" CPUs", | | 263 | panic("rump kernels are not lightweight enough for \"%d\" CPUs", |
264 | numcpu); | | 264 | numcpu); |
265 | } | | 265 | } |
266 | | | 266 | |
267 | rump_thread_init(); | | 267 | rump_thread_init(); |
268 | rump_cpus_bootstrap(&numcpu); | | 268 | rump_cpus_bootstrap(&numcpu); |
269 | | | 269 | |
270 | rumpuser_clock_gettime(RUMPUSER_CLOCK_RELWALL, &sec, &nsec); | | 270 | rumpuser_clock_gettime(RUMPUSER_CLOCK_RELWALL, &sec, &nsec); |
271 | bts.tv_sec = sec; | | 271 | bts.tv_sec = sec; |
272 | bts.tv_nsec = nsec; | | 272 | bts.tv_nsec = nsec; |
273 | | | 273 | |
274 | initmsgbuf(rump_msgbuf, sizeof(rump_msgbuf)); | | 274 | initmsgbuf(rump_msgbuf, sizeof(rump_msgbuf)); |
275 | aprint_verbose("%s%s", copyright, version); | | 275 | aprint_verbose("%s%s", copyright, version); |
276 | | | 276 | |
277 | rump_intr_init(numcpu); | | 277 | rump_intr_init(numcpu); |
278 | | | 278 | |
279 | rump_tsleep_init(); | | 279 | rump_tsleep_init(); |
280 | | | 280 | |
281 | rumpuser_mutex_init(&rump_giantlock, RUMPUSER_MTX_SPIN); | | 281 | rumpuser_mutex_init(&rump_giantlock, RUMPUSER_MTX_SPIN); |
282 | ksyms_init(); | | 282 | ksyms_init(); |
283 | uvm_init(); | | 283 | uvm_init(); |
284 | evcnt_init(); | | 284 | evcnt_init(); |
285 | | | 285 | |
286 | kcpuset_sysinit(); | | 286 | kcpuset_sysinit(); |
287 | once_init(); | | 287 | once_init(); |
288 | kernconfig_lock_init(); | | 288 | kernconfig_lock_init(); |
289 | prop_kern_init(); | | 289 | prop_kern_init(); |
290 | | | 290 | |
291 | kmem_init(); | | 291 | kmem_init(); |
292 | | | 292 | |
293 | uvm_ra_init(); | | 293 | uvm_ra_init(); |
294 | uao_init(); | | 294 | uao_init(); |
295 | | | 295 | |
296 | mutex_obj_init(); | | 296 | mutex_obj_init(); |
297 | rw_obj_init(); | | 297 | rw_obj_init(); |
298 | callout_startup(); | | 298 | callout_startup(); |
299 | | | 299 | |
300 | kprintf_init(); | | 300 | kprintf_init(); |
301 | percpu_init(); | | 301 | percpu_init(); |
302 | pserialize_init(); | | 302 | pserialize_init(); |
303 | | | 303 | |
304 | kauth_init(); | | 304 | kauth_init(); |
305 | | | 305 | |
306 | secmodel_init(); | | 306 | secmodel_init(); |
307 | sysctl_init(); | | 307 | sysctl_init(); |
308 | /* | | 308 | /* |
309 | * The above call to sysctl_init() only initializes sysctl nodes | | 309 | * The above call to sysctl_init() only initializes sysctl nodes |
310 | * from link sets. Initialize sysctls in case we used ctors. | | 310 | * from link sets. Initialize sysctls in case we used ctors. |
311 | */ | | 311 | */ |
312 | #ifdef RUMP_USE_CTOR | | 312 | #ifdef RUMP_USE_CTOR |
313 | { | | 313 | { |
314 | struct sysctl_setup_chain *ssc; | | 314 | struct sysctl_setup_chain *ssc; |
315 | | | 315 | |
316 | while ((ssc = LIST_FIRST(&sysctl_boot_chain)) != NULL) { | | 316 | while ((ssc = LIST_FIRST(&sysctl_boot_chain)) != NULL) { |
317 | LIST_REMOVE(ssc, ssc_entries); | | 317 | LIST_REMOVE(ssc, ssc_entries); |
318 | ssc->ssc_func(NULL); | | 318 | ssc->ssc_func(NULL); |
319 | } | | 319 | } |
320 | } | | 320 | } |
321 | #endif /* RUMP_USE_CTOR */ | | 321 | #endif /* RUMP_USE_CTOR */ |
322 | | | 322 | |
323 | rnd_init(); | | 323 | rnd_init(); |
324 | cprng_init(); | | 324 | cprng_init(); |
325 | kern_cprng = cprng_strong_create("kernel", IPL_VM, | | 325 | kern_cprng = cprng_strong_create("kernel", IPL_VM, |
326 | CPRNG_INIT_ANY|CPRNG_REKEY_ANY); | | 326 | CPRNG_INIT_ANY|CPRNG_REKEY_ANY); |
327 | | | 327 | |
328 | rump_hyperentropy_init(); | | 328 | rump_hyperentropy_init(); |
329 | | | 329 | |
330 | procinit(); | | 330 | procinit(); |
331 | proc0_init(); | | 331 | proc0_init(); |
332 | uid_init(); | | 332 | uid_init(); |
333 | chgproccnt(0, 1); | | 333 | chgproccnt(0, 1); |
334 | | | 334 | |
335 | l->l_proc = &proc0; | | 335 | l->l_proc = &proc0; |
336 | lwp_update_creds(l); | | 336 | lwp_update_creds(l); |
337 | | | 337 | |
338 | lwpinit_specificdata(); | | 338 | lwpinit_specificdata(); |
339 | lwp_initspecific(&lwp0); | | 339 | lwp_initspecific(&lwp0); |
340 | | | 340 | |
341 | /* Must be called after lwpinit_specificdata */ | | 341 | /* Must be called after lwpinit_specificdata */ |
342 | psref_init(); | | 342 | psref_init(); |
343 | | | 343 | |
344 | threadpools_init(); | | 344 | threadpools_init(); |
345 | | | 345 | |
346 | loginit(); | | 346 | loginit(); |
347 | | | 347 | |
348 | rump_biglock_init(); | | 348 | rump_biglock_init(); |
349 | | | 349 | |
350 | rump_scheduler_init(numcpu); | | 350 | rump_scheduler_init(numcpu); |
351 | /* revert temporary context and schedule a semireal context */ | | 351 | /* revert temporary context and schedule a semireal context */ |
352 | rump_lwproc_curlwp_clear(l); | | 352 | rump_lwproc_curlwp_clear(l); |
353 | initproc = &proc0; /* borrow proc0 before we get initproc started */ | | 353 | initproc = &proc0; /* borrow proc0 before we get initproc started */ |
354 | rump_schedule(); | | 354 | rump_schedule(); |
355 | bootlwp = curlwp; | | 355 | bootlwp = curlwp; |
356 | | | 356 | |
357 | inittimecounter(); | | 357 | inittimecounter(); |
358 | ntp_init(); | | 358 | ntp_init(); |
359 | | | 359 | |
360 | #ifdef KTRACE | | 360 | #ifdef KTRACE |
361 | ktrinit(); | | 361 | ktrinit(); |
362 | #endif | | 362 | #endif |
363 | | | 363 | |
364 | tc_setclock(&bts); | | 364 | tc_setclock(&bts); |
365 | | | 365 | |
366 | extern krwlock_t exec_lock; | | 366 | extern krwlock_t exec_lock; |
367 | rw_init(&exec_lock); | | 367 | rw_init(&exec_lock); |
368 | | | 368 | |
369 | /* we are mostly go. do per-cpu subsystem init */ | | 369 | /* we are mostly go. do per-cpu subsystem init */ |
370 | for (i = 0; i < numcpu; i++) { | | 370 | for (i = 0; i < numcpu; i++) { |
371 | struct cpu_info *ci = cpu_lookup(i); | | 371 | struct cpu_info *ci = cpu_lookup(i); |
372 | | | 372 | |
373 | /* attach non-bootstrap CPUs */ | | 373 | /* attach non-bootstrap CPUs */ |
374 | if (i > 0) { | | 374 | if (i > 0) { |
375 | rump_cpu_attach(ci); | | 375 | rump_cpu_attach(ci); |
376 | ncpu++; | | 376 | ncpu++; |
377 | } | | 377 | } |
378 | snprintf(ci->ci_cpuname, sizeof ci->ci_cpuname, "cpu%d", i); | | 378 | snprintf(ci->ci_cpuname, sizeof ci->ci_cpuname, "cpu%d", i); |
379 | | | 379 | |
380 | callout_init_cpu(ci); | | 380 | callout_init_cpu(ci); |
381 | softint_init(ci); | | 381 | softint_init(ci); |
382 | xc_init_cpu(ci); | | 382 | xc_init_cpu(ci); |
383 | pool_cache_cpu_init(ci); | | 383 | pool_cache_cpu_init(ci); |
384 | selsysinit(ci); | | 384 | selsysinit(ci); |
385 | percpu_init_cpu(ci); | | 385 | percpu_init_cpu(ci); |
386 | | | 386 | |
387 | TAILQ_INIT(&ci->ci_data.cpu_ld_locks); | | 387 | TAILQ_INIT(&ci->ci_data.cpu_ld_locks); |
388 | __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock); | | 388 | __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock); |
389 | | | 389 | |
390 | aprint_verbose("cpu%d at thinair0: rump virtual cpu\n", i); | | 390 | aprint_verbose("cpu%d at thinair0: rump virtual cpu\n", i); |
391 | } | | 391 | } |
392 | ncpuonline = ncpu; | | 392 | ncpuonline = ncpu; |
393 | | | 393 | |
394 | /* Once all CPUs are detected, initialize the per-CPU cprng_fast. */ | | 394 | /* Once all CPUs are detected, initialize the per-CPU cprng_fast. */ |
395 | cprng_fast_init(); | | 395 | cprng_fast_init(); |
396 | | | 396 | |
397 | mp_online = true; | | 397 | mp_online = true; |
398 | | | 398 | |
399 | /* CPUs are up. allow kernel threads to run */ | | 399 | /* CPUs are up. allow kernel threads to run */ |
400 | rump_thread_allow(NULL); | | 400 | rump_thread_allow(NULL); |
401 | | | 401 | |
402 | rnd_init_softint(); | | 402 | rnd_init_softint(); |
403 | | | 403 | |
404 | kqueue_init(); | | 404 | kqueue_init(); |
405 | iostat_init(); | | 405 | iostat_init(); |
406 | fd_sys_init(); | | 406 | fd_sys_init(); |
407 | module_init(); | | 407 | module_init(); |
408 | module_hook_init(); | | 408 | module_hook_init(); |
409 | devsw_init(); | | 409 | devsw_init(); |
410 | pipe_init(); | | 410 | pipe_init(); |
411 | resource_init(); | | 411 | resource_init(); |
412 | procinit_sysctl(); | | 412 | procinit_sysctl(); |
413 | time_init(); | | 413 | time_init(); |
414 | time_init2(); | | 414 | time_init2(); |
415 | config_init(); | | 415 | config_init(); |
416 | | | 416 | |
417 | /* start page baroness */ | | 417 | /* start page baroness */ |
418 | if (rump_threads) { | | 418 | if (rump_threads) { |
419 | if (kthread_create(PRI_PGDAEMON, KTHREAD_MPSAFE, NULL, | | 419 | if (kthread_create(PRI_PGDAEMON, KTHREAD_MPSAFE, NULL, |
420 | uvm_pageout, NULL, &uvm.pagedaemon_lwp, "pdaemon") != 0) | | 420 | uvm_pageout, NULL, &uvm.pagedaemon_lwp, "pdaemon") != 0) |
421 | panic("pagedaemon create failed"); | | 421 | panic("pagedaemon create failed"); |
422 | } else | | 422 | } else |
423 | uvm.pagedaemon_lwp = NULL; /* doesn't match curlwp */ | | 423 | uvm.pagedaemon_lwp = NULL; /* doesn't match curlwp */ |
424 | | | 424 | |
425 | /* process dso's */ | | 425 | /* process dso's */ |
426 | rumpuser_dl_bootstrap(add_linkedin_modules, | | 426 | rumpuser_dl_bootstrap(add_linkedin_modules, |
427 | rump_kernelfsym_load, rump_component_load, add_static_evcnt); | | 427 | rump_kernelfsym_load, rump_component_load, add_static_evcnt); |
428 | | | 428 | |
429 | rump_component_addlocal(); | | 429 | rump_component_addlocal(); |
430 | rump_component_init(RUMP_COMPONENT_KERN); | | 430 | rump_component_init(RUMP_COMPONENT_KERN); |
431 | | | 431 | |
432 | /* initialize factions, if present */ | | 432 | /* initialize factions, if present */ |
433 | rump_component_init(RUMP__FACTION_VFS); | | 433 | rump_component_init(RUMP__FACTION_VFS); |
434 | /* pnbuf_cache is used even without vfs */ | | 434 | /* pnbuf_cache is used even without vfs */ |
435 | if (rump_component_count(RUMP__FACTION_VFS) == 0) { | | 435 | if (rump_component_count(RUMP__FACTION_VFS) == 0) { |
436 | pnbuf_cache = pool_cache_init(MAXPATHLEN, 0, 0, 0, "pnbufpl", | | 436 | pnbuf_cache = pool_cache_init(MAXPATHLEN, 0, 0, 0, "pnbufpl", |
437 | NULL, IPL_NONE, NULL, NULL, NULL); | | 437 | NULL, IPL_NONE, NULL, NULL, NULL); |
438 | } | | 438 | } |
439 | rump_component_init(RUMP__FACTION_NET); | | 439 | rump_component_init(RUMP__FACTION_NET); |
440 | rump_component_init(RUMP__FACTION_DEV); | | 440 | rump_component_init(RUMP__FACTION_DEV); |
441 | KASSERT(rump_component_count(RUMP__FACTION_VFS) <= 1 | | 441 | KASSERT(rump_component_count(RUMP__FACTION_VFS) <= 1 |
442 | && rump_component_count(RUMP__FACTION_NET) <= 1 | | 442 | && rump_component_count(RUMP__FACTION_NET) <= 1 |
443 | && rump_component_count(RUMP__FACTION_DEV) <= 1); | | 443 | && rump_component_count(RUMP__FACTION_DEV) <= 1); |
444 | | | 444 | |
445 | rump_component_init(RUMP_COMPONENT_KERN_VFS); | | 445 | rump_component_init(RUMP_COMPONENT_KERN_VFS); |
446 | | | 446 | |
447 | /* | | 447 | /* |
448 | * if we initialized the tty component above, the tyttymtx is | | 448 | * if we initialized the tty component above, the tyttymtx is |
449 | * now initialized. otherwise, we need to initialize it. | | 449 | * now initialized. otherwise, we need to initialize it. |
450 | */ | | 450 | */ |
451 | if (!rump_ttycomponent) | | 451 | if (!rump_ttycomponent) |
452 | mutex_init(&tty_lock, MUTEX_DEFAULT, IPL_VM); | | 452 | mutex_init(&tty_lock, MUTEX_DEFAULT, IPL_VM); |
453 | | | 453 | |
454 | cold = 0; | | 454 | cold = 0; |
455 | | | 455 | |
456 | sysctl_finalize(); | | 456 | sysctl_finalize(); |
457 | | | 457 | |
458 | module_init_class(MODULE_CLASS_ANY); | | 458 | module_init_class(MODULE_CLASS_ANY); |
459 | | | 459 | |
460 | if (rumpuser_getparam(RUMPUSER_PARAM_HOSTNAME, | | 460 | if (rumpuser_getparam(RUMPUSER_PARAM_HOSTNAME, |
461 | hostname, MAXHOSTNAMELEN) != 0) { | | 461 | hostname, MAXHOSTNAMELEN) != 0) { |
462 | panic("mandatory hypervisor configuration (HOSTNAME) missing"); | | 462 | panic("mandatory hypervisor configuration (HOSTNAME) missing"); |
463 | } | | 463 | } |
464 | hostnamelen = strlen(hostname); | | 464 | hostnamelen = strlen(hostname); |
465 | | | 465 | |
466 | sigemptyset(&sigcantmask); | | 466 | sigemptyset(&sigcantmask); |
467 | | | 467 | |
468 | if (rump_threads) | | 468 | if (rump_threads) |
469 | vmem_rehash_start(); | | 469 | vmem_rehash_start(); |
470 | | | 470 | |
471 | /* | | 471 | /* |
472 | * Create init (proc 1), used to attach implicit threads in rump. | | 472 | * Create init (proc 1), used to attach implicit threads in rump. |
473 | * (note: must be done after vfsinit to get cwdi) | | 473 | * (note: must be done after vfsinit to get cwdi) |
474 | */ | | 474 | */ |
475 | initlwp = rump__lwproc_alloclwp(NULL); | | 475 | initlwp = rump__lwproc_alloclwp(NULL); |
476 | mutex_enter(proc_lock); | | 476 | mutex_enter(proc_lock); |
477 | initproc = proc_find_raw(1); | | 477 | initproc = proc_find_raw(1); |
478 | mutex_exit(proc_lock); | | 478 | mutex_exit(proc_lock); |
479 | if (initproc == NULL) | | 479 | if (initproc == NULL) |
480 | panic("where in the world is initproc?"); | | 480 | panic("where in the world is initproc?"); |
481 | strlcpy(initproc->p_comm, "rumplocal", sizeof(initproc->p_comm)); | | 481 | strlcpy(initproc->p_comm, "rumplocal", sizeof(initproc->p_comm)); |
482 | | | 482 | |
483 | rump_component_init(RUMP_COMPONENT_POSTINIT); | | 483 | rump_component_init(RUMP_COMPONENT_POSTINIT); |
484 | | | 484 | |
485 | /* load syscalls */ | | 485 | /* load syscalls */ |
486 | rump_component_init(RUMP_COMPONENT_SYSCALL); | | 486 | rump_component_init(RUMP_COMPONENT_SYSCALL); |
487 | | | 487 | |
488 | /* component inits done */ | | 488 | /* component inits done */ |
489 | bootlwp = NULL; | | 489 | bootlwp = NULL; |
490 | | | 490 | |
491 | /* open 0/1/2 for init */ | | 491 | /* open 0/1/2 for init */ |
492 | KASSERT(rump_lwproc_curlwp() == NULL); | | 492 | KASSERT(rump_lwproc_curlwp() == NULL); |
493 | rump_lwproc_switch(initlwp); | | 493 | rump_lwproc_switch(initlwp); |
494 | rump_consdev_init(); | | 494 | rump_consdev_init(); |
495 | rump_lwproc_switch(NULL); | | 495 | rump_lwproc_switch(NULL); |
496 | | | 496 | |
497 | /* release cpu */ | | 497 | /* release cpu */ |
498 | rump_unschedule(); | | 498 | rump_unschedule(); |
499 | | | 499 | |
500 | return 0; | | 500 | return 0; |
501 | } | | 501 | } |
502 | /* historic compat */ | | 502 | /* historic compat */ |
503 | __strong_alias(rump__init,rump_init); | | 503 | __strong_alias(rump__init,rump_init); |
504 | | | 504 | |
505 | static int compcounter[RUMP_COMPONENT_MAX]; | | 505 | static int compcounter[RUMP_COMPONENT_MAX]; |
506 | static int compinited[RUMP_COMPONENT_MAX]; | | 506 | static int compinited[RUMP_COMPONENT_MAX]; |
507 | | | 507 | |
508 | /* | | 508 | /* |
509 | * Yea, this is O(n^2), but we're only looking at a handful of components. | | 509 | * Yea, this is O(n^2), but we're only looking at a handful of components. |
510 | * Components are always initialized from the thread that called rump_init(). | | 510 | * Components are always initialized from the thread that called rump_init(). |
511 | */ | | 511 | */ |
512 | static LIST_HEAD(, rump_component) rchead = LIST_HEAD_INITIALIZER(rchead); | | 512 | static LIST_HEAD(, rump_component) rchead = LIST_HEAD_INITIALIZER(rchead); |
513 | | | 513 | |
514 | #ifdef RUMP_USE_CTOR | | 514 | #ifdef RUMP_USE_CTOR |
515 | struct modinfo_boot_chain modinfo_boot_chain \ | | 515 | struct modinfo_boot_chain modinfo_boot_chain \ |
516 | = LIST_HEAD_INITIALIZER(modinfo_boot_chain); | | 516 | = LIST_HEAD_INITIALIZER(modinfo_boot_chain); |
517 | | | 517 | |
518 | static void | | 518 | static void |
519 | rump_component_addlocal(void) | | 519 | rump_component_addlocal(void) |
520 | { | | 520 | { |
521 | struct modinfo_chain *mc; | | 521 | struct modinfo_chain *mc; |
522 | | | 522 | |
523 | while ((mc = LIST_FIRST(&modinfo_boot_chain)) != NULL) { | | 523 | while ((mc = LIST_FIRST(&modinfo_boot_chain)) != NULL) { |
524 | LIST_REMOVE(mc, mc_entries); | | 524 | LIST_REMOVE(mc, mc_entries); |
525 | module_builtin_add(&mc->mc_info, 1, false); | | 525 | module_builtin_add(&mc->mc_info, 1, false); |
526 | } | | 526 | } |
527 | } | | 527 | } |
528 | | | 528 | |
529 | #else /* RUMP_USE_CTOR */ | | 529 | #else /* RUMP_USE_CTOR */ |
530 | | | 530 | |
531 | static void | | 531 | static void |
532 | rump_component_addlocal(void) | | 532 | rump_component_addlocal(void) |
533 | { | | 533 | { |
534 | __link_set_decl(rump_components, struct rump_component); | | 534 | __link_set_decl(rump_components, struct rump_component); |
535 | struct rump_component *const *rc; | | 535 | struct rump_component *const *rc; |
536 | | | 536 | |
537 | __link_set_foreach(rc, rump_components) { | | 537 | __link_set_foreach(rc, rump_components) { |
538 | rump_component_load(*rc); | | 538 | rump_component_load(*rc); |
539 | } | | 539 | } |
540 | } | | 540 | } |
541 | #endif /* RUMP_USE_CTOR */ | | 541 | #endif /* RUMP_USE_CTOR */ |
542 | | | 542 | |
543 | void | | 543 | void |
544 | rump_component_load(const struct rump_component *rc_const) | | 544 | rump_component_load(const struct rump_component *rc_const) |
545 | { | | 545 | { |
546 | struct rump_component *rc, *rc_iter; | | 546 | struct rump_component *rc, *rc_iter; |
547 | | | 547 | |
548 | /* time for rump component loading and unloading has passed */ | | 548 | /* time for rump component loading and unloading has passed */ |
549 | if (!cold) | | 549 | if (!cold) |
550 | return; | | 550 | return; |
551 | | | 551 | |
552 | /* | | 552 | /* |
553 | * XXX: this is ok since the "const" was removed from the | | 553 | * XXX: this is ok since the "const" was removed from the |
554 | * definition of RUMP_COMPONENT(). | | 554 | * definition of RUMP_COMPONENT(). |
555 | * | | 555 | * |
556 | * However, to preserve the hypercall interface, the const | | 556 | * However, to preserve the hypercall interface, the const |
557 | * remains here. This can be fixed in the next hypercall revision. | | 557 | * remains here. This can be fixed in the next hypercall revision. |
558 | */ | | 558 | */ |
559 | rc = __UNCONST(rc_const); | | 559 | rc = __UNCONST(rc_const); |
560 | | | 560 | |
561 | KASSERT(!rump_inited || curlwp == bootlwp); | | 561 | KASSERT(!rump_inited || curlwp == bootlwp); |
562 | | | 562 | |
563 | LIST_FOREACH(rc_iter, &rchead, rc_entries) { | | 563 | LIST_FOREACH(rc_iter, &rchead, rc_entries) { |
564 | if (rc_iter == rc) | | 564 | if (rc_iter == rc) |
565 | return; | | 565 | return; |
566 | } | | 566 | } |
567 | | | 567 | |
568 | LIST_INSERT_HEAD(&rchead, rc, rc_entries); | | 568 | LIST_INSERT_HEAD(&rchead, rc, rc_entries); |
569 | KASSERT(rc->rc_type < RUMP_COMPONENT_MAX); | | 569 | KASSERT(rc->rc_type < RUMP_COMPONENT_MAX); |
570 | compcounter[rc->rc_type]++; | | 570 | compcounter[rc->rc_type]++; |
571 | } | | 571 | } |
572 | | | 572 | |
573 | void | | 573 | void |
574 | rump_component_unload(struct rump_component *rc) | | 574 | rump_component_unload(struct rump_component *rc) |
575 | { | | 575 | { |
576 | | | 576 | |
577 | /* | | 577 | /* |
578 | * Checking for cold is enough because rump_init() both | | 578 | * Checking for cold is enough because rump_init() both |
579 | * flips it and handles component loading. | | 579 | * flips it and handles component loading. |
580 | */ | | 580 | */ |
581 | if (!cold) | | 581 | if (!cold) |
582 | return; | | 582 | return; |
583 | | | 583 | |
584 | LIST_REMOVE(rc, rc_entries); | | 584 | LIST_REMOVE(rc, rc_entries); |
585 | } | | 585 | } |
586 | | | 586 | |
587 | int | | 587 | int |
588 | rump_component_count(enum rump_component_type type) | | 588 | rump_component_count(enum rump_component_type type) |
589 | { | | 589 | { |
590 | | | 590 | |
591 | KASSERT(curlwp == bootlwp); | | 591 | KASSERT(curlwp == bootlwp); |
592 | KASSERT(type < RUMP_COMPONENT_MAX); | | 592 | KASSERT(type < RUMP_COMPONENT_MAX); |
593 | return compcounter[type]; | | 593 | return compcounter[type]; |
594 | } | | 594 | } |
595 | | | 595 | |
596 | void | | 596 | void |
597 | rump_component_init(enum rump_component_type type) | | 597 | rump_component_init(enum rump_component_type type) |
598 | { | | 598 | { |
599 | const struct rump_component *rc, *rc_safe; | | 599 | const struct rump_component *rc, *rc_safe; |
600 | | | 600 | |
601 | KASSERT(curlwp == bootlwp); | | 601 | KASSERT(curlwp == bootlwp); |
602 | KASSERT(!compinited[type]); | | 602 | KASSERT(!compinited[type]); |
603 | LIST_FOREACH_SAFE(rc, &rchead, rc_entries, rc_safe) { | | 603 | LIST_FOREACH_SAFE(rc, &rchead, rc_entries, rc_safe) { |
604 | if (rc->rc_type == type) { | | 604 | if (rc->rc_type == type) { |
605 | rc->rc_init(); | | 605 | rc->rc_init(); |
606 | LIST_REMOVE(rc, rc_entries); | | 606 | LIST_REMOVE(rc, rc_entries); |
607 | } | | 607 | } |
608 | } | | 608 | } |
609 | compinited[type] = 1; | | 609 | compinited[type] = 1; |
610 | } | | 610 | } |
611 | | | 611 | |
612 | /* | | 612 | /* |
613 | * Initialize a module which has already been loaded and linked | | 613 | * Initialize a module which has already been loaded and linked |
614 | * with dlopen(). This is fundamentally the same as a builtin module. | | 614 | * with dlopen(). This is fundamentally the same as a builtin module. |
615 | * | | 615 | * |
616 | * XXX: this interface does not really work in the RUMP_USE_CTOR case, | | 616 | * XXX: this interface does not really work in the RUMP_USE_CTOR case, |
617 | * but I'm not sure it's anything to cry about. In feeling blue, | | 617 | * but I'm not sure it's anything to cry about. In feeling blue, |
618 | * things could somehow be handled via modinfo_boot_chain. | | 618 | * things could somehow be handled via modinfo_boot_chain. |
619 | */ | | 619 | */ |
620 | int | | 620 | int |
621 | rump_module_init(const struct modinfo * const *mip, size_t nmodinfo) | | 621 | rump_module_init(const struct modinfo * const *mip, size_t nmodinfo) |
622 | { | | 622 | { |
623 | | | 623 | |
624 | return module_builtin_add(mip, nmodinfo, true); | | 624 | return module_builtin_add(mip, nmodinfo, true); |
625 | } | | 625 | } |
626 | | | 626 | |
627 | /* | | 627 | /* |
628 | * Finish module (flawless victory, fatality!). | | 628 | * Finish module (flawless victory, fatality!). |
629 | */ | | 629 | */ |
630 | int | | 630 | int |
631 | rump_module_fini(const struct modinfo *mi) | | 631 | rump_module_fini(const struct modinfo *mi) |
632 | { | | 632 | { |
633 | | | 633 | |
634 | return module_builtin_remove(mi, true); | | 634 | return module_builtin_remove(mi, true); |
635 | } | | 635 | } |
636 | | | 636 | |
637 | /* | | 637 | /* |
638 | * Add loaded and linked module to the builtin list. It will | | 638 | * Add loaded and linked module to the builtin list. It will |
639 | * later be initialized with module_init_class(). | | 639 | * later be initialized with module_init_class(). |
640 | */ | | 640 | */ |
641 | | | 641 | |
642 | static void | | 642 | static void |
643 | add_linkedin_modules(const struct modinfo * const *mip, size_t nmodinfo) | | 643 | add_linkedin_modules(const struct modinfo * const *mip, size_t nmodinfo) |
644 | { | | 644 | { |
645 | | | 645 | |
646 | module_builtin_add(mip, nmodinfo, false); | | 646 | module_builtin_add(mip, nmodinfo, false); |
647 | } | | 647 | } |
648 | | | 648 | |
649 | /* | | 649 | /* |
650 | * Add an evcnt. | | 650 | * Add an evcnt. |
651 | */ | | 651 | */ |
652 | static void | | 652 | static void |
653 | add_static_evcnt(struct evcnt *ev) | | 653 | add_static_evcnt(struct evcnt *ev) |
654 | { | | 654 | { |
655 | | | 655 | |
656 | evcnt_attach_static(ev); | | 656 | evcnt_attach_static(ev); |
657 | } | | 657 | } |
658 | | | 658 | |
659 | int | | 659 | int |
660 | rump_kernelfsym_load(void *symtab, uint64_t symsize, | | 660 | rump_kernelfsym_load(void *symtab, uint64_t symsize, |
661 | char *strtab, uint64_t strsize) | | 661 | char *strtab, uint64_t strsize) |
662 | { | | 662 | { |
663 | static int inited = 0; | | 663 | static int inited = 0; |
664 | Elf64_Ehdr ehdr; | | 664 | Elf64_Ehdr ehdr; |
665 | | | 665 | |
666 | if (inited) | | 666 | if (inited) |
667 | return EBUSY; | | 667 | return EBUSY; |
668 | inited = 1; | | 668 | inited = 1; |
669 | | | 669 | |
670 | /* | | 670 | /* |
671 | * Use 64bit header since it's bigger. Shouldn't make a | | 671 | * Use 64bit header since it's bigger. Shouldn't make a |
672 | * difference, since we're passing in all zeroes anyway. | | 672 | * difference, since we're passing in all zeroes anyway. |
673 | */ | | 673 | */ |
674 | memset(&ehdr, 0, sizeof(ehdr)); | | 674 | memset(&ehdr, 0, sizeof(ehdr)); |
675 | ksyms_addsyms_explicit(&ehdr, symtab, symsize, strtab, strsize); | | 675 | ksyms_addsyms_explicit(&ehdr, symtab, symsize, strtab, strsize); |
676 | | | 676 | |
677 | return 0; | | 677 | return 0; |
678 | } | | 678 | } |
679 | | | 679 | |
680 | int | | 680 | int |
681 | rump_boot_gethowto() | | 681 | rump_boot_gethowto() |
682 | { | | 682 | { |
683 | | | 683 | |
684 | return boothowto; | | 684 | return boothowto; |
685 | } | | 685 | } |
686 | | | 686 | |
687 | void | | 687 | void |
688 | rump_boot_sethowto(int howto) | | 688 | rump_boot_sethowto(int howto) |
689 | { | | 689 | { |
690 | | | 690 | |
691 | boothowto = howto; | | 691 | boothowto = howto; |
692 | } | | 692 | } |
693 | | | 693 | |
694 | int | | 694 | int |
695 | rump_getversion(void) | | 695 | rump_getversion(void) |
696 | { | | 696 | { |
697 | | | 697 | |
698 | return __NetBSD_Version__; | | 698 | return __NetBSD_Version__; |
699 | } | | 699 | } |
700 | /* compat */ | | 700 | /* compat */ |
701 | __strong_alias(rump_pub_getversion,rump_getversion); | | 701 | __strong_alias(rump_pub_getversion,rump_getversion); |
702 | | | 702 | |
703 | /* | | 703 | /* |
704 | * Note: may be called unscheduled. Not fully safe since no locking | | 704 | * Note: may be called unscheduled. Not fully safe since no locking |
705 | * of allevents (currently that's not even available). | | 705 | * of allevents (currently that's not even available). |
706 | */ | | 706 | */ |
707 | void | | 707 | void |
708 | rump_printevcnts() | | 708 | rump_printevcnts() |
709 | { | | 709 | { |
710 | struct evcnt *ev; | | 710 | struct evcnt *ev; |
711 | | | 711 | |
712 | TAILQ_FOREACH(ev, &allevents, ev_list) | | 712 | TAILQ_FOREACH(ev, &allevents, ev_list) |
713 | rumpuser_dprintf("%s / %s: %" PRIu64 "\n", | | 713 | rumpuser_dprintf("%s / %s: %" PRIu64 "\n", |
714 | ev->ev_group, ev->ev_name, ev->ev_count); | | 714 | ev->ev_group, ev->ev_name, ev->ev_count); |
715 | } | | 715 | } |
716 | | | 716 | |
717 | /* | | 717 | /* |
718 | * If you use this interface ... well ... all bets are off. | | 718 | * If you use this interface ... well ... all bets are off. |
719 | * The original purpose is for the p2k fs server library to be | | 719 | * The original purpose is for the p2k fs server library to be |
720 | * able to use the same pid/lid for VOPs as the host kernel. | | 720 | * able to use the same pid/lid for VOPs as the host kernel. |
721 | */ | | 721 | */ |
722 | void | | 722 | void |
723 | rump_allbetsareoff_setid(pid_t pid, int lid) | | 723 | rump_allbetsareoff_setid(pid_t pid, int lid) |
724 | { | | 724 | { |
725 | struct lwp *l = curlwp; | | 725 | struct lwp *l = curlwp; |
726 | struct proc *p = l->l_proc; | | 726 | struct proc *p = l->l_proc; |
727 | | | 727 | |
728 | l->l_lid = lid; | | 728 | l->l_lid = lid; |
729 | p->p_pid = pid; | | 729 | p->p_pid = pid; |
730 | } | | 730 | } |
731 | | | 731 | |
732 | static void | | 732 | static void |
733 | ipiemu(void *a1, void *a2) | | 733 | ipiemu(void *a1, void *a2) |
734 | { | | 734 | { |
735 | | | 735 | |
736 | xc__highpri_intr(NULL); | | 736 | xc__highpri_intr(NULL); |
737 | } | | 737 | } |
738 | | | 738 | |
739 | void | | 739 | void |
740 | rump_xc_highpri(struct cpu_info *ci) | | 740 | rump_xc_highpri(struct cpu_info *ci) |
741 | { | | 741 | { |
742 | | | 742 | |
743 | if (ci) | | 743 | if (ci) |
744 | xc_unicast(0, ipiemu, NULL, NULL, ci); | | 744 | xc_unicast(0, ipiemu, NULL, NULL, ci); |
745 | else | | 745 | else |
746 | xc_broadcast(0, ipiemu, NULL, NULL); | | 746 | xc_broadcast(0, ipiemu, NULL, NULL); |
747 | } | | 747 | } |
748 | | | 748 | |
749 | int | | 749 | int |
750 | rump_syscall(int num, void *data, size_t dlen, register_t *retval) | | 750 | rump_syscall(int num, void *data, size_t dlen, register_t *retval) |
751 | { | | 751 | { |
752 | struct proc *p; | | 752 | struct proc *p; |
753 | struct emul *e; | | 753 | struct emul *e; |
754 | struct sysent *callp; | | 754 | struct sysent *callp; |
755 | const int *etrans = NULL; | | 755 | const int *etrans = NULL; |
756 | int rv; | | 756 | int rv; |
757 | | | 757 | |
758 | rump_schedule(); | | 758 | rump_schedule(); |
759 | p = curproc; | | 759 | p = curproc; |
760 | e = p->p_emul; | | 760 | e = p->p_emul; |
761 | #ifndef __HAVE_MINIMAL_EMUL | | 761 | #ifndef __HAVE_MINIMAL_EMUL |
762 | KASSERT(num > 0 && num < e->e_nsysent); | | 762 | KASSERT(num > 0 && num < e->e_nsysent); |
763 | #endif | | 763 | #endif |
764 | callp = e->e_sysent + num; | | 764 | callp = e->e_sysent + num; |
765 | | | 765 | |
766 | rv = sy_invoke(callp, curlwp, data, retval, num); | | 766 | rv = sy_invoke(callp, curlwp, data, retval, num); |
767 | | | 767 | |
768 | /* | | 768 | /* |
769 | * I hope that (!__HAVE_MINIMAL_EMUL || __HAVE_SYSCALL_INTERN) is | | 769 | * I hope that (!__HAVE_MINIMAL_EMUL || __HAVE_SYSCALL_INTERN) is |
770 | * an invariant ... | | 770 | * an invariant ... |
771 | */ | | 771 | */ |
772 | #if !defined(__HAVE_MINIMAL_EMUL) | | 772 | #if !defined(__HAVE_MINIMAL_EMUL) |
773 | etrans = e->e_errno; | | 773 | etrans = e->e_errno; |
774 | #elif defined(__HAVE_SYSCALL_INTERN) | | 774 | #elif defined(__HAVE_SYSCALL_INTERN) |
775 | etrans = p->p_emuldata; | | 775 | etrans = p->p_emuldata; |
776 | #endif | | 776 | #endif |
777 | | | 777 | |
778 | if (etrans) { | | 778 | if (etrans) { |
779 | rv = etrans[rv]; | | 779 | rv = etrans[rv]; |
780 | /* | | 780 | /* |
781 | * XXX: small hack since Linux etrans vectors on some | | 781 | * XXX: small hack since Linux etrans vectors on some |
782 | * archs contain negative errnos, but rump_syscalls | | 782 | * archs contain negative errnos, but rump_syscalls |
783 | * uses the -1 + errno ABI. Note that these | | 783 | * uses the -1 + errno ABI. Note that these |
784 | * negative values are always the result of translation, | | 784 | * negative values are always the result of translation, |
785 | * otherwise the above translation method would not | | 785 | * otherwise the above translation method would not |
786 | * work very well. | | 786 | * work very well. |
787 | */ | | 787 | */ |
788 | if (rv < 0) | | 788 | if (rv < 0) |
789 | rv = -rv; | | 789 | rv = -rv; |
790 | } | | 790 | } |
791 | rump_unschedule(); | | 791 | rump_unschedule(); |
792 | | | 792 | |
793 | return rv; | | 793 | return rv; |
794 | } | | 794 | } |
795 | | | 795 | |
796 | void | | 796 | void |
797 | rump_syscall_boot_establish(const struct rump_onesyscall *calls, size_t ncall) | | 797 | rump_syscall_boot_establish(const struct rump_onesyscall *calls, size_t ncall) |
798 | { | | 798 | { |
799 | struct sysent *callp; | | 799 | struct sysent *callp; |
800 | size_t i; | | 800 | size_t i; |
801 | | | 801 | |
802 | for (i = 0; i < ncall; i++) { | | 802 | for (i = 0; i < ncall; i++) { |
803 | callp = rump_sysent + calls[i].ros_num; | | 803 | callp = rump_sysent + calls[i].ros_num; |
804 | KASSERT(bootlwp != NULL | | 804 | KASSERT(bootlwp != NULL |
805 | && callp->sy_call == (sy_call_t *)(void *)enosys); | | 805 | && callp->sy_call == (sy_call_t *)(void *)enosys); |
806 | callp->sy_call = calls[i].ros_handler; | | 806 | callp->sy_call = calls[i].ros_handler; |
807 | } | | 807 | } |
808 | } | | 808 | } |
809 | | | 809 | |
810 | struct rump_boot_etfs *ebstart; | | 810 | struct rump_boot_etfs *ebstart; |
811 | void | | 811 | void |
812 | rump_boot_etfs_register(struct rump_boot_etfs *eb) | | 812 | rump_boot_etfs_register(struct rump_boot_etfs *eb) |
813 | { | | 813 | { |
814 | | | 814 | |
815 | /* | | 815 | /* |
816 | * Could use atomics, but, since caller would need to synchronize | | 816 | * Could use atomics, but, since caller would need to synchronize |
817 | * against calling rump_init() anyway, easier to just specify the | | 817 | * against calling rump_init() anyway, easier to just specify the |
818 | * interface as "caller serializes". This solve-by-specification | | 818 | * interface as "caller serializes". This solve-by-specification |
819 | * approach avoids the grey area of using atomics before rump_init() | | 819 | * approach avoids the grey area of using atomics before rump_init() |
820 | * runs. | | 820 | * runs. |
821 | */ | | 821 | */ |
822 | eb->_eb_next = ebstart; | | 822 | eb->_eb_next = ebstart; |
823 | eb->eb_status = -1; | | 823 | eb->eb_status = -1; |
824 | ebstart = eb; | | 824 | ebstart = eb; |
825 | } | | 825 | } |