| @@ -1,707 +1,697 @@ | | | @@ -1,707 +1,697 @@ |
1 | /* $NetBSD: booke_machdep.c,v 1.32 2020/07/06 10:08:16 rin Exp $ */ | | 1 | /* $NetBSD: booke_machdep.c,v 1.33 2021/03/30 14:29:54 rin Exp $ */ |
2 | /*- | | 2 | /*- |
3 | * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. | | 3 | * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. |
4 | * All rights reserved. | | 4 | * All rights reserved. |
5 | * | | 5 | * |
6 | * This code is derived from software contributed to The NetBSD Foundation | | 6 | * This code is derived from software contributed to The NetBSD Foundation |
7 | * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects | | 7 | * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects |
8 | * Agency and which was developed by Matt Thomas of 3am Software Foundry. | | 8 | * Agency and which was developed by Matt Thomas of 3am Software Foundry. |
9 | * | | 9 | * |
10 | * This material is based upon work supported by the Defense Advanced Research | | 10 | * This material is based upon work supported by the Defense Advanced Research |
11 | * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under | | 11 | * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under |
12 | * Contract No. N66001-09-C-2073. | | 12 | * Contract No. N66001-09-C-2073. |
13 | * Approved for Public Release, Distribution Unlimited | | 13 | * Approved for Public Release, Distribution Unlimited |
14 | * | | 14 | * |
15 | * Redistribution and use in source and binary forms, with or without | | 15 | * Redistribution and use in source and binary forms, with or without |
16 | * modification, are permitted provided that the following conditions | | 16 | * modification, are permitted provided that the following conditions |
17 | * are met: | | 17 | * are met: |
18 | * 1. Redistributions of source code must retain the above copyright | | 18 | * 1. Redistributions of source code must retain the above copyright |
19 | * notice, this list of conditions and the following disclaimer. | | 19 | * notice, this list of conditions and the following disclaimer. |
20 | * 2. Redistributions in binary form must reproduce the above copyright | | 20 | * 2. Redistributions in binary form must reproduce the above copyright |
21 | * notice, this list of conditions and the following disclaimer in the | | 21 | * notice, this list of conditions and the following disclaimer in the |
22 | * documentation and/or other materials provided with the distribution. | | 22 | * documentation and/or other materials provided with the distribution. |
23 | * | | 23 | * |
24 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS | | 24 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
25 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED | | 25 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
26 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | | 26 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS | | 27 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | | 28 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | | 29 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | | 30 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
31 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | | 31 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
32 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | | 32 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
33 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | | 33 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
34 | * POSSIBILITY OF SUCH DAMAGE. | | 34 | * POSSIBILITY OF SUCH DAMAGE. |
35 | */ | | 35 | */ |
36 | | | 36 | |
37 | #define __INTR_PRIVATE | | 37 | #define __INTR_PRIVATE |
38 | #define _POWERPC_BUS_DMA_PRIVATE | | 38 | #define _POWERPC_BUS_DMA_PRIVATE |
39 | | | 39 | |
40 | #include <sys/cdefs.h> | | 40 | #include <sys/cdefs.h> |
41 | __KERNEL_RCSID(0, "$NetBSD: booke_machdep.c,v 1.32 2020/07/06 10:08:16 rin Exp $"); | | 41 | __KERNEL_RCSID(0, "$NetBSD: booke_machdep.c,v 1.33 2021/03/30 14:29:54 rin Exp $"); |
42 | | | 42 | |
43 | #include "ksyms.h" | | 43 | #include "ksyms.h" |
44 | | | 44 | |
45 | #ifdef _KERNEL_OPT | | 45 | #ifdef _KERNEL_OPT |
46 | #include "opt_ddb.h" | | 46 | #include "opt_ddb.h" |
47 | #include "opt_modular.h" | | 47 | #include "opt_modular.h" |
48 | #include "opt_multiprocessor.h" | | 48 | #include "opt_multiprocessor.h" |
49 | #endif | | 49 | #endif |
50 | | | 50 | |
51 | #include <sys/param.h> | | 51 | #include <sys/param.h> |
52 | #include <sys/cpu.h> | | 52 | #include <sys/cpu.h> |
53 | #include <sys/device.h> | | 53 | #include <sys/device.h> |
54 | #include <sys/intr.h> | | 54 | #include <sys/intr.h> |
55 | #include <sys/mount.h> | | 55 | #include <sys/mount.h> |
56 | #include <sys/msgbuf.h> | | 56 | #include <sys/msgbuf.h> |
57 | #include <sys/kernel.h> | | 57 | #include <sys/kernel.h> |
58 | #include <sys/reboot.h> | | 58 | #include <sys/reboot.h> |
59 | #include <sys/bus.h> | | 59 | #include <sys/bus.h> |
60 | #include <sys/cpu.h> | | 60 | #include <sys/cpu.h> |
61 | | | 61 | |
62 | #include <uvm/uvm_extern.h> | | 62 | #include <uvm/uvm_extern.h> |
63 | | | 63 | |
64 | #include <dev/cons.h> | | 64 | #include <dev/cons.h> |
65 | | | 65 | |
66 | #include <powerpc/pcb.h> | | 66 | #include <powerpc/pcb.h> |
67 | #include <powerpc/spr.h> | | 67 | #include <powerpc/spr.h> |
68 | #include <powerpc/booke/spr.h> | | 68 | #include <powerpc/booke/spr.h> |
69 | #include <powerpc/booke/cpuvar.h> | | 69 | #include <powerpc/booke/cpuvar.h> |
70 | | | 70 | |
71 | /* | | 71 | /* |
72 | * Global variables used here and there | | 72 | * Global variables used here and there |
73 | */ | | 73 | */ |
74 | paddr_t msgbuf_paddr; | | 74 | paddr_t msgbuf_paddr; |
75 | psize_t pmemsize; | | 75 | psize_t pmemsize; |
76 | struct vm_map *phys_map; | | 76 | struct vm_map *phys_map; |
77 | | | 77 | |
78 | #ifdef MODULAR | | 78 | #ifdef MODULAR |
79 | register_t cpu_psluserset = PSL_USERSET; | | 79 | register_t cpu_psluserset = PSL_USERSET; |
80 | register_t cpu_pslusermod = PSL_USERMOD; | | 80 | register_t cpu_pslusermod = PSL_USERMOD; |
81 | register_t cpu_pslusermask = PSL_USERMASK; | | 81 | register_t cpu_pslusermask = PSL_USERMASK; |
82 | #endif | | 82 | #endif |
83 | | | 83 | |
84 | static bus_addr_t booke_dma_phys_to_bus_mem(bus_dma_tag_t, bus_addr_t); | | 84 | static bus_addr_t booke_dma_phys_to_bus_mem(bus_dma_tag_t, bus_addr_t); |
85 | static bus_addr_t booke_dma_bus_mem_to_phys(bus_dma_tag_t, bus_addr_t); | | 85 | static bus_addr_t booke_dma_bus_mem_to_phys(bus_dma_tag_t, bus_addr_t); |
86 | | | 86 | |
87 | | | 87 | |
88 | struct powerpc_bus_dma_tag booke_bus_dma_tag = { | | 88 | struct powerpc_bus_dma_tag booke_bus_dma_tag = { |
89 | ._dmamap_create = _bus_dmamap_create, | | 89 | ._dmamap_create = _bus_dmamap_create, |
90 | ._dmamap_destroy = _bus_dmamap_destroy, | | 90 | ._dmamap_destroy = _bus_dmamap_destroy, |
91 | ._dmamap_load = _bus_dmamap_load, | | 91 | ._dmamap_load = _bus_dmamap_load, |
92 | ._dmamap_load_mbuf = _bus_dmamap_load_mbuf, | | 92 | ._dmamap_load_mbuf = _bus_dmamap_load_mbuf, |
93 | ._dmamap_load_uio = _bus_dmamap_load_uio, | | 93 | ._dmamap_load_uio = _bus_dmamap_load_uio, |
94 | ._dmamap_load_raw = _bus_dmamap_load_raw, | | 94 | ._dmamap_load_raw = _bus_dmamap_load_raw, |
95 | ._dmamap_unload = _bus_dmamap_unload, | | 95 | ._dmamap_unload = _bus_dmamap_unload, |
96 | /* | | 96 | /* |
97 | * The caches on BookE are coherent so we don't need to do any special | | 97 | * The caches on BookE are coherent so we don't need to do any special |
98 | * cache synchronization. | | 98 | * cache synchronization. |
99 | */ | | 99 | */ |
100 | //._dmamap_sync = _bus_dmamap_sync, | | 100 | //._dmamap_sync = _bus_dmamap_sync, |
101 | ._dmamem_alloc = _bus_dmamem_alloc, | | 101 | ._dmamem_alloc = _bus_dmamem_alloc, |
102 | ._dmamem_free = _bus_dmamem_free, | | 102 | ._dmamem_free = _bus_dmamem_free, |
103 | ._dmamem_map = _bus_dmamem_map, | | 103 | ._dmamem_map = _bus_dmamem_map, |
104 | ._dmamem_unmap = _bus_dmamem_unmap, | | 104 | ._dmamem_unmap = _bus_dmamem_unmap, |
105 | ._dmamem_mmap = _bus_dmamem_mmap, | | 105 | ._dmamem_mmap = _bus_dmamem_mmap, |
106 | ._dma_phys_to_bus_mem = booke_dma_phys_to_bus_mem, | | 106 | ._dma_phys_to_bus_mem = booke_dma_phys_to_bus_mem, |
107 | ._dma_bus_mem_to_phys = booke_dma_bus_mem_to_phys, | | 107 | ._dma_bus_mem_to_phys = booke_dma_bus_mem_to_phys, |
108 | }; | | 108 | }; |
109 | | | 109 | |
110 | static bus_addr_t | | 110 | static bus_addr_t |
111 | booke_dma_phys_to_bus_mem(bus_dma_tag_t t, bus_addr_t a) | | 111 | booke_dma_phys_to_bus_mem(bus_dma_tag_t t, bus_addr_t a) |
112 | { | | 112 | { |
113 | return a; | | 113 | return a; |
114 | } | | 114 | } |
115 | | | 115 | |
116 | static bus_addr_t | | 116 | static bus_addr_t |
117 | booke_dma_bus_mem_to_phys(bus_dma_tag_t t, bus_addr_t a) | | 117 | booke_dma_bus_mem_to_phys(bus_dma_tag_t t, bus_addr_t a) |
118 | { | | 118 | { |
119 | return a; | | 119 | return a; |
120 | } | | 120 | } |
121 | | | 121 | |
122 | struct cpu_md_ops cpu_md_ops; | | 122 | struct cpu_md_ops cpu_md_ops; |
123 | | | 123 | |
124 | struct cpu_softc cpu_softc[] = { | | 124 | struct cpu_softc cpu_softc[] = { |
125 | [0] = { | | 125 | [0] = { |
126 | .cpu_ci = &cpu_info[0], | | 126 | .cpu_ci = &cpu_info[0], |
127 | }, | | 127 | }, |
128 | #ifdef MULTIPROCESSOR | | 128 | #ifdef MULTIPROCESSOR |
129 | [CPU_MAXNUM-1] = { | | 129 | [CPU_MAXNUM-1] = { |
130 | .cpu_ci = &cpu_info[CPU_MAXNUM-1], | | 130 | .cpu_ci = &cpu_info[CPU_MAXNUM-1], |
131 | }, | | 131 | }, |
132 | #endif | | 132 | #endif |
133 | }; | | 133 | }; |
134 | struct cpu_info cpu_info[] = { | | 134 | struct cpu_info cpu_info[] = { |
135 | [0] = { | | 135 | [0] = { |
136 | .ci_curlwp = &lwp0, | | 136 | .ci_curlwp = &lwp0, |
137 | .ci_tlb_info = &pmap_tlb0_info, | | 137 | .ci_tlb_info = &pmap_tlb0_info, |
138 | .ci_softc = &cpu_softc[0], | | 138 | .ci_softc = &cpu_softc[0], |
139 | .ci_cpl = IPL_HIGH, | | 139 | .ci_cpl = IPL_HIGH, |
140 | .ci_idepth = -1, | | 140 | .ci_idepth = -1, |
141 | .ci_pmap_kern_segtab = &pmap_kern_segtab, | | 141 | .ci_pmap_kern_segtab = &pmap_kern_segtab, |
142 | }, | | 142 | }, |
143 | #ifdef MULTIPROCESSOR | | 143 | #ifdef MULTIPROCESSOR |
144 | [CPU_MAXNUM-1] = { | | 144 | [CPU_MAXNUM-1] = { |
145 | .ci_curlwp = NULL, | | 145 | .ci_curlwp = NULL, |
146 | .ci_tlb_info = &pmap_tlb0_info, | | 146 | .ci_tlb_info = &pmap_tlb0_info, |
147 | .ci_softc = &cpu_softc[CPU_MAXNUM-1], | | 147 | .ci_softc = &cpu_softc[CPU_MAXNUM-1], |
148 | .ci_cpl = IPL_HIGH, | | 148 | .ci_cpl = IPL_HIGH, |
149 | .ci_idepth = -1, | | 149 | .ci_idepth = -1, |
150 | .ci_pmap_kern_segtab = &pmap_kern_segtab, | | 150 | .ci_pmap_kern_segtab = &pmap_kern_segtab, |
151 | }, | | 151 | }, |
152 | #endif | | 152 | #endif |
153 | }; | | 153 | }; |
154 | __CTASSERT(__arraycount(cpu_info) == __arraycount(cpu_softc)); | | 154 | __CTASSERT(__arraycount(cpu_info) == __arraycount(cpu_softc)); |
155 | | | 155 | |
156 | /* | | 156 | /* |
157 | * This should probably be in autoconf! XXX | | 157 | * This should probably be in autoconf! XXX |
158 | */ | | 158 | */ |
159 | char machine[] = MACHINE; /* from <machine/param.h> */ | | 159 | char machine[] = MACHINE; /* from <machine/param.h> */ |
160 | char machine_arch[] = MACHINE_ARCH; /* from <machine/param.h> */ | | 160 | char machine_arch[] = MACHINE_ARCH; /* from <machine/param.h> */ |
161 | | | 161 | |
162 | char bootpath[256]; | | 162 | char bootpath[256]; |
163 | | | 163 | |
164 | #if NKSYMS || defined(DDB) || defined(MODULAR) | | 164 | #if NKSYMS || defined(DDB) || defined(MODULAR) |
165 | void *startsym, *endsym; | | 165 | void *startsym, *endsym; |
166 | #endif | | 166 | #endif |
167 | | | 167 | |
168 | #if defined(MULTIPROCESSOR) | | 168 | #if defined(MULTIPROCESSOR) |
169 | volatile struct cpu_hatch_data cpu_hatch_data __cacheline_aligned; | | 169 | volatile struct cpu_hatch_data cpu_hatch_data __cacheline_aligned; |
170 | #endif | | 170 | #endif |
171 | | | 171 | |
172 | int fake_mapiodev = 1; | | 172 | int fake_mapiodev = 1; |
173 | | | 173 | |
174 | void | | 174 | void |
175 | booke_cpu_startup(const char *model) | | 175 | booke_cpu_startup(const char *model) |
176 | { | | 176 | { |
177 | vaddr_t minaddr, maxaddr; | | 177 | vaddr_t minaddr, maxaddr; |
178 | char pbuf[9]; | | 178 | char pbuf[9]; |
179 | | | 179 | |
180 | cpu_setmodel("%s", model); | | 180 | cpu_setmodel("%s", model); |
181 | | | 181 | |
182 | printf("%s%s", copyright, version); | | 182 | printf("%s%s", copyright, version); |
183 | | | 183 | |
184 | format_bytes(pbuf, sizeof(pbuf), ctob((uint64_t)physmem)); | | 184 | format_bytes(pbuf, sizeof(pbuf), ctob((uint64_t)physmem)); |
185 | printf("total memory = %s\n", pbuf); | | 185 | printf("total memory = %s\n", pbuf); |
186 | | | 186 | |
187 | minaddr = 0; | | 187 | minaddr = 0; |
188 | /* | | 188 | /* |
189 | * Allocate a submap for physio | | 189 | * Allocate a submap for physio |
190 | */ | | 190 | */ |
191 | phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, | | 191 | phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, |
192 | VM_PHYS_SIZE, 0, false, NULL); | | 192 | VM_PHYS_SIZE, 0, false, NULL); |
193 | | | 193 | |
194 | /* | | 194 | /* |
195 | * No need to allocate an mbuf cluster submap. Mbuf clusters | | 195 | * No need to allocate an mbuf cluster submap. Mbuf clusters |
196 | * are allocated via the pool allocator, and we use direct-mapped | | 196 | * are allocated via the pool allocator, and we use direct-mapped |
197 | * pool pages. | | 197 | * pool pages. |
198 | */ | | 198 | */ |
199 | | | 199 | |
200 | format_bytes(pbuf, sizeof(pbuf), ptoa(uvm_availmem(false))); | | 200 | format_bytes(pbuf, sizeof(pbuf), ptoa(uvm_availmem(false))); |
201 | printf("avail memory = %s\n", pbuf); | | 201 | printf("avail memory = %s\n", pbuf); |
202 | | | 202 | |
203 | /* | | 203 | /* |
204 | * Register the tlb's evcnts | | 204 | * Register the tlb's evcnts |
205 | */ | | 205 | */ |
206 | pmap_tlb_info_evcnt_attach(curcpu()->ci_tlb_info); | | 206 | pmap_tlb_info_evcnt_attach(curcpu()->ci_tlb_info); |
207 | | | 207 | |
208 | /* | | 208 | /* |
209 | * Set up the board properties database. | | 209 | * Set up the board properties database. |
210 | */ | | 210 | */ |
211 | board_info_init(); | | 211 | board_info_init(); |
212 | | | 212 | |
213 | /* | | 213 | /* |
214 | * Now that we have VM, malloc()s are OK in bus_space. | | 214 | * Now that we have VM, malloc()s are OK in bus_space. |
215 | */ | | 215 | */ |
216 | bus_space_mallocok(); | | 216 | bus_space_mallocok(); |
217 | fake_mapiodev = 0; | | 217 | fake_mapiodev = 0; |
218 | | | 218 | |
219 | #ifdef MULTIPROCESSOR | | 219 | #ifdef MULTIPROCESSOR |
220 | pmap_kernel()->pm_active = kcpuset_running; | | 220 | pmap_kernel()->pm_active = kcpuset_running; |
221 | pmap_kernel()->pm_onproc = kcpuset_running; | | 221 | pmap_kernel()->pm_onproc = kcpuset_running; |
222 | | | 222 | |
223 | for (size_t i = 1; i < __arraycount(cpu_info); i++) { | | 223 | for (size_t i = 1; i < __arraycount(cpu_info); i++) { |
224 | struct cpu_info * const ci = &cpu_info[i]; | | 224 | struct cpu_info * const ci = &cpu_info[i]; |
225 | struct cpu_softc * const cpu = &cpu_softc[i]; | | 225 | struct cpu_softc * const cpu = &cpu_softc[i]; |
226 | cpu->cpu_ci = ci; | | 226 | cpu->cpu_ci = ci; |
227 | cpu->cpu_bst = cpu_softc[0].cpu_bst; | | 227 | cpu->cpu_bst = cpu_softc[0].cpu_bst; |
228 | cpu->cpu_le_bst = cpu_softc[0].cpu_le_bst; | | 228 | cpu->cpu_le_bst = cpu_softc[0].cpu_le_bst; |
229 | cpu->cpu_bsh = cpu_softc[0].cpu_bsh; | | 229 | cpu->cpu_bsh = cpu_softc[0].cpu_bsh; |
230 | cpu->cpu_highmem = cpu_softc[0].cpu_highmem; | | 230 | cpu->cpu_highmem = cpu_softc[0].cpu_highmem; |
231 | ci->ci_softc = cpu; | | 231 | ci->ci_softc = cpu; |
232 | ci->ci_tlb_info = &pmap_tlb0_info; | | 232 | ci->ci_tlb_info = &pmap_tlb0_info; |
233 | ci->ci_cpl = IPL_HIGH; | | 233 | ci->ci_cpl = IPL_HIGH; |
234 | ci->ci_idepth = -1; | | 234 | ci->ci_idepth = -1; |
235 | ci->ci_pmap_kern_segtab = curcpu()->ci_pmap_kern_segtab; | | 235 | ci->ci_pmap_kern_segtab = curcpu()->ci_pmap_kern_segtab; |
236 | } | | 236 | } |
237 | | | 237 | |
238 | kcpuset_create(&cpuset_info.cpus_running, true); | | 238 | kcpuset_create(&cpuset_info.cpus_running, true); |
239 | kcpuset_create(&cpuset_info.cpus_hatched, true); | | 239 | kcpuset_create(&cpuset_info.cpus_hatched, true); |
240 | kcpuset_create(&cpuset_info.cpus_paused, true); | | 240 | kcpuset_create(&cpuset_info.cpus_paused, true); |
241 | kcpuset_create(&cpuset_info.cpus_resumed, true); | | 241 | kcpuset_create(&cpuset_info.cpus_resumed, true); |
242 | kcpuset_create(&cpuset_info.cpus_halted, true); | | 242 | kcpuset_create(&cpuset_info.cpus_halted, true); |
243 | | | 243 | |
244 | kcpuset_set(cpuset_info.cpus_running, cpu_number()); | | 244 | kcpuset_set(cpuset_info.cpus_running, cpu_number()); |
245 | #endif /* MULTIPROCESSOR */ | | 245 | #endif /* MULTIPROCESSOR */ |
246 | } | | 246 | } |
247 | | | 247 | |
248 | static void | | 248 | static void |
249 | dumpsys(void) | | 249 | dumpsys(void) |
250 | { | | 250 | { |
251 | | | 251 | |
252 | printf("dumpsys: TBD\n"); | | 252 | printf("dumpsys: TBD\n"); |
253 | } | | 253 | } |
254 | | | 254 | |
255 | /* | | 255 | /* |
256 | * Halt or reboot the machine after syncing/dumping according to howto. | | 256 | * Halt or reboot the machine after syncing/dumping according to howto. |
257 | */ | | 257 | */ |
258 | void | | 258 | void |
259 | cpu_reboot(int howto, char *what) | | 259 | cpu_reboot(int howto, char *what) |
260 | { | | 260 | { |
261 | static int syncing; | | 261 | static int syncing; |
262 | static char str[256]; | | 262 | static char str[256]; |
263 | char *ap = str, *ap1 = ap; | | 263 | char *ap = str, *ap1 = ap; |
264 | | | 264 | |
265 | boothowto = howto; | | 265 | boothowto = howto; |
266 | if (!cold && !(howto & RB_NOSYNC) && !syncing) { | | 266 | if (!cold && !(howto & RB_NOSYNC) && !syncing) { |
267 | syncing = 1; | | 267 | syncing = 1; |
268 | vfs_shutdown(); /* sync */ | | 268 | vfs_shutdown(); /* sync */ |
269 | resettodr(); /* set wall clock */ | | 269 | resettodr(); /* set wall clock */ |
270 | } | | 270 | } |
271 | | | 271 | |
272 | splhigh(); | | 272 | splhigh(); |
273 | | | 273 | |
274 | if (!cold && (howto & RB_DUMP)) | | 274 | if (!cold && (howto & RB_DUMP)) |
275 | dumpsys(); | | 275 | dumpsys(); |
276 | | | 276 | |
277 | doshutdownhooks(); | | 277 | doshutdownhooks(); |
278 | | | 278 | |
279 | pmf_system_shutdown(boothowto); | | 279 | pmf_system_shutdown(boothowto); |
280 | | | 280 | |
281 | if ((howto & RB_POWERDOWN) == RB_POWERDOWN) { | | 281 | if ((howto & RB_POWERDOWN) == RB_POWERDOWN) { |
282 | /* Power off here if we know how...*/ | | 282 | /* Power off here if we know how...*/ |
283 | } | | 283 | } |
284 | | | 284 | |
285 | if (howto & RB_HALT) { | | 285 | if (howto & RB_HALT) { |
286 | printf("The operating system has halted.\n" | | 286 | printf("The operating system has halted.\n" |
287 | "Press any key to reboot.\n\n"); | | 287 | "Press any key to reboot.\n\n"); |
288 | cnpollc(1); /* For proper keyboard command handling */ | | 288 | cnpollc(1); /* For proper keyboard command handling */ |
289 | cngetc(); | | 289 | cngetc(); |
290 | cnpollc(0); | | 290 | cnpollc(0); |
291 | | | | |
292 | printf("rebooting...\n\n"); | | | |
293 | goto reboot; /* XXX for now... */ | | | |
294 | | | | |
295 | #ifdef DDB | | | |
296 | printf("dropping to debugger\n"); | | | |
297 | while(1) | | | |
298 | Debugger(); | | | |
299 | #endif | | | |
300 | } | | 291 | } |
301 | | | 292 | |
302 | printf("rebooting\n\n"); | | 293 | printf("rebooting\n\n"); |
303 | if (what && *what) { | | 294 | if (what && *what) { |
304 | if (strlen(what) > sizeof str - 5) | | 295 | if (strlen(what) > sizeof str - 5) |
305 | printf("boot string too large, ignored\n"); | | 296 | printf("boot string too large, ignored\n"); |
306 | else { | | 297 | else { |
307 | strcpy(str, what); | | 298 | strcpy(str, what); |
308 | ap1 = ap = str + strlen(str); | | 299 | ap1 = ap = str + strlen(str); |
309 | *ap++ = ' '; | | 300 | *ap++ = ' '; |
310 | } | | 301 | } |
311 | } | | 302 | } |
312 | *ap++ = '-'; | | 303 | *ap++ = '-'; |
313 | if (howto & RB_SINGLE) | | 304 | if (howto & RB_SINGLE) |
314 | *ap++ = 's'; | | 305 | *ap++ = 's'; |
315 | if (howto & RB_KDB) | | 306 | if (howto & RB_KDB) |
316 | *ap++ = 'd'; | | 307 | *ap++ = 'd'; |
317 | *ap++ = 0; | | 308 | *ap++ = 0; |
318 | if (ap[-2] == '-') | | 309 | if (ap[-2] == '-') |
319 | *ap1 = 0; | | 310 | *ap1 = 0; |
320 | | | 311 | |
321 | /* flush cache for msgbuf */ | | 312 | /* flush cache for msgbuf */ |
322 | dcache_wb(msgbuf_paddr, round_page(MSGBUFSIZE)); | | 313 | dcache_wb(msgbuf_paddr, round_page(MSGBUFSIZE)); |
323 | | | 314 | |
324 | reboot: | | | |
325 | __asm volatile("msync; isync"); | | 315 | __asm volatile("msync; isync"); |
326 | (*cpu_md_ops.md_cpu_reset)(); | | 316 | (*cpu_md_ops.md_cpu_reset)(); |
327 | | | 317 | |
328 | printf("%s: md_cpu_reset() failed!\n", __func__); | | 318 | printf("%s: md_cpu_reset() failed!\n", __func__); |
329 | #ifdef DDB | | 319 | #ifdef DDB |
330 | for (;;) | | 320 | for (;;) |
331 | Debugger(); | | 321 | Debugger(); |
332 | #else | | 322 | #else |
333 | for (;;) | | 323 | for (;;) |
334 | /* nothing */; | | 324 | /* nothing */; |
335 | #endif | | 325 | #endif |
336 | } | | 326 | } |
337 | | | 327 | |
338 | /* | | 328 | /* |
339 | * mapiodev: | | 329 | * mapiodev: |
340 | * | | 330 | * |
341 | * Allocate vm space and mapin the I/O address. Use reserved TLB | | 331 | * Allocate vm space and mapin the I/O address. Use reserved TLB |
342 | * mapping if one is found. | | 332 | * mapping if one is found. |
343 | */ | | 333 | */ |
344 | void * | | 334 | void * |
345 | mapiodev(paddr_t pa, psize_t len, bool prefetchable) | | 335 | mapiodev(paddr_t pa, psize_t len, bool prefetchable) |
346 | { | | 336 | { |
347 | const vsize_t off = pa & PAGE_MASK; | | 337 | const vsize_t off = pa & PAGE_MASK; |
348 | | | 338 | |
349 | /* | | 339 | /* |
350 | * See if we have reserved TLB entry for the pa. This needs to be | | 340 | * See if we have reserved TLB entry for the pa. This needs to be |
351 | * true for console as we can't use uvm during early bootstrap. | | 341 | * true for console as we can't use uvm during early bootstrap. |
352 | */ | | 342 | */ |
353 | void * const p = tlb_mapiodev(pa, len, prefetchable); | | 343 | void * const p = tlb_mapiodev(pa, len, prefetchable); |
354 | if (p != NULL) | | 344 | if (p != NULL) |
355 | return p; | | 345 | return p; |
356 | | | 346 | |
357 | if (fake_mapiodev) | | 347 | if (fake_mapiodev) |
358 | panic("mapiodev: no TLB entry reserved for %llx+%llx", | | 348 | panic("mapiodev: no TLB entry reserved for %llx+%llx", |
359 | (long long)pa, (long long)len); | | 349 | (long long)pa, (long long)len); |
360 | | | 350 | |
361 | const paddr_t orig_pa = pa; | | 351 | const paddr_t orig_pa = pa; |
362 | const psize_t orig_len = len; | | 352 | const psize_t orig_len = len; |
363 | vsize_t align = 0; | | 353 | vsize_t align = 0; |
364 | pa = trunc_page(pa); | | 354 | pa = trunc_page(pa); |
365 | len = round_page(off + len); | | 355 | len = round_page(off + len); |
366 | /* | | 356 | /* |
367 | * If we are allocating a large amount (>= 1MB) try to get an | | 357 | * If we are allocating a large amount (>= 1MB) try to get an |
368 | * aligned VA region for it so try to do a large mapping for it. | | 358 | * aligned VA region for it so try to do a large mapping for it. |
369 | */ | | 359 | */ |
370 | if ((len & (len - 1)) == 0 && len >= 0x100000) | | 360 | if ((len & (len - 1)) == 0 && len >= 0x100000) |
371 | align = len; | | 361 | align = len; |
372 | | | 362 | |
373 | vaddr_t va = uvm_km_alloc(kernel_map, len, align, UVM_KMF_VAONLY); | | 363 | vaddr_t va = uvm_km_alloc(kernel_map, len, align, UVM_KMF_VAONLY); |
374 | | | 364 | |
375 | if (va == 0 && align > 0) { | | 365 | if (va == 0 && align > 0) { |
376 | /* | | 366 | /* |
377 | * Large aligned request failed. Let's just get anything. | | 367 | * Large aligned request failed. Let's just get anything. |
378 | */ | | 368 | */ |
379 | align = 0; | | 369 | align = 0; |
380 | va = uvm_km_alloc(kernel_map, len, align, UVM_KMF_VAONLY); | | 370 | va = uvm_km_alloc(kernel_map, len, align, UVM_KMF_VAONLY); |
381 | } | | 371 | } |
382 | if (va == 0) | | 372 | if (va == 0) |
383 | return NULL; | | 373 | return NULL; |
384 | | | 374 | |
385 | if (align) { | | 375 | if (align) { |
386 | /* | | 376 | /* |
387 | * Now try to map that via one big TLB entry. | | 377 | * Now try to map that via one big TLB entry. |
388 | */ | | 378 | */ |
389 | pt_entry_t pte = pte_make_kenter_pa(pa, NULL, | | 379 | pt_entry_t pte = pte_make_kenter_pa(pa, NULL, |
390 | VM_PROT_READ|VM_PROT_WRITE, | | 380 | VM_PROT_READ|VM_PROT_WRITE, |
391 | prefetchable ? 0 : PMAP_NOCACHE); | | 381 | prefetchable ? 0 : PMAP_NOCACHE); |
392 | if (!tlb_ioreserve(va, len, pte)) { | | 382 | if (!tlb_ioreserve(va, len, pte)) { |
393 | void * const p0 = tlb_mapiodev(orig_pa, orig_len, | | 383 | void * const p0 = tlb_mapiodev(orig_pa, orig_len, |
394 | prefetchable); | | 384 | prefetchable); |
395 | KASSERT(p0 != NULL); | | 385 | KASSERT(p0 != NULL); |
396 | return p0; | | 386 | return p0; |
397 | } | | 387 | } |
398 | } | | 388 | } |
399 | | | 389 | |
400 | for (va += len, pa += len; len > 0; len -= PAGE_SIZE) { | | 390 | for (va += len, pa += len; len > 0; len -= PAGE_SIZE) { |
401 | va -= PAGE_SIZE; | | 391 | va -= PAGE_SIZE; |
402 | pa -= PAGE_SIZE; | | 392 | pa -= PAGE_SIZE; |
403 | pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE, | | 393 | pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE, |
404 | prefetchable ? 0 : PMAP_NOCACHE); | | 394 | prefetchable ? 0 : PMAP_NOCACHE); |
405 | } | | 395 | } |
406 | pmap_update(pmap_kernel()); | | 396 | pmap_update(pmap_kernel()); |
407 | return (void *)(va + off); | | 397 | return (void *)(va + off); |
408 | } | | 398 | } |
409 | | | 399 | |
410 | void | | 400 | void |
411 | unmapiodev(vaddr_t va, vsize_t len) | | 401 | unmapiodev(vaddr_t va, vsize_t len) |
412 | { | | 402 | { |
413 | /* Nothing to do for reserved (ie. not uvm_km_alloc'd) mappings. */ | | 403 | /* Nothing to do for reserved (ie. not uvm_km_alloc'd) mappings. */ |
414 | if (va < VM_MIN_KERNEL_ADDRESS || va > VM_MAX_KERNEL_ADDRESS) { | | 404 | if (va < VM_MIN_KERNEL_ADDRESS || va > VM_MAX_KERNEL_ADDRESS) { |
415 | tlb_unmapiodev(va, len); | | 405 | tlb_unmapiodev(va, len); |
416 | return; | | 406 | return; |
417 | } | | 407 | } |
418 | | | 408 | |
419 | len = round_page((va & PAGE_MASK) + len); | | 409 | len = round_page((va & PAGE_MASK) + len); |
420 | va = trunc_page(va); | | 410 | va = trunc_page(va); |
421 | | | 411 | |
422 | pmap_kremove(va, len); | | 412 | pmap_kremove(va, len); |
423 | uvm_km_free(kernel_map, va, len, UVM_KMF_VAONLY); | | 413 | uvm_km_free(kernel_map, va, len, UVM_KMF_VAONLY); |
424 | } | | 414 | } |
425 | | | 415 | |
426 | void | | 416 | void |
427 | cpu_evcnt_attach(struct cpu_info *ci) | | 417 | cpu_evcnt_attach(struct cpu_info *ci) |
428 | { | | 418 | { |
429 | struct cpu_softc * const cpu = ci->ci_softc; | | 419 | struct cpu_softc * const cpu = ci->ci_softc; |
430 | const char * const xname = ci->ci_data.cpu_name; | | 420 | const char * const xname = ci->ci_data.cpu_name; |
431 | | | 421 | |
432 | evcnt_attach_dynamic_nozero(&ci->ci_ev_clock, EVCNT_TYPE_INTR, | | 422 | evcnt_attach_dynamic_nozero(&ci->ci_ev_clock, EVCNT_TYPE_INTR, |
433 | NULL, xname, "clock"); | | 423 | NULL, xname, "clock"); |
434 | evcnt_attach_dynamic_nozero(&cpu->cpu_ev_late_clock, EVCNT_TYPE_INTR, | | 424 | evcnt_attach_dynamic_nozero(&cpu->cpu_ev_late_clock, EVCNT_TYPE_INTR, |
435 | NULL, xname, "late clock"); | | 425 | NULL, xname, "late clock"); |
436 | evcnt_attach_dynamic_nozero(&cpu->cpu_ev_exec_trap_sync, EVCNT_TYPE_TRAP, | | 426 | evcnt_attach_dynamic_nozero(&cpu->cpu_ev_exec_trap_sync, EVCNT_TYPE_TRAP, |
437 | NULL, xname, "exec pages synced (trap)"); | | 427 | NULL, xname, "exec pages synced (trap)"); |
438 | evcnt_attach_dynamic_nozero(&ci->ci_ev_traps, EVCNT_TYPE_TRAP, | | 428 | evcnt_attach_dynamic_nozero(&ci->ci_ev_traps, EVCNT_TYPE_TRAP, |
439 | NULL, xname, "traps"); | | 429 | NULL, xname, "traps"); |
440 | evcnt_attach_dynamic_nozero(&ci->ci_ev_kdsi, EVCNT_TYPE_TRAP, | | 430 | evcnt_attach_dynamic_nozero(&ci->ci_ev_kdsi, EVCNT_TYPE_TRAP, |
441 | &ci->ci_ev_traps, xname, "kernel DSI traps"); | | 431 | &ci->ci_ev_traps, xname, "kernel DSI traps"); |
442 | evcnt_attach_dynamic_nozero(&ci->ci_ev_udsi, EVCNT_TYPE_TRAP, | | 432 | evcnt_attach_dynamic_nozero(&ci->ci_ev_udsi, EVCNT_TYPE_TRAP, |
443 | &ci->ci_ev_traps, xname, "user DSI traps"); | | 433 | &ci->ci_ev_traps, xname, "user DSI traps"); |
444 | evcnt_attach_dynamic_nozero(&ci->ci_ev_udsi_fatal, EVCNT_TYPE_TRAP, | | 434 | evcnt_attach_dynamic_nozero(&ci->ci_ev_udsi_fatal, EVCNT_TYPE_TRAP, |
445 | &ci->ci_ev_udsi, xname, "user DSI failures"); | | 435 | &ci->ci_ev_udsi, xname, "user DSI failures"); |
446 | evcnt_attach_dynamic_nozero(&ci->ci_ev_kisi, EVCNT_TYPE_TRAP, | | 436 | evcnt_attach_dynamic_nozero(&ci->ci_ev_kisi, EVCNT_TYPE_TRAP, |
447 | &ci->ci_ev_traps, xname, "kernel ISI traps"); | | 437 | &ci->ci_ev_traps, xname, "kernel ISI traps"); |
448 | evcnt_attach_dynamic_nozero(&ci->ci_ev_isi, EVCNT_TYPE_TRAP, | | 438 | evcnt_attach_dynamic_nozero(&ci->ci_ev_isi, EVCNT_TYPE_TRAP, |
449 | &ci->ci_ev_traps, xname, "user ISI traps"); | | 439 | &ci->ci_ev_traps, xname, "user ISI traps"); |
450 | evcnt_attach_dynamic_nozero(&ci->ci_ev_isi_fatal, EVCNT_TYPE_TRAP, | | 440 | evcnt_attach_dynamic_nozero(&ci->ci_ev_isi_fatal, EVCNT_TYPE_TRAP, |
451 | &ci->ci_ev_isi, xname, "user ISI failures"); | | 441 | &ci->ci_ev_isi, xname, "user ISI failures"); |
452 | evcnt_attach_dynamic_nozero(&ci->ci_ev_scalls, EVCNT_TYPE_TRAP, | | 442 | evcnt_attach_dynamic_nozero(&ci->ci_ev_scalls, EVCNT_TYPE_TRAP, |
453 | &ci->ci_ev_traps, xname, "system call traps"); | | 443 | &ci->ci_ev_traps, xname, "system call traps"); |
454 | evcnt_attach_dynamic_nozero(&ci->ci_ev_pgm, EVCNT_TYPE_TRAP, | | 444 | evcnt_attach_dynamic_nozero(&ci->ci_ev_pgm, EVCNT_TYPE_TRAP, |
455 | &ci->ci_ev_traps, xname, "PGM traps"); | | 445 | &ci->ci_ev_traps, xname, "PGM traps"); |
456 | evcnt_attach_dynamic_nozero(&ci->ci_ev_debug, EVCNT_TYPE_TRAP, | | 446 | evcnt_attach_dynamic_nozero(&ci->ci_ev_debug, EVCNT_TYPE_TRAP, |
457 | &ci->ci_ev_traps, xname, "debug traps"); | | 447 | &ci->ci_ev_traps, xname, "debug traps"); |
458 | evcnt_attach_dynamic_nozero(&ci->ci_ev_fpu, EVCNT_TYPE_TRAP, | | 448 | evcnt_attach_dynamic_nozero(&ci->ci_ev_fpu, EVCNT_TYPE_TRAP, |
459 | &ci->ci_ev_traps, xname, "FPU unavailable traps"); | | 449 | &ci->ci_ev_traps, xname, "FPU unavailable traps"); |
460 | evcnt_attach_dynamic_nozero(&ci->ci_ev_fpusw, EVCNT_TYPE_MISC, | | 450 | evcnt_attach_dynamic_nozero(&ci->ci_ev_fpusw, EVCNT_TYPE_MISC, |
461 | &ci->ci_ev_fpu, xname, "FPU context switches"); | | 451 | &ci->ci_ev_fpu, xname, "FPU context switches"); |
462 | evcnt_attach_dynamic_nozero(&ci->ci_ev_ali, EVCNT_TYPE_TRAP, | | 452 | evcnt_attach_dynamic_nozero(&ci->ci_ev_ali, EVCNT_TYPE_TRAP, |
463 | &ci->ci_ev_traps, xname, "user alignment traps"); | | 453 | &ci->ci_ev_traps, xname, "user alignment traps"); |
464 | evcnt_attach_dynamic_nozero(&ci->ci_ev_ali_fatal, EVCNT_TYPE_TRAP, | | 454 | evcnt_attach_dynamic_nozero(&ci->ci_ev_ali_fatal, EVCNT_TYPE_TRAP, |
465 | &ci->ci_ev_ali, xname, "user alignment traps"); | | 455 | &ci->ci_ev_ali, xname, "user alignment traps"); |
466 | evcnt_attach_dynamic_nozero(&ci->ci_ev_umchk, EVCNT_TYPE_TRAP, | | 456 | evcnt_attach_dynamic_nozero(&ci->ci_ev_umchk, EVCNT_TYPE_TRAP, |
467 | &ci->ci_ev_umchk, xname, "user MCHK failures"); | | 457 | &ci->ci_ev_umchk, xname, "user MCHK failures"); |
468 | evcnt_attach_dynamic_nozero(&ci->ci_ev_vec, EVCNT_TYPE_TRAP, | | 458 | evcnt_attach_dynamic_nozero(&ci->ci_ev_vec, EVCNT_TYPE_TRAP, |
469 | &ci->ci_ev_traps, xname, "SPE unavailable"); | | 459 | &ci->ci_ev_traps, xname, "SPE unavailable"); |
470 | evcnt_attach_dynamic_nozero(&ci->ci_ev_vecsw, EVCNT_TYPE_MISC, | | 460 | evcnt_attach_dynamic_nozero(&ci->ci_ev_vecsw, EVCNT_TYPE_MISC, |
471 | &ci->ci_ev_vec, xname, "SPE context switches"); | | 461 | &ci->ci_ev_vec, xname, "SPE context switches"); |
472 | evcnt_attach_dynamic_nozero(&ci->ci_ev_ipi, EVCNT_TYPE_INTR, | | 462 | evcnt_attach_dynamic_nozero(&ci->ci_ev_ipi, EVCNT_TYPE_INTR, |
473 | NULL, xname, "IPIs"); | | 463 | NULL, xname, "IPIs"); |
474 | evcnt_attach_dynamic_nozero(&ci->ci_ev_tlbmiss_soft, EVCNT_TYPE_TRAP, | | 464 | evcnt_attach_dynamic_nozero(&ci->ci_ev_tlbmiss_soft, EVCNT_TYPE_TRAP, |
475 | &ci->ci_ev_traps, xname, "soft tlb misses"); | | 465 | &ci->ci_ev_traps, xname, "soft tlb misses"); |
476 | evcnt_attach_dynamic_nozero(&ci->ci_ev_dtlbmiss_hard, EVCNT_TYPE_TRAP, | | 466 | evcnt_attach_dynamic_nozero(&ci->ci_ev_dtlbmiss_hard, EVCNT_TYPE_TRAP, |
477 | &ci->ci_ev_traps, xname, "data tlb misses"); | | 467 | &ci->ci_ev_traps, xname, "data tlb misses"); |
478 | evcnt_attach_dynamic_nozero(&ci->ci_ev_itlbmiss_hard, EVCNT_TYPE_TRAP, | | 468 | evcnt_attach_dynamic_nozero(&ci->ci_ev_itlbmiss_hard, EVCNT_TYPE_TRAP, |
479 | &ci->ci_ev_traps, xname, "inst tlb misses"); | | 469 | &ci->ci_ev_traps, xname, "inst tlb misses"); |
480 | } | | 470 | } |
481 | | | 471 | |
482 | #ifdef MULTIPROCESSOR | | 472 | #ifdef MULTIPROCESSOR |
483 | register_t | | 473 | register_t |
484 | cpu_hatch(void) | | 474 | cpu_hatch(void) |
485 | { | | 475 | { |
486 | struct cpuset_info * const csi = &cpuset_info; | | 476 | struct cpuset_info * const csi = &cpuset_info; |
487 | const size_t id = cpu_number(); | | 477 | const size_t id = cpu_number(); |
488 | | | 478 | |
489 | /* | | 479 | /* |
490 | * We've hatched so tell the spinup code. | | 480 | * We've hatched so tell the spinup code. |
491 | */ | | 481 | */ |
492 | kcpuset_set(csi->cpus_hatched, id); | | 482 | kcpuset_set(csi->cpus_hatched, id); |
493 | | | 483 | |
494 | /* | | 484 | /* |
495 | * Loop until running bit for this cpu is set. | | 485 | * Loop until running bit for this cpu is set. |
496 | */ | | 486 | */ |
497 | while (!kcpuset_isset(csi->cpus_running, id)) { | | 487 | while (!kcpuset_isset(csi->cpus_running, id)) { |
498 | continue; | | 488 | continue; |
499 | } | | 489 | } |
500 | | | 490 | |
501 | /* | | 491 | /* |
502 | * Now that we are active, start the clocks. | | 492 | * Now that we are active, start the clocks. |
503 | */ | | 493 | */ |
504 | cpu_initclocks(); | | 494 | cpu_initclocks(); |
505 | | | 495 | |
506 | /* | | 496 | /* |
507 | * Return sp of the idlelwp. Which we should be already using but ... | | 497 | * Return sp of the idlelwp. Which we should be already using but ... |
508 | */ | | 498 | */ |
509 | return curcpu()->ci_curpcb->pcb_sp; | | 499 | return curcpu()->ci_curpcb->pcb_sp; |
510 | } | | 500 | } |
511 | | | 501 | |
512 | void | | 502 | void |
513 | cpu_boot_secondary_processors(void) | | 503 | cpu_boot_secondary_processors(void) |
514 | { | | 504 | { |
515 | volatile struct cpuset_info * const csi = &cpuset_info; | | 505 | volatile struct cpuset_info * const csi = &cpuset_info; |
516 | CPU_INFO_ITERATOR cii; | | 506 | CPU_INFO_ITERATOR cii; |
517 | struct cpu_info *ci; | | 507 | struct cpu_info *ci; |
518 | kcpuset_t *running; | | 508 | kcpuset_t *running; |
519 | | | 509 | |
520 | kcpuset_create(&running, true); | | 510 | kcpuset_create(&running, true); |
521 | | | 511 | |
522 | for (CPU_INFO_FOREACH(cii, ci)) { | | 512 | for (CPU_INFO_FOREACH(cii, ci)) { |
523 | /* | | 513 | /* |
524 | * Skip this CPU if it didn't successfully hatch. | | 514 | * Skip this CPU if it didn't successfully hatch. |
525 | */ | | 515 | */ |
526 | if (!kcpuset_isset(csi->cpus_hatched, cpu_index(ci))) | | 516 | if (!kcpuset_isset(csi->cpus_hatched, cpu_index(ci))) |
527 | continue; | | 517 | continue; |
528 | | | 518 | |
529 | KASSERT(!CPU_IS_PRIMARY(ci)); | | 519 | KASSERT(!CPU_IS_PRIMARY(ci)); |
530 | KASSERT(ci->ci_data.cpu_idlelwp); | | 520 | KASSERT(ci->ci_data.cpu_idlelwp); |
531 | | | 521 | |
532 | kcpuset_set(running, cpu_index(ci)); | | 522 | kcpuset_set(running, cpu_index(ci)); |
533 | } | | 523 | } |
534 | KASSERT(kcpuset_match(csi->cpus_hatched, running)); | | 524 | KASSERT(kcpuset_match(csi->cpus_hatched, running)); |
535 | if (!kcpuset_iszero(running)) { | | 525 | if (!kcpuset_iszero(running)) { |
536 | kcpuset_merge(csi->cpus_running, running); | | 526 | kcpuset_merge(csi->cpus_running, running); |
537 | } | | 527 | } |
538 | kcpuset_destroy(running); | | 528 | kcpuset_destroy(running); |
539 | } | | 529 | } |
540 | #endif | | 530 | #endif |
541 | | | 531 | |
542 | uint32_t | | 532 | uint32_t |
543 | cpu_read_4(bus_addr_t a) | | 533 | cpu_read_4(bus_addr_t a) |
544 | { | | 534 | { |
545 | struct cpu_softc * const cpu = curcpu()->ci_softc; | | 535 | struct cpu_softc * const cpu = curcpu()->ci_softc; |
546 | // printf(" %s(%p, %x, %x)", __func__, cpu->cpu_bst, cpu->cpu_bsh, a); | | 536 | // printf(" %s(%p, %x, %x)", __func__, cpu->cpu_bst, cpu->cpu_bsh, a); |
547 | return bus_space_read_4(cpu->cpu_bst, cpu->cpu_bsh, a); | | 537 | return bus_space_read_4(cpu->cpu_bst, cpu->cpu_bsh, a); |
548 | } | | 538 | } |
549 | | | 539 | |
550 | uint8_t | | 540 | uint8_t |
551 | cpu_read_1(bus_addr_t a) | | 541 | cpu_read_1(bus_addr_t a) |
552 | { | | 542 | { |
553 | struct cpu_softc * const cpu = curcpu()->ci_softc; | | 543 | struct cpu_softc * const cpu = curcpu()->ci_softc; |
554 | // printf(" %s(%p, %x, %x)", __func__, cpu->cpu_bst, cpu->cpu_bsh, a); | | 544 | // printf(" %s(%p, %x, %x)", __func__, cpu->cpu_bst, cpu->cpu_bsh, a); |
555 | return bus_space_read_1(cpu->cpu_bst, cpu->cpu_bsh, a); | | 545 | return bus_space_read_1(cpu->cpu_bst, cpu->cpu_bsh, a); |
556 | } | | 546 | } |
557 | | | 547 | |
558 | void | | 548 | void |
559 | cpu_write_4(bus_addr_t a, uint32_t v) | | 549 | cpu_write_4(bus_addr_t a, uint32_t v) |
560 | { | | 550 | { |
561 | struct cpu_softc * const cpu = curcpu()->ci_softc; | | 551 | struct cpu_softc * const cpu = curcpu()->ci_softc; |
562 | bus_space_write_4(cpu->cpu_bst, cpu->cpu_bsh, a, v); | | 552 | bus_space_write_4(cpu->cpu_bst, cpu->cpu_bsh, a, v); |
563 | } | | 553 | } |
564 | | | 554 | |
565 | void | | 555 | void |
566 | cpu_write_1(bus_addr_t a, uint8_t v) | | 556 | cpu_write_1(bus_addr_t a, uint8_t v) |
567 | { | | 557 | { |
568 | struct cpu_softc * const cpu = curcpu()->ci_softc; | | 558 | struct cpu_softc * const cpu = curcpu()->ci_softc; |
569 | bus_space_write_1(cpu->cpu_bst, cpu->cpu_bsh, a, v); | | 559 | bus_space_write_1(cpu->cpu_bst, cpu->cpu_bsh, a, v); |
570 | } | | 560 | } |
571 | | | 561 | |
572 | void | | 562 | void |
573 | booke_sstep(struct trapframe *tf) | | 563 | booke_sstep(struct trapframe *tf) |
574 | { | | 564 | { |
575 | uint32_t insn; | | 565 | uint32_t insn; |
576 | | | 566 | |
577 | KASSERT(tf->tf_srr1 & PSL_DE); | | 567 | KASSERT(tf->tf_srr1 & PSL_DE); |
578 | if (ufetch_32((const void *)tf->tf_srr0, &insn) != 0) | | 568 | if (ufetch_32((const void *)tf->tf_srr0, &insn) != 0) |
579 | return; | | 569 | return; |
580 | | | 570 | |
581 | register_t dbcr0 = DBCR0_IAC1 | DBCR0_IDM; | | 571 | register_t dbcr0 = DBCR0_IAC1 | DBCR0_IDM; |
582 | register_t dbcr1 = DBCR1_IAC1US_USER | DBCR1_IAC1ER_DS1; | | 572 | register_t dbcr1 = DBCR1_IAC1US_USER | DBCR1_IAC1ER_DS1; |
583 | if ((insn >> 28) == 4) { | | 573 | if ((insn >> 28) == 4) { |
584 | uint32_t iac2 = 0; | | 574 | uint32_t iac2 = 0; |
585 | if ((insn >> 26) == 0x12) { | | 575 | if ((insn >> 26) == 0x12) { |
586 | const int32_t off = (((int32_t)insn << 6) >> 6) & ~3; | | 576 | const int32_t off = (((int32_t)insn << 6) >> 6) & ~3; |
587 | iac2 = ((insn & 2) ? 0 : tf->tf_srr0) + off; | | 577 | iac2 = ((insn & 2) ? 0 : tf->tf_srr0) + off; |
588 | dbcr0 |= DBCR0_IAC2; | | 578 | dbcr0 |= DBCR0_IAC2; |
589 | } else if ((insn >> 26) == 0x10) { | | 579 | } else if ((insn >> 26) == 0x10) { |
590 | const int16_t off = insn & ~3; | | 580 | const int16_t off = insn & ~3; |
591 | iac2 = ((insn & 2) ? 0 : tf->tf_srr0) + off; | | 581 | iac2 = ((insn & 2) ? 0 : tf->tf_srr0) + off; |
592 | dbcr0 |= DBCR0_IAC2; | | 582 | dbcr0 |= DBCR0_IAC2; |
593 | } else if ((insn & 0xfc00fffe) == 0x4c000420) { | | 583 | } else if ((insn & 0xfc00fffe) == 0x4c000420) { |
594 | iac2 = tf->tf_ctr; | | 584 | iac2 = tf->tf_ctr; |
595 | dbcr0 |= DBCR0_IAC2; | | 585 | dbcr0 |= DBCR0_IAC2; |
596 | } else if ((insn & 0xfc00fffe) == 0x4c000020) { | | 586 | } else if ((insn & 0xfc00fffe) == 0x4c000020) { |
597 | iac2 = tf->tf_lr; | | 587 | iac2 = tf->tf_lr; |
598 | dbcr0 |= DBCR0_IAC2; | | 588 | dbcr0 |= DBCR0_IAC2; |
599 | } | | 589 | } |
600 | if (dbcr0 & DBCR0_IAC2) { | | 590 | if (dbcr0 & DBCR0_IAC2) { |
601 | dbcr1 |= DBCR1_IAC2US_USER | DBCR1_IAC2ER_DS1; | | 591 | dbcr1 |= DBCR1_IAC2US_USER | DBCR1_IAC2ER_DS1; |
602 | mtspr(SPR_IAC2, iac2); | | 592 | mtspr(SPR_IAC2, iac2); |
603 | } | | 593 | } |
604 | } | | 594 | } |
605 | mtspr(SPR_IAC1, tf->tf_srr0 + 4); | | 595 | mtspr(SPR_IAC1, tf->tf_srr0 + 4); |
606 | mtspr(SPR_DBCR1, dbcr1); | | 596 | mtspr(SPR_DBCR1, dbcr1); |
607 | mtspr(SPR_DBCR0, dbcr0); | | 597 | mtspr(SPR_DBCR0, dbcr0); |
608 | } | | 598 | } |
609 | | | 599 | |
610 | #ifdef DIAGNOSTIC | | 600 | #ifdef DIAGNOSTIC |
611 | static inline void | | 601 | static inline void |
612 | swap_data(uint64_t *data, size_t a, size_t b) | | 602 | swap_data(uint64_t *data, size_t a, size_t b) |
613 | { | | 603 | { |
614 | uint64_t swap = data[a]; | | 604 | uint64_t swap = data[a]; |
615 | data[a] = data[b]; | | 605 | data[a] = data[b]; |
616 | data[b] = swap; | | 606 | data[b] = swap; |
617 | } | | 607 | } |
618 | | | 608 | |
619 | static void | | 609 | static void |
620 | sort_data(uint64_t *data, size_t count) | | 610 | sort_data(uint64_t *data, size_t count) |
621 | { | | 611 | { |
622 | #if 0 | | 612 | #if 0 |
623 | /* | | 613 | /* |
624 | * Mostly classic bubble sort | | 614 | * Mostly classic bubble sort |
625 | */ | | 615 | */ |
626 | do { | | 616 | do { |
627 | size_t new_count = 0; | | 617 | size_t new_count = 0; |
628 | for (size_t i = 1; i < count; i++) { | | 618 | for (size_t i = 1; i < count; i++) { |
629 | if (tbs[i - 1] > tbs[i]) { | | 619 | if (tbs[i - 1] > tbs[i]) { |
630 | swap_tbs(tbs, i - 1, i); | | 620 | swap_tbs(tbs, i - 1, i); |
631 | new_count = i; | | 621 | new_count = i; |
632 | } | | 622 | } |
633 | } | | 623 | } |
634 | count = new_count; | | 624 | count = new_count; |
635 | } while (count > 0); | | 625 | } while (count > 0); |
636 | #else | | 626 | #else |
637 | /* | | 627 | /* |
638 | * Comb sort | | 628 | * Comb sort |
639 | */ | | 629 | */ |
640 | size_t gap = count; | | 630 | size_t gap = count; |
641 | bool swapped = false; | | 631 | bool swapped = false; |
642 | while (gap > 1 || swapped) { | | 632 | while (gap > 1 || swapped) { |
643 | if (gap > 1) { | | 633 | if (gap > 1) { |
644 | /* | | 634 | /* |
645 | * phi = (1 + sqrt(5)) / 2 [golden ratio] | | 635 | * phi = (1 + sqrt(5)) / 2 [golden ratio] |
646 | * N = 1 / (1 - e^-phi)) = 1.247330950103979 | | 636 | * N = 1 / (1 - e^-phi)) = 1.247330950103979 |
647 | * | | 637 | * |
648 | * We want to but can't use floating point to calculate | | 638 | * We want to but can't use floating point to calculate |
649 | * gap = (size_t)((double)gap / N) | | 639 | * gap = (size_t)((double)gap / N) |
650 | * | | 640 | * |
651 | * So we will use the multicative inverse of N | | 641 | * So we will use the multicative inverse of N |
652 | * (module 65536) to achieve the division. | | 642 | * (module 65536) to achieve the division. |
653 | * | | 643 | * |
654 | * iN = 2^16 / 1.24733... = 52540 | | 644 | * iN = 2^16 / 1.24733... = 52540 |
655 | * x / N == (x * iN) / 65536 | | 645 | * x / N == (x * iN) / 65536 |
656 | */ | | 646 | */ |
657 | gap = (gap * 52540) / 65536; | | 647 | gap = (gap * 52540) / 65536; |
658 | } | | 648 | } |
659 | | | 649 | |
660 | swapped = false; | | 650 | swapped = false; |
661 | | | 651 | |
662 | for (size_t i = 0; gap + i < count; i++) { | | 652 | for (size_t i = 0; gap + i < count; i++) { |
663 | if (data[i] > data[i + gap]) { | | 653 | if (data[i] > data[i + gap]) { |
664 | swap_data(data, i, i + gap); | | 654 | swap_data(data, i, i + gap); |
665 | swapped = true; | | 655 | swapped = true; |
666 | } | | 656 | } |
667 | } | | 657 | } |
668 | } | | 658 | } |
669 | #endif | | 659 | #endif |
670 | } | | 660 | } |
671 | #endif | | 661 | #endif |
672 | | | 662 | |
673 | void | | 663 | void |
674 | dump_splhist(struct cpu_info *ci, void (*pr)(const char *, ...)) | | 664 | dump_splhist(struct cpu_info *ci, void (*pr)(const char *, ...)) |
675 | { | | 665 | { |
676 | #ifdef DIAGNOSTIC | | 666 | #ifdef DIAGNOSTIC |
677 | struct cpu_softc * const cpu = ci->ci_softc; | | 667 | struct cpu_softc * const cpu = ci->ci_softc; |
678 | uint64_t tbs[NIPL*NIPL]; | | 668 | uint64_t tbs[NIPL*NIPL]; |
679 | size_t ntbs = 0; | | 669 | size_t ntbs = 0; |
680 | for (size_t to = 0; to < NIPL; to++) { | | 670 | for (size_t to = 0; to < NIPL; to++) { |
681 | for (size_t from = 0; from < NIPL; from++) { | | 671 | for (size_t from = 0; from < NIPL; from++) { |
682 | uint64_t tb = cpu->cpu_spl_tb[to][from]; | | 672 | uint64_t tb = cpu->cpu_spl_tb[to][from]; |
683 | if (tb == 0) | | 673 | if (tb == 0) |
684 | continue; | | 674 | continue; |
685 | tbs[ntbs++] = (tb << 8) | (to << 4) | from; | | 675 | tbs[ntbs++] = (tb << 8) | (to << 4) | from; |
686 | } | | 676 | } |
687 | } | | 677 | } |
688 | sort_data(tbs, ntbs); | | 678 | sort_data(tbs, ntbs); |
689 | | | 679 | |
690 | if (pr == NULL) | | 680 | if (pr == NULL) |
691 | pr = printf; | | 681 | pr = printf; |
692 | uint64_t last_tb = 0; | | 682 | uint64_t last_tb = 0; |
693 | for (size_t i = 0; i < ntbs; i++) { | | 683 | for (size_t i = 0; i < ntbs; i++) { |
694 | uint64_t tb = tbs[i]; | | 684 | uint64_t tb = tbs[i]; |
695 | size_t from = tb & 15; | | 685 | size_t from = tb & 15; |
696 | size_t to = (tb >> 4) & 15; | | 686 | size_t to = (tb >> 4) & 15; |
697 | tb >>= 8; | | 687 | tb >>= 8; |
698 | (*pr)("%s(%zu) from %zu at %"PRId64"", | | 688 | (*pr)("%s(%zu) from %zu at %"PRId64"", |
699 | from < to ? "splraise" : "splx", | | 689 | from < to ? "splraise" : "splx", |
700 | to, from, tb); | | 690 | to, from, tb); |
701 | if (last_tb && from != IPL_NONE) | | 691 | if (last_tb && from != IPL_NONE) |
702 | (*pr)(" (+%"PRId64")", tb - last_tb); | | 692 | (*pr)(" (+%"PRId64")", tb - last_tb); |
703 | (*pr)("\n"); | | 693 | (*pr)("\n"); |
704 | last_tb = tb; | | 694 | last_tb = tb; |
705 | } | | 695 | } |
706 | #endif | | 696 | #endif |
707 | } | | 697 | } |