| @@ -1,1018 +1,1018 @@ | | | @@ -1,1018 +1,1018 @@ |
1 | /* $NetBSD: lapic.c,v 1.89 2022/09/07 00:40:19 knakahara Exp $ */ | | 1 | /* $NetBSD: lapic.c,v 1.90 2024/02/25 18:27:54 andvar Exp $ */ |
2 | | | 2 | |
3 | /*- | | 3 | /*- |
4 | * Copyright (c) 2000, 2008, 2020 The NetBSD Foundation, Inc. | | 4 | * Copyright (c) 2000, 2008, 2020 The NetBSD Foundation, Inc. |
5 | * All rights reserved. | | 5 | * All rights reserved. |
6 | * | | 6 | * |
7 | * This code is derived from software contributed to The NetBSD Foundation | | 7 | * This code is derived from software contributed to The NetBSD Foundation |
8 | * by RedBack Networks Inc. | | 8 | * by RedBack Networks Inc. |
9 | * | | 9 | * |
10 | * Author: Bill Sommerfeld | | 10 | * Author: Bill Sommerfeld |
11 | * | | 11 | * |
12 | * Redistribution and use in source and binary forms, with or without | | 12 | * Redistribution and use in source and binary forms, with or without |
13 | * modification, are permitted provided that the following conditions | | 13 | * modification, are permitted provided that the following conditions |
14 | * are met: | | 14 | * are met: |
15 | * 1. Redistributions of source code must retain the above copyright | | 15 | * 1. Redistributions of source code must retain the above copyright |
16 | * notice, this list of conditions and the following disclaimer. | | 16 | * notice, this list of conditions and the following disclaimer. |
17 | * 2. Redistributions in binary form must reproduce the above copyright | | 17 | * 2. Redistributions in binary form must reproduce the above copyright |
18 | * notice, this list of conditions and the following disclaimer in the | | 18 | * notice, this list of conditions and the following disclaimer in the |
19 | * documentation and/or other materials provided with the distribution. | | 19 | * documentation and/or other materials provided with the distribution. |
20 | * | | 20 | * |
21 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS | | 21 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
22 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED | | 22 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
23 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | | 23 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
24 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS | | 24 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
25 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | | 25 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
26 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | | 26 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
27 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | | 27 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
28 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | | 28 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
29 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | | 29 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
30 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | | 30 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
31 | * POSSIBILITY OF SUCH DAMAGE. | | 31 | * POSSIBILITY OF SUCH DAMAGE. |
32 | */ | | 32 | */ |
33 | | | 33 | |
34 | #include <sys/cdefs.h> | | 34 | #include <sys/cdefs.h> |
35 | __KERNEL_RCSID(0, "$NetBSD: lapic.c,v 1.89 2022/09/07 00:40:19 knakahara Exp $"); | | 35 | __KERNEL_RCSID(0, "$NetBSD: lapic.c,v 1.90 2024/02/25 18:27:54 andvar Exp $"); |
36 | | | 36 | |
37 | #include "acpica.h" | | 37 | #include "acpica.h" |
38 | #include "ioapic.h" | | 38 | #include "ioapic.h" |
39 | #include "opt_acpi.h" | | 39 | #include "opt_acpi.h" |
40 | #include "opt_ddb.h" | | 40 | #include "opt_ddb.h" |
41 | #include "opt_mpbios.h" /* for MPDEBUG */ | | 41 | #include "opt_mpbios.h" /* for MPDEBUG */ |
42 | #include "opt_multiprocessor.h" | | 42 | #include "opt_multiprocessor.h" |
43 | #include "opt_ntp.h" | | 43 | #include "opt_ntp.h" |
44 | #include "opt_xen.h" | | 44 | #include "opt_xen.h" |
45 | | | 45 | |
46 | | | 46 | |
47 | #include <sys/param.h> | | 47 | #include <sys/param.h> |
48 | #include <sys/proc.h> | | 48 | #include <sys/proc.h> |
49 | #include <sys/systm.h> | | 49 | #include <sys/systm.h> |
50 | #include <sys/device.h> | | 50 | #include <sys/device.h> |
51 | #include <sys/timetc.h> | | 51 | #include <sys/timetc.h> |
52 | | | 52 | |
53 | #include <uvm/uvm_extern.h> | | 53 | #include <uvm/uvm_extern.h> |
54 | | | 54 | |
55 | #include <dev/ic/i8253reg.h> | | 55 | #include <dev/ic/i8253reg.h> |
56 | | | 56 | |
57 | #include <x86/machdep.h> | | 57 | #include <x86/machdep.h> |
58 | #include <machine/cpu.h> | | 58 | #include <machine/cpu.h> |
59 | #include <machine/cpu_counter.h> | | 59 | #include <machine/cpu_counter.h> |
60 | #include <machine/cpufunc.h> | | 60 | #include <machine/cpufunc.h> |
61 | #include <machine/cpuvar.h> | | 61 | #include <machine/cpuvar.h> |
62 | #include <machine/pmap.h> | | 62 | #include <machine/pmap.h> |
63 | #include <machine/vmparam.h> | | 63 | #include <machine/vmparam.h> |
64 | #include <machine/mpacpi.h> | | 64 | #include <machine/mpacpi.h> |
65 | #include <machine/mpbiosvar.h> | | 65 | #include <machine/mpbiosvar.h> |
66 | #include <machine/pcb.h> | | 66 | #include <machine/pcb.h> |
67 | #include <machine/pmap_private.h> | | 67 | #include <machine/pmap_private.h> |
68 | #include <machine/specialreg.h> | | 68 | #include <machine/specialreg.h> |
69 | #include <machine/segments.h> | | 69 | #include <machine/segments.h> |
70 | #include <x86/x86/tsc.h> | | 70 | #include <x86/x86/tsc.h> |
71 | #include <x86/i82093var.h> | | 71 | #include <x86/i82093var.h> |
72 | | | 72 | |
73 | #include <machine/apicvar.h> | | 73 | #include <machine/apicvar.h> |
74 | #include <machine/i82489reg.h> | | 74 | #include <machine/i82489reg.h> |
75 | #include <machine/i82489var.h> | | 75 | #include <machine/i82489var.h> |
76 | | | 76 | |
77 | #ifndef XENPV | | 77 | #ifndef XENPV |
78 | #if NACPICA > 0 | | 78 | #if NACPICA > 0 |
79 | #include <dev/acpi/acpica.h> | | 79 | #include <dev/acpi/acpica.h> |
80 | #include <dev/acpi/acpivar.h> | | 80 | #include <dev/acpi/acpivar.h> |
81 | #endif | | 81 | #endif |
82 | | | 82 | |
83 | #ifdef DDB | | 83 | #ifdef DDB |
84 | #include <machine/db_machdep.h> | | 84 | #include <machine/db_machdep.h> |
85 | #ifdef MULTIPROCESSOR | | 85 | #ifdef MULTIPROCESSOR |
86 | #ifdef __x86_64__ | | 86 | #ifdef __x86_64__ |
87 | typedef void (vector)(void); | | 87 | typedef void (vector)(void); |
88 | extern vector Xintr_x2apic_ddbipi; | | 88 | extern vector Xintr_x2apic_ddbipi; |
89 | extern int ddb_vec; | | 89 | extern int ddb_vec; |
90 | #endif | | 90 | #endif |
91 | #endif | | 91 | #endif |
92 | #endif | | 92 | #endif |
93 | | | 93 | |
94 | #include <dev/vmt/vmtreg.h> /* for vmt_hvcall() */ | | 94 | #include <dev/vmt/vmtreg.h> /* for vmt_hvcall() */ |
95 | #include <dev/vmt/vmtvar.h> /* for vmt_hvcall() */ | | 95 | #include <dev/vmt/vmtvar.h> /* for vmt_hvcall() */ |
96 | | | 96 | |
97 | /* Referenced from vector.S */ | | 97 | /* Referenced from vector.S */ |
98 | void lapic_clockintr(void *, struct intrframe *); | | 98 | void lapic_clockintr(void *, struct intrframe *); |
99 | | | 99 | |
100 | static void lapic_delay(unsigned int); | | 100 | static void lapic_delay(unsigned int); |
101 | static uint32_t lapic_gettick(void); | | 101 | static uint32_t lapic_gettick(void); |
102 | static void lapic_setup_bsp(paddr_t); | | 102 | static void lapic_setup_bsp(paddr_t); |
103 | static void lapic_map(paddr_t); | | 103 | static void lapic_map(paddr_t); |
104 | | | 104 | |
105 | static void lapic_hwmask(struct pic *, int); | | 105 | static void lapic_hwmask(struct pic *, int); |
106 | static void lapic_hwunmask(struct pic *, int); | | 106 | static void lapic_hwunmask(struct pic *, int); |
107 | static void lapic_setup(struct pic *, struct cpu_info *, int, int, int); | | 107 | static void lapic_setup(struct pic *, struct cpu_info *, int, int, int); |
108 | /* Make it public to call via ddb */ | | 108 | /* Make it public to call via ddb */ |
109 | void lapic_dump(void); | | 109 | void lapic_dump(void); |
110 | | | 110 | |
111 | struct pic local_pic = { | | 111 | struct pic local_pic = { |
112 | .pic_name = "lapic", | | 112 | .pic_name = "lapic", |
113 | .pic_type = PIC_LAPIC, | | 113 | .pic_type = PIC_LAPIC, |
114 | .pic_lock = __SIMPLELOCK_UNLOCKED, | | 114 | .pic_lock = __SIMPLELOCK_UNLOCKED, |
115 | .pic_hwmask = lapic_hwmask, | | 115 | .pic_hwmask = lapic_hwmask, |
116 | .pic_hwunmask = lapic_hwunmask, | | 116 | .pic_hwunmask = lapic_hwunmask, |
117 | .pic_addroute = lapic_setup, | | 117 | .pic_addroute = lapic_setup, |
118 | .pic_delroute = lapic_setup, | | 118 | .pic_delroute = lapic_setup, |
119 | .pic_intr_get_devname = x86_intr_get_devname, | | 119 | .pic_intr_get_devname = x86_intr_get_devname, |
120 | .pic_intr_get_assigned = x86_intr_get_assigned, | | 120 | .pic_intr_get_assigned = x86_intr_get_assigned, |
121 | .pic_intr_get_count = x86_intr_get_count, | | 121 | .pic_intr_get_count = x86_intr_get_count, |
122 | }; | | 122 | }; |
123 | | | 123 | |
124 | static int i82489_ipi(int vec, int target, int dl); | | 124 | static int i82489_ipi(int vec, int target, int dl); |
125 | static int x2apic_ipi(int vec, int target, int dl); | | 125 | static int x2apic_ipi(int vec, int target, int dl); |
126 | int (*x86_ipi)(int, int, int) = i82489_ipi; | | 126 | int (*x86_ipi)(int, int, int) = i82489_ipi; |
127 | | | 127 | |
128 | bool x2apic_mode __read_mostly; | | 128 | bool x2apic_mode __read_mostly; |
129 | #ifdef LAPIC_ENABLE_X2APIC | | 129 | #ifdef LAPIC_ENABLE_X2APIC |
130 | bool x2apic_enable = true; | | 130 | bool x2apic_enable = true; |
131 | #else | | 131 | #else |
132 | bool x2apic_enable = false; | | 132 | bool x2apic_enable = false; |
133 | #endif | | 133 | #endif |
134 | | | 134 | |
135 | static bool lapic_broken_periodic __read_mostly; | | 135 | static bool lapic_broken_periodic __read_mostly; |
136 | | | 136 | |
137 | static uint32_t | | 137 | static uint32_t |
138 | i82489_readreg(u_int reg) | | 138 | i82489_readreg(u_int reg) |
139 | { | | 139 | { |
140 | return *((volatile uint32_t *)(local_apic_va + reg)); | | 140 | return *((volatile uint32_t *)(local_apic_va + reg)); |
141 | } | | 141 | } |
142 | | | 142 | |
143 | static void | | 143 | static void |
144 | i82489_writereg(u_int reg, uint32_t val) | | 144 | i82489_writereg(u_int reg, uint32_t val) |
145 | { | | 145 | { |
146 | *((volatile uint32_t *)(local_apic_va + reg)) = val; | | 146 | *((volatile uint32_t *)(local_apic_va + reg)) = val; |
147 | } | | 147 | } |
148 | | | 148 | |
149 | static uint32_t | | 149 | static uint32_t |
150 | i82489_cpu_number(void) | | 150 | i82489_cpu_number(void) |
151 | { | | 151 | { |
152 | return i82489_readreg(LAPIC_ID) >> LAPIC_ID_SHIFT; | | 152 | return i82489_readreg(LAPIC_ID) >> LAPIC_ID_SHIFT; |
153 | } | | 153 | } |
154 | | | 154 | |
155 | static uint32_t | | 155 | static uint32_t |
156 | x2apic_readreg(u_int reg) | | 156 | x2apic_readreg(u_int reg) |
157 | { | | 157 | { |
158 | return rdmsr(MSR_X2APIC_BASE + (reg >> 4)); | | 158 | return rdmsr(MSR_X2APIC_BASE + (reg >> 4)); |
159 | } | | 159 | } |
160 | | | 160 | |
161 | static void | | 161 | static void |
162 | x2apic_writereg(u_int reg, uint32_t val) | | 162 | x2apic_writereg(u_int reg, uint32_t val) |
163 | { | | 163 | { |
164 | x86_mfence(); | | 164 | x86_mfence(); |
165 | wrmsr(MSR_X2APIC_BASE + (reg >> 4), val); | | 165 | wrmsr(MSR_X2APIC_BASE + (reg >> 4), val); |
166 | } | | 166 | } |
167 | | | 167 | |
168 | static void | | 168 | static void |
169 | x2apic_writereg64(u_int reg, uint64_t val) | | 169 | x2apic_writereg64(u_int reg, uint64_t val) |
170 | { | | 170 | { |
171 | KDASSERT(reg == LAPIC_ICRLO); | | 171 | KDASSERT(reg == LAPIC_ICRLO); |
172 | x86_mfence(); | | 172 | x86_mfence(); |
173 | wrmsr(MSR_X2APIC_BASE + (reg >> 4), val); | | 173 | wrmsr(MSR_X2APIC_BASE + (reg >> 4), val); |
174 | } | | 174 | } |
175 | | | 175 | |
176 | static void | | 176 | static void |
177 | x2apic_write_icr(uint32_t hi, uint32_t lo) | | 177 | x2apic_write_icr(uint32_t hi, uint32_t lo) |
178 | { | | 178 | { |
179 | x2apic_writereg64(LAPIC_ICRLO, ((uint64_t)hi << 32) | lo); | | 179 | x2apic_writereg64(LAPIC_ICRLO, ((uint64_t)hi << 32) | lo); |
180 | } | | 180 | } |
181 | | | 181 | |
182 | static uint32_t | | 182 | static uint32_t |
183 | x2apic_cpu_number(void) | | 183 | x2apic_cpu_number(void) |
184 | { | | 184 | { |
185 | return x2apic_readreg(LAPIC_ID); | | 185 | return x2apic_readreg(LAPIC_ID); |
186 | } | | 186 | } |
187 | | | 187 | |
188 | uint32_t | | 188 | uint32_t |
189 | lapic_readreg(u_int reg) | | 189 | lapic_readreg(u_int reg) |
190 | { | | 190 | { |
191 | if (x2apic_mode) | | 191 | if (x2apic_mode) |
192 | return x2apic_readreg(reg); | | 192 | return x2apic_readreg(reg); |
193 | return i82489_readreg(reg); | | 193 | return i82489_readreg(reg); |
194 | } | | 194 | } |
195 | | | 195 | |
196 | void | | 196 | void |
197 | lapic_writereg(u_int reg, uint32_t val) | | 197 | lapic_writereg(u_int reg, uint32_t val) |
198 | { | | 198 | { |
199 | if (x2apic_mode) | | 199 | if (x2apic_mode) |
200 | x2apic_writereg(reg, val); | | 200 | x2apic_writereg(reg, val); |
201 | else | | 201 | else |
202 | i82489_writereg(reg, val); | | 202 | i82489_writereg(reg, val); |
203 | } | | 203 | } |
204 | | | 204 | |
205 | void | | 205 | void |
206 | lapic_write_tpri(uint32_t val) | | 206 | lapic_write_tpri(uint32_t val) |
207 | { | | 207 | { |
208 | | | 208 | |
209 | val &= LAPIC_TPRI_MASK; | | 209 | val &= LAPIC_TPRI_MASK; |
210 | #ifdef i386 | | 210 | #ifdef i386 |
211 | lapic_writereg(LAPIC_TPRI, val); | | 211 | lapic_writereg(LAPIC_TPRI, val); |
212 | #else | | 212 | #else |
213 | lcr8(val >> 4); | | 213 | lcr8(val >> 4); |
214 | #endif | | 214 | #endif |
215 | } | | 215 | } |
216 | | | 216 | |
217 | uint32_t | | 217 | uint32_t |
218 | lapic_cpu_number(void) | | 218 | lapic_cpu_number(void) |
219 | { | | 219 | { |
220 | if (x2apic_mode) | | 220 | if (x2apic_mode) |
221 | return x2apic_cpu_number(); | | 221 | return x2apic_cpu_number(); |
222 | return i82489_cpu_number(); | | 222 | return i82489_cpu_number(); |
223 | } | | 223 | } |
224 | | | 224 | |
225 | static void | | 225 | static void |
226 | lapic_enable_x2apic(void) | | 226 | lapic_enable_x2apic(void) |
227 | { | | 227 | { |
228 | uint64_t apicbase; | | 228 | uint64_t apicbase; |
229 | | | 229 | |
230 | apicbase = rdmsr(MSR_APICBASE); | | 230 | apicbase = rdmsr(MSR_APICBASE); |
231 | if (!ISSET(apicbase, APICBASE_EN)) { | | 231 | if (!ISSET(apicbase, APICBASE_EN)) { |
232 | apicbase |= APICBASE_EN; | | 232 | apicbase |= APICBASE_EN; |
233 | wrmsr(MSR_APICBASE, apicbase); | | 233 | wrmsr(MSR_APICBASE, apicbase); |
234 | } | | 234 | } |
235 | apicbase |= APICBASE_EXTD; | | 235 | apicbase |= APICBASE_EXTD; |
236 | wrmsr(MSR_APICBASE, apicbase); | | 236 | wrmsr(MSR_APICBASE, apicbase); |
237 | } | | 237 | } |
238 | | | 238 | |
239 | bool | | 239 | bool |
240 | lapic_is_x2apic(void) | | 240 | lapic_is_x2apic(void) |
241 | { | | 241 | { |
242 | uint64_t msr; | | 242 | uint64_t msr; |
243 | | | 243 | |
244 | if (!ISSET(cpu_feature[0], CPUID_APIC) || | | 244 | if (!ISSET(cpu_feature[0], CPUID_APIC) || |
245 | rdmsr_safe(MSR_APICBASE, &msr) == EFAULT) | | 245 | rdmsr_safe(MSR_APICBASE, &msr) == EFAULT) |
246 | return false; | | 246 | return false; |
247 | return (msr & (APICBASE_EN | APICBASE_EXTD)) == | | 247 | return (msr & (APICBASE_EN | APICBASE_EXTD)) == |
248 | (APICBASE_EN | APICBASE_EXTD); | | 248 | (APICBASE_EN | APICBASE_EXTD); |
249 | } | | 249 | } |
250 | | | 250 | |
251 | /* | | 251 | /* |
252 | * Initialize the local APIC on the BSP. | | 252 | * Initialize the local APIC on the BSP. |
253 | */ | | 253 | */ |
254 | static void | | 254 | static void |
255 | lapic_setup_bsp(paddr_t lapic_base) | | 255 | lapic_setup_bsp(paddr_t lapic_base) |
256 | { | | 256 | { |
257 | u_int regs[6]; | | 257 | u_int regs[6]; |
258 | const char *reason = NULL; | | 258 | const char *reason = NULL; |
259 | const char *hw_vendor; | | 259 | const char *hw_vendor; |
260 | bool bios_x2apic; | | 260 | bool bios_x2apic; |
261 | | | 261 | |
262 | if (ISSET(cpu_feature[1], CPUID2_X2APIC)) { | | 262 | if (ISSET(cpu_feature[1], CPUID2_X2APIC)) { |
263 | #if NACPICA > 0 | | 263 | #if NACPICA > 0 |
264 | if (acpi_present) { | | 264 | if (acpi_present) { |
265 | ACPI_TABLE_DMAR *dmar; | | 265 | ACPI_TABLE_DMAR *dmar; |
266 | ACPI_STATUS status; | | 266 | ACPI_STATUS status; |
267 | | | 267 | |
268 | /* | | 268 | /* |
269 | * Automatically detect several configurations where | | 269 | * Automatically detect several configurations where |
270 | * x2APIC mode is known to cause troubles. User can | | 270 | * x2APIC mode is known to cause troubles. User can |
271 | * override the setting with hw.x2apic_enable tunable. | | 271 | * override the setting with hw.x2apic_enable tunable. |
272 | */ | | 272 | */ |
273 | status = AcpiGetTable(ACPI_SIG_DMAR, 1, | | 273 | status = AcpiGetTable(ACPI_SIG_DMAR, 1, |
274 | (ACPI_TABLE_HEADER **)&dmar); | | 274 | (ACPI_TABLE_HEADER **)&dmar); |
275 | if (ACPI_SUCCESS(status)) { | | 275 | if (ACPI_SUCCESS(status)) { |
276 | if (ISSET(dmar->Flags, ACPI_DMAR_X2APIC_OPT_OUT)) { | | 276 | if (ISSET(dmar->Flags, ACPI_DMAR_X2APIC_OPT_OUT)) { |
277 | reason = "by DMAR table"; | | 277 | reason = "by DMAR table"; |
278 | } | | 278 | } |
279 | AcpiPutTable(&dmar->Header); | | 279 | AcpiPutTable(&dmar->Header); |
280 | } | | 280 | } |
281 | } | | 281 | } |
282 | #endif /* NACPICA > 0 */ | | 282 | #endif /* NACPICA > 0 */ |
283 | if (vm_guest == VM_GUEST_VMWARE) { | | 283 | if (vm_guest == VM_GUEST_VMWARE) { |
284 | vmt_hvcall(VM_CMD_GET_VCPU_INFO, regs); | | 284 | vmt_hvcall(VM_CMD_GET_VCPU_INFO, regs); |
285 | if (ISSET(regs[0], VCPUINFO_VCPU_RESERVED) || | | 285 | if (ISSET(regs[0], VCPUINFO_VCPU_RESERVED) || |
286 | !ISSET(regs[0], VCPUINFO_LEGACY_X2APIC)) | | 286 | !ISSET(regs[0], VCPUINFO_LEGACY_X2APIC)) |
287 | reason = "inside VMWare without intr " | | 287 | reason = "inside VMWare without intr " |
288 | "redirection"; | | 288 | "redirection"; |
289 | } else if (vm_guest == VM_GUEST_XENHVM) { | | 289 | } else if (vm_guest == VM_GUEST_XENHVM) { |
290 | reason = "due to running under XEN"; | | 290 | reason = "due to running under XEN"; |
291 | } else if (vm_guest == VM_GUEST_NO && | | 291 | } else if (vm_guest == VM_GUEST_NO && |
292 | CPUID_TO_FAMILY(curcpu()->ci_signature) == 6 && | | 292 | CPUID_TO_FAMILY(curcpu()->ci_signature) == 6 && |
293 | CPUID_TO_MODEL(curcpu()->ci_signature) == 0x2a) { | | 293 | CPUID_TO_MODEL(curcpu()->ci_signature) == 0x2a) { |
294 | hw_vendor = pmf_get_platform("board-vendor"); | | 294 | hw_vendor = pmf_get_platform("board-vendor"); |
295 | if (hw_vendor != NULL) { | | 295 | if (hw_vendor != NULL) { |
296 | /* | | 296 | /* |
297 | * It seems that some Lenovo and ASUS | | 297 | * It seems that some Lenovo and ASUS |
298 | * SandyBridge-based notebook BIOSes have a bug | | 298 | * SandyBridge-based notebook BIOSes have a bug |
299 | * which prevents booting AP in x2APIC mode. | | 299 | * which prevents booting AP in x2APIC mode. |
300 | * Since the only way to detect mobile CPU is | | 300 | * Since the only way to detect mobile CPU is |
301 | * to check northbridge pci id, which cannot be | | 301 | * to check northbridge pci id, which cannot be |
302 | * done that early, disable x2APIC for all | | 302 | * done that early, disable x2APIC for all |
303 | * Lenovo and ASUS SandyBridge machines. | | 303 | * Lenovo and ASUS SandyBridge machines. |
304 | */ | | 304 | */ |
305 | if (strcmp(hw_vendor, "LENOVO") == 0 || | | 305 | if (strcmp(hw_vendor, "LENOVO") == 0 || |
306 | strcmp(hw_vendor, "ASUSTeK Computer Inc.") == 0) { | | 306 | strcmp(hw_vendor, "ASUSTeK Computer Inc.") == 0) { |
307 | reason = "for a suspected SandyBridge " | | 307 | reason = "for a suspected SandyBridge " |
308 | "BIOS bug"; | | 308 | "BIOS bug"; |
309 | } | | 309 | } |
310 | } | | 310 | } |
311 | } | | 311 | } |
312 | bios_x2apic = lapic_is_x2apic(); | | 312 | bios_x2apic = lapic_is_x2apic(); |
313 | if (reason != NULL && bios_x2apic) { | | 313 | if (reason != NULL && bios_x2apic) { |
314 | aprint_verbose("x2APIC should be disabled %s but " | | 314 | aprint_verbose("x2APIC should be disabled %s but " |
315 | "already enabled by BIOS; enabling.\n", reason); | | 315 | "already enabled by BIOS; enabling.\n", reason); |
316 | reason = NULL; | | 316 | reason = NULL; |
317 | } | | 317 | } |
318 | if (reason == NULL) | | 318 | if (reason == NULL) |
319 | x2apic_mode = true; | | 319 | x2apic_mode = true; |
320 | else | | 320 | else |
321 | aprint_verbose("x2APIC available but disabled %s\n", | | 321 | aprint_verbose("x2APIC available but disabled %s\n", |
322 | reason); | | 322 | reason); |
323 | if (x2apic_enable != x2apic_mode) { | | 323 | if (x2apic_enable != x2apic_mode) { |
324 | if (bios_x2apic && !x2apic_enable) | | 324 | if (bios_x2apic && !x2apic_enable) |
325 | aprint_verbose("x2APIC disabled by user and " | | 325 | aprint_verbose("x2APIC disabled by user and " |
326 | "enabled by BIOS; ignoring user setting.\n"); | | 326 | "enabled by BIOS; ignoring user setting.\n"); |
327 | else | | 327 | else |
328 | x2apic_mode = x2apic_enable; | | 328 | x2apic_mode = x2apic_enable; |
329 | } | | 329 | } |
330 | } | | 330 | } |
331 | if (x2apic_mode) { | | 331 | if (x2apic_mode) { |
332 | x86_ipi = x2apic_ipi; | | 332 | x86_ipi = x2apic_ipi; |
333 | #if NIOAPIC > 0 | | 333 | #if NIOAPIC > 0 |
334 | struct ioapic_softc *ioapic; | | 334 | struct ioapic_softc *ioapic; |
335 | for (ioapic = ioapics; ioapic != NULL; ioapic = ioapic->sc_next) { | | 335 | for (ioapic = ioapics; ioapic != NULL; ioapic = ioapic->sc_next) { |
336 | ioapic->sc_pic.pic_edge_stubs = x2apic_edge_stubs; | | 336 | ioapic->sc_pic.pic_edge_stubs = x2apic_edge_stubs; |
337 | ioapic->sc_pic.pic_level_stubs = x2apic_level_stubs; | | 337 | ioapic->sc_pic.pic_level_stubs = x2apic_level_stubs; |
338 | } | | 338 | } |
339 | #endif | | 339 | #endif |
340 | #if defined(DDB) && defined(MULTIPROCESSOR) | | 340 | #if defined(DDB) && defined(MULTIPROCESSOR) |
341 | #ifdef __x86_64__ | | 341 | #ifdef __x86_64__ |
342 | struct idt_vec *iv = &(cpu_info_primary.ci_idtvec); | | 342 | struct idt_vec *iv = &(cpu_info_primary.ci_idtvec); |
343 | idt_descriptor_t *idt = iv->iv_idt; | | 343 | idt_descriptor_t *idt = iv->iv_idt; |
344 | set_idtgate(&idt[ddb_vec], &Xintr_x2apic_ddbipi, 1, | | 344 | set_idtgate(&idt[ddb_vec], &Xintr_x2apic_ddbipi, 1, |
345 | SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); | | 345 | SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); |
346 | #else | | 346 | #else |
347 | /* | | 347 | /* |
348 | * Set DDB IPI handler in cpu_set_tss_gates() when cpu0 is | | 348 | * Set DDB IPI handler in cpu_set_tss_gates() when cpu0 is |
349 | * attached. | | 349 | * attached. |
350 | */ | | 350 | */ |
351 | #endif | | 351 | #endif |
352 | #endif | | 352 | #endif |
353 | | | 353 | |
354 | x86_disable_intr(); | | 354 | x86_disable_intr(); |
355 | lapic_enable_x2apic(); | | 355 | lapic_enable_x2apic(); |
356 | #ifdef MULTIPROCESSOR | | 356 | #ifdef MULTIPROCESSOR |
357 | cpu_init_first(); /* Catch up to changed cpu_number() */ | | 357 | cpu_init_first(); /* Catch up to changed cpu_number() */ |
358 | #endif | | 358 | #endif |
359 | lapic_write_tpri(0); | | 359 | lapic_write_tpri(0); |
360 | x86_enable_intr(); | | 360 | x86_enable_intr(); |
361 | } else | | 361 | } else |
362 | lapic_map(lapic_base); | | 362 | lapic_map(lapic_base); |
363 | } | | 363 | } |
364 | | | 364 | |
365 | static void | | 365 | static void |
366 | lapic_map(paddr_t lapic_base) | | 366 | lapic_map(paddr_t lapic_base) |
367 | { | | 367 | { |
368 | pt_entry_t *pte; | | 368 | pt_entry_t *pte; |
369 | vaddr_t va = local_apic_va; | | 369 | vaddr_t va = local_apic_va; |
370 | | | 370 | |
371 | /* | | 371 | /* |
372 | * If the CPU has an APIC MSR, use it and ignore the supplied value: | | 372 | * If the CPU has an APIC MSR, use it and ignore the supplied value: |
373 | * some ACPI implementations have been observed to pass bad values. | | 373 | * some ACPI implementations have been observed to pass bad values. |
374 | * Additionally, ensure that the lapic is enabled as we are committed | | 374 | * Additionally, ensure that the lapic is enabled as we are committed |
375 | * to using it at this point. Be conservative and assume that the MSR | | 375 | * to using it at this point. Be conservative and assume that the MSR |
376 | * is not present on the Pentium (is it?). | | 376 | * is not present on the Pentium (is it?). |
377 | */ | | 377 | */ |
378 | if (CPUID_TO_FAMILY(curcpu()->ci_signature) >= 6) { | | 378 | if (CPUID_TO_FAMILY(curcpu()->ci_signature) >= 6) { |
379 | lapic_base = (paddr_t)rdmsr(MSR_APICBASE); | | 379 | lapic_base = (paddr_t)rdmsr(MSR_APICBASE); |
380 | if ((lapic_base & APICBASE_PHYSADDR) == 0) { | | 380 | if ((lapic_base & APICBASE_PHYSADDR) == 0) { |
381 | lapic_base |= LAPIC_BASE; | | 381 | lapic_base |= LAPIC_BASE; |
382 | } | | 382 | } |
383 | wrmsr(MSR_APICBASE, lapic_base | APICBASE_EN); | | 383 | wrmsr(MSR_APICBASE, lapic_base | APICBASE_EN); |
384 | lapic_base &= APICBASE_PHYSADDR; | | 384 | lapic_base &= APICBASE_PHYSADDR; |
385 | } | | 385 | } |
386 | | | 386 | |
387 | x86_disable_intr(); | | 387 | x86_disable_intr(); |
388 | | | 388 | |
389 | /* | | 389 | /* |
390 | * Map local apic. If we have a local apic, it's safe to assume | | 390 | * Map local apic. If we have a local apic, it's safe to assume |
391 | * we're on a 486 or better and can use invlpg and non-cacheable PTE's | | 391 | * we're on a 486 or better and can use invlpg and non-cacheable PTE's |
392 | * | | 392 | * |
393 | * Whap the PTE "by hand" rather than calling pmap_kenter_pa because | | 393 | * Whap the PTE "by hand" rather than calling pmap_kenter_pa because |
394 | * the latter will attempt to invoke TLB shootdown code just as we | | 394 | * the latter will attempt to invoke TLB shootdown code just as we |
395 | * might have changed the value of cpu_number().. | | 395 | * might have changed the value of cpu_number().. |
396 | */ | | 396 | */ |
397 | | | 397 | |
398 | pte = kvtopte(va); | | 398 | pte = kvtopte(va); |
399 | *pte = lapic_base | PTE_W | PTE_P | PTE_PCD | pmap_pg_g | pmap_pg_nx; | | 399 | *pte = lapic_base | PTE_W | PTE_P | PTE_PCD | pmap_pg_g | pmap_pg_nx; |
400 | invlpg(va); | | 400 | invlpg(va); |
401 | | | 401 | |
402 | #ifdef MULTIPROCESSOR | | 402 | #ifdef MULTIPROCESSOR |
403 | cpu_init_first(); /* Catch up to changed cpu_number() */ | | 403 | cpu_init_first(); /* Catch up to changed cpu_number() */ |
404 | #endif | | 404 | #endif |
405 | | | 405 | |
406 | lapic_write_tpri(0); | | 406 | lapic_write_tpri(0); |
407 | x86_enable_intr(); | | 407 | x86_enable_intr(); |
408 | } | | 408 | } |
409 | | | 409 | |
410 | /* | | 410 | /* |
411 | * enable local apic | | 411 | * enable local apic |
412 | */ | | 412 | */ |
413 | void | | 413 | void |
414 | lapic_enable(void) | | 414 | lapic_enable(void) |
415 | { | | 415 | { |
416 | lapic_writereg(LAPIC_SVR, LAPIC_SVR_ENABLE | LAPIC_SPURIOUS_VECTOR); | | 416 | lapic_writereg(LAPIC_SVR, LAPIC_SVR_ENABLE | LAPIC_SPURIOUS_VECTOR); |
417 | } | | 417 | } |
418 | | | 418 | |
419 | void | | 419 | void |
420 | lapic_set_lvt(void) | | 420 | lapic_set_lvt(void) |
421 | { | | 421 | { |
422 | struct cpu_info *ci = curcpu(); | | 422 | struct cpu_info *ci = curcpu(); |
423 | int i; | | 423 | int i; |
424 | struct mp_intr_map *mpi; | | 424 | struct mp_intr_map *mpi; |
425 | uint32_t lint0, lint1; | | 425 | uint32_t lint0, lint1; |
426 | | | 426 | |
427 | #ifdef MULTIPROCESSOR | | 427 | #ifdef MULTIPROCESSOR |
428 | if (mp_verbose) { | | 428 | if (mp_verbose) { |
429 | apic_format_redir(device_xname(ci->ci_dev), "prelint", 0, | | 429 | apic_format_redir(device_xname(ci->ci_dev), "prelint", 0, |
430 | APIC_VECTYPE_LAPIC_LVT, 0, lapic_readreg(LAPIC_LVT_LINT0)); | | 430 | APIC_VECTYPE_LAPIC_LVT, 0, lapic_readreg(LAPIC_LVT_LINT0)); |
431 | apic_format_redir(device_xname(ci->ci_dev), "prelint", 1, | | 431 | apic_format_redir(device_xname(ci->ci_dev), "prelint", 1, |
432 | APIC_VECTYPE_LAPIC_LVT, 0, lapic_readreg(LAPIC_LVT_LINT1)); | | 432 | APIC_VECTYPE_LAPIC_LVT, 0, lapic_readreg(LAPIC_LVT_LINT1)); |
433 | } | | 433 | } |
434 | #endif | | 434 | #endif |
435 | | | 435 | |
436 | /* | | 436 | /* |
437 | * If an I/O APIC has been attached, assume that it is used instead of | | 437 | * If an I/O APIC has been attached, assume that it is used instead of |
438 | * the 8259A for interrupt delivery. Otherwise request the LAPIC to | | 438 | * the 8259A for interrupt delivery. Otherwise request the LAPIC to |
439 | * get external interrupts via LINT0 for the primary CPU. | | 439 | * get external interrupts via LINT0 for the primary CPU. |
440 | */ | | 440 | */ |
441 | lint0 = LAPIC_DLMODE_EXTINT; | | 441 | lint0 = LAPIC_DLMODE_EXTINT; |
442 | if (nioapics > 0 || !CPU_IS_PRIMARY(curcpu())) | | 442 | if (nioapics > 0 || !CPU_IS_PRIMARY(curcpu())) |
443 | lint0 |= LAPIC_LVT_MASKED; | | 443 | lint0 |= LAPIC_LVT_MASKED; |
444 | lapic_writereg(LAPIC_LVT_LINT0, lint0); | | 444 | lapic_writereg(LAPIC_LVT_LINT0, lint0); |
445 | | | 445 | |
446 | /* | | 446 | /* |
447 | * Non Maskable Interrupts are to be delivered to the primary CPU. | | 447 | * Non Maskable Interrupts are to be delivered to the primary CPU. |
448 | */ | | 448 | */ |
449 | lint1 = LAPIC_DLMODE_NMI; | | 449 | lint1 = LAPIC_DLMODE_NMI; |
450 | if (!CPU_IS_PRIMARY(curcpu())) | | 450 | if (!CPU_IS_PRIMARY(curcpu())) |
451 | lint1 |= LAPIC_LVT_MASKED; | | 451 | lint1 |= LAPIC_LVT_MASKED; |
452 | lapic_writereg(LAPIC_LVT_LINT1, lint1); | | 452 | lapic_writereg(LAPIC_LVT_LINT1, lint1); |
453 | | | 453 | |
454 | for (i = 0; i < mp_nintr; i++) { | | 454 | for (i = 0; i < mp_nintr; i++) { |
455 | mpi = &mp_intrs[i]; | | 455 | mpi = &mp_intrs[i]; |
456 | if (mpi->ioapic == NULL && (mpi->cpu_id == MPS_ALL_APICS || | | 456 | if (mpi->ioapic == NULL && (mpi->cpu_id == MPS_ALL_APICS || |
457 | mpi->cpu_id == ci->ci_cpuid)) { | | 457 | mpi->cpu_id == ci->ci_cpuid)) { |
458 | if (mpi->ioapic_pin > 1) | | 458 | if (mpi->ioapic_pin > 1) |
459 | aprint_error_dev(ci->ci_dev, | | 459 | aprint_error_dev(ci->ci_dev, |
460 | "%s: WARNING: bad pin value %d\n", | | 460 | "%s: WARNING: bad pin value %d\n", |
461 | __func__, mpi->ioapic_pin); | | 461 | __func__, mpi->ioapic_pin); |
462 | if (mpi->ioapic_pin == 0) | | 462 | if (mpi->ioapic_pin == 0) |
463 | lapic_writereg(LAPIC_LVT_LINT0, mpi->redir); | | 463 | lapic_writereg(LAPIC_LVT_LINT0, mpi->redir); |
464 | else | | 464 | else |
465 | lapic_writereg(LAPIC_LVT_LINT1, mpi->redir); | | 465 | lapic_writereg(LAPIC_LVT_LINT1, mpi->redir); |
466 | } | | 466 | } |
467 | } | | 467 | } |
468 | | | 468 | |
469 | #ifdef MULTIPROCESSOR | | 469 | #ifdef MULTIPROCESSOR |
470 | if (mp_verbose) | | 470 | if (mp_verbose) |
471 | lapic_dump(); | | 471 | lapic_dump(); |
472 | #endif | | 472 | #endif |
473 | } | | 473 | } |
474 | | | 474 | |
475 | /* | | 475 | /* |
476 | * Initialize fixed idt vectors for use by local apic. | | 476 | * Initialize fixed idt vectors for use by local apic. |
477 | */ | | 477 | */ |
478 | void | | 478 | void |
479 | lapic_boot_init(paddr_t lapic_base) | | 479 | lapic_boot_init(paddr_t lapic_base) |
480 | { | | 480 | { |
481 | struct idt_vec *iv = &(cpu_info_primary.ci_idtvec); | | 481 | struct idt_vec *iv = &(cpu_info_primary.ci_idtvec); |
482 | | | 482 | |
483 | lapic_setup_bsp(lapic_base); | | 483 | lapic_setup_bsp(lapic_base); |
484 | | | 484 | |
485 | #ifdef MULTIPROCESSOR | | 485 | #ifdef MULTIPROCESSOR |
486 | idt_vec_reserve(iv, LAPIC_IPI_VECTOR); | | 486 | idt_vec_reserve(iv, LAPIC_IPI_VECTOR); |
487 | idt_vec_set(iv, LAPIC_IPI_VECTOR, | | 487 | idt_vec_set(iv, LAPIC_IPI_VECTOR, |
488 | x2apic_mode ? Xintr_x2apic_ipi : Xintr_lapic_ipi); | | 488 | x2apic_mode ? Xintr_x2apic_ipi : Xintr_lapic_ipi); |
489 | | | 489 | |
490 | idt_vec_reserve(iv, LAPIC_TLB_VECTOR); | | 490 | idt_vec_reserve(iv, LAPIC_TLB_VECTOR); |
491 | idt_vec_set(iv, LAPIC_TLB_VECTOR, | | 491 | idt_vec_set(iv, LAPIC_TLB_VECTOR, |
492 | x2apic_mode ? Xintr_x2apic_tlb : Xintr_lapic_tlb); | | 492 | x2apic_mode ? Xintr_x2apic_tlb : Xintr_lapic_tlb); |
493 | #endif | | 493 | #endif |
494 | idt_vec_reserve(iv, LAPIC_SPURIOUS_VECTOR); | | 494 | idt_vec_reserve(iv, LAPIC_SPURIOUS_VECTOR); |
495 | idt_vec_set(iv, LAPIC_SPURIOUS_VECTOR, Xintrspurious); | | 495 | idt_vec_set(iv, LAPIC_SPURIOUS_VECTOR, Xintrspurious); |
496 | | | 496 | |
497 | idt_vec_reserve(iv, LAPIC_TIMER_VECTOR); | | 497 | idt_vec_reserve(iv, LAPIC_TIMER_VECTOR); |
498 | idt_vec_set(iv, LAPIC_TIMER_VECTOR, | | 498 | idt_vec_set(iv, LAPIC_TIMER_VECTOR, |
499 | x2apic_mode ? Xintr_x2apic_ltimer : Xintr_lapic_ltimer); | | 499 | x2apic_mode ? Xintr_x2apic_ltimer : Xintr_lapic_ltimer); |
500 | } | | 500 | } |
501 | | | 501 | |
502 | static uint32_t | | 502 | static uint32_t |
503 | lapic_gettick(void) | | 503 | lapic_gettick(void) |
504 | { | | 504 | { |
505 | return lapic_readreg(LAPIC_CCR_TIMER); | | 505 | return lapic_readreg(LAPIC_CCR_TIMER); |
506 | } | | 506 | } |
507 | | | 507 | |
508 | #include <sys/kernel.h> /* for hz */ | | 508 | #include <sys/kernel.h> /* for hz */ |
509 | | | 509 | |
510 | uint32_t lapic_tval; | | 510 | uint32_t lapic_tval; |
511 | | | 511 | |
512 | /* | | 512 | /* |
513 | * this gets us up to a 4GHz busclock.... | | 513 | * this gets us up to a 4GHz busclock.... |
514 | */ | | 514 | */ |
515 | uint32_t lapic_per_second; | | 515 | uint32_t lapic_per_second; |
516 | uint32_t lapic_frac_usec_per_cycle; | | 516 | uint32_t lapic_frac_usec_per_cycle; |
517 | uint64_t lapic_frac_cycle_per_usec; | | 517 | uint64_t lapic_frac_cycle_per_usec; |
518 | uint32_t lapic_delaytab[26]; | | 518 | uint32_t lapic_delaytab[26]; |
519 | | | 519 | |
520 | static u_int | | 520 | static u_int |
521 | lapic_get_timecount(struct timecounter *tc) | | 521 | lapic_get_timecount(struct timecounter *tc) |
522 | { | | 522 | { |
523 | struct cpu_info *ci; | | 523 | struct cpu_info *ci; |
524 | uint32_t cur_timer; | | 524 | uint32_t cur_timer; |
525 | int s; | | 525 | int s; |
526 | | | 526 | |
527 | s = splhigh(); | | 527 | s = splhigh(); |
528 | ci = curcpu(); | | 528 | ci = curcpu(); |
529 | | | 529 | |
530 | /* | | 530 | /* |
531 | * Check for a race against the clockinterrupt. | | 531 | * Check for a race against the clockinterrupt. |
532 | * The update of ci_lapic_counter is blocked by splhigh() and | | 532 | * The update of ci_lapic_counter is blocked by splhigh() and |
533 | * the check for a pending clockinterrupt compensates for that. | | 533 | * the check for a pending clockinterrupt compensates for that. |
534 | * | | 534 | * |
535 | * If the current tick is almost the Initial Counter, explicitly | | 535 | * If the current tick is almost the Initial Counter, explicitly |
536 | * check for the pending interrupt bit as the interrupt delivery | | 536 | * check for the pending interrupt bit as the interrupt delivery |
537 | * could be asynchronious and compensate as well. | | 537 | * could be asynchronous and compensate as well. |
538 | * | | 538 | * |
539 | * This can't be done without splhigh() as the calling code might | | 539 | * This can't be done without splhigh() as the calling code might |
540 | * have masked the clockinterrupt already. | | 540 | * have masked the clockinterrupt already. |
541 | * | | 541 | * |
542 | * This code assumes that clockinterrupts are not missed. | | 542 | * This code assumes that clockinterrupts are not missed. |
543 | */ | | 543 | */ |
544 | cur_timer = lapic_gettick(); | | 544 | cur_timer = lapic_gettick(); |
545 | if (cur_timer >= lapic_tval - 1) { | | 545 | if (cur_timer >= lapic_tval - 1) { |
546 | uint16_t reg = LAPIC_IRR + LAPIC_TIMER_VECTOR / 32 * 16; | | 546 | uint16_t reg = LAPIC_IRR + LAPIC_TIMER_VECTOR / 32 * 16; |
547 | | | 547 | |
548 | if (lapic_readreg(reg) & (1 << (LAPIC_TIMER_VECTOR % 32))) { | | 548 | if (lapic_readreg(reg) & (1 << (LAPIC_TIMER_VECTOR % 32))) { |
549 | cur_timer -= lapic_tval; | | 549 | cur_timer -= lapic_tval; |
550 | } | | 550 | } |
551 | } else if (ci->ci_ipending & (1ULL << LIR_TIMER)) | | 551 | } else if (ci->ci_ipending & (1ULL << LIR_TIMER)) |
552 | cur_timer = lapic_gettick() - lapic_tval; | | 552 | cur_timer = lapic_gettick() - lapic_tval; |
553 | cur_timer = ci->ci_lapic_counter - cur_timer; | | 553 | cur_timer = ci->ci_lapic_counter - cur_timer; |
554 | splx(s); | | 554 | splx(s); |
555 | | | 555 | |
556 | return cur_timer; | | 556 | return cur_timer; |
557 | } | | 557 | } |
558 | | | 558 | |
559 | static struct timecounter lapic_timecounter = { | | 559 | static struct timecounter lapic_timecounter = { |
560 | .tc_get_timecount = lapic_get_timecount, | | 560 | .tc_get_timecount = lapic_get_timecount, |
561 | .tc_counter_mask = ~0u, | | 561 | .tc_counter_mask = ~0u, |
562 | .tc_name = "lapic", | | 562 | .tc_name = "lapic", |
563 | .tc_quality = | | 563 | .tc_quality = |
564 | #ifndef MULTIPROCESSOR | | 564 | #ifndef MULTIPROCESSOR |
565 | 2100, | | 565 | 2100, |
566 | #else | | 566 | #else |
567 | -100, /* per CPU state */ | | 567 | -100, /* per CPU state */ |
568 | #endif | | 568 | #endif |
569 | }; | | 569 | }; |
570 | | | 570 | |
571 | extern u_int i8254_get_timecount(struct timecounter *); | | 571 | extern u_int i8254_get_timecount(struct timecounter *); |
572 | | | 572 | |
573 | void | | 573 | void |
574 | lapic_clockintr(void *arg, struct intrframe *frame) | | 574 | lapic_clockintr(void *arg, struct intrframe *frame) |
575 | { | | 575 | { |
576 | struct cpu_info *ci = curcpu(); | | 576 | struct cpu_info *ci = curcpu(); |
577 | | | 577 | |
578 | ci->ci_lapic_counter += lapic_tval; | | 578 | ci->ci_lapic_counter += lapic_tval; |
579 | ci->ci_isources[LIR_TIMER]->is_evcnt.ev_count++; | | 579 | ci->ci_isources[LIR_TIMER]->is_evcnt.ev_count++; |
580 | hardclock((struct clockframe *)frame); | | 580 | hardclock((struct clockframe *)frame); |
581 | } | | 581 | } |
582 | | | 582 | |
583 | void | | 583 | void |
584 | lapic_reset(void) | | 584 | lapic_reset(void) |
585 | { | | 585 | { |
586 | | | 586 | |
587 | /* | | 587 | /* |
588 | * Mask the clock interrupt and set mode, | | 588 | * Mask the clock interrupt and set mode, |
589 | * then set divisor, | | 589 | * then set divisor, |
590 | * then unmask and set the vector. | | 590 | * then unmask and set the vector. |
591 | */ | | 591 | */ |
592 | lapic_writereg(LAPIC_LVT_TIMER, | | 592 | lapic_writereg(LAPIC_LVT_TIMER, |
593 | LAPIC_LVT_TMM_PERIODIC | LAPIC_LVT_MASKED); | | 593 | LAPIC_LVT_TMM_PERIODIC | LAPIC_LVT_MASKED); |
594 | lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1); | | 594 | lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1); |
595 | lapic_writereg(LAPIC_ICR_TIMER, lapic_tval); | | 595 | lapic_writereg(LAPIC_ICR_TIMER, lapic_tval); |
596 | lapic_writereg(LAPIC_LVT_TIMER, | | 596 | lapic_writereg(LAPIC_LVT_TIMER, |
597 | LAPIC_LVT_TMM_PERIODIC | LAPIC_TIMER_VECTOR); | | 597 | LAPIC_LVT_TMM_PERIODIC | LAPIC_TIMER_VECTOR); |
598 | lapic_writereg(LAPIC_EOI, 0); | | 598 | lapic_writereg(LAPIC_EOI, 0); |
599 | } | | 599 | } |
600 | | | 600 | |
601 | static void | | 601 | static void |
602 | lapic_initclock(void) | | 602 | lapic_initclock(void) |
603 | { | | 603 | { |
604 | | | 604 | |
605 | if (curcpu() == &cpu_info_primary) { | | 605 | if (curcpu() == &cpu_info_primary) { |
606 | /* | | 606 | /* |
607 | * Recalibrate the timer using the cycle counter, now that | | 607 | * Recalibrate the timer using the cycle counter, now that |
608 | * the cycle counter itself has been recalibrated. | | 608 | * the cycle counter itself has been recalibrated. |
609 | */ | | 609 | */ |
610 | lapic_calibrate_timer(true); | | 610 | lapic_calibrate_timer(true); |
611 | | | 611 | |
612 | /* | | 612 | /* |
613 | * Hook up time counter. This assume that all LAPICs have | | 613 | * Hook up time counter. This assume that all LAPICs have |
614 | * the same frequency. | | 614 | * the same frequency. |
615 | */ | | 615 | */ |
616 | lapic_timecounter.tc_frequency = lapic_per_second; | | 616 | lapic_timecounter.tc_frequency = lapic_per_second; |
617 | tc_init(&lapic_timecounter); | | 617 | tc_init(&lapic_timecounter); |
618 | } | | 618 | } |
619 | | | 619 | |
620 | /* Start local apic countdown timer running, in repeated mode. */ | | 620 | /* Start local apic countdown timer running, in repeated mode. */ |
621 | lapic_reset(); | | 621 | lapic_reset(); |
622 | } | | 622 | } |
623 | | | 623 | |
624 | /* | | 624 | /* |
625 | * Calibrate the local apic count-down timer (which is running at | | 625 | * Calibrate the local apic count-down timer (which is running at |
626 | * bus-clock speed) vs. the i8254 counter/timer (which is running at | | 626 | * bus-clock speed) vs. the i8254 counter/timer (which is running at |
627 | * a fixed rate). | | 627 | * a fixed rate). |
628 | * | | 628 | * |
629 | * The Intel MP spec says: "An MP operating system may use the IRQ8 | | 629 | * The Intel MP spec says: "An MP operating system may use the IRQ8 |
630 | * real-time clock as a reference to determine the actual APIC timer clock | | 630 | * real-time clock as a reference to determine the actual APIC timer clock |
631 | * speed." | | 631 | * speed." |
632 | * | | 632 | * |
633 | * We're actually using the IRQ0 timer. Hmm. | | 633 | * We're actually using the IRQ0 timer. Hmm. |
634 | */ | | 634 | */ |
635 | void | | 635 | void |
636 | lapic_calibrate_timer(bool secondpass) | | 636 | lapic_calibrate_timer(bool secondpass) |
637 | { | | 637 | { |
638 | struct cpu_info *ci = curcpu(); | | 638 | struct cpu_info *ci = curcpu(); |
639 | uint64_t tmp; | | 639 | uint64_t tmp; |
640 | int i; | | 640 | int i; |
641 | char tbuf[9]; | | 641 | char tbuf[9]; |
642 | | | 642 | |
643 | KASSERT(ci == &cpu_info_primary); | | 643 | KASSERT(ci == &cpu_info_primary); |
644 | | | 644 | |
645 | aprint_debug_dev(ci->ci_dev, "[re]calibrating local timer\n"); | | 645 | aprint_debug_dev(ci->ci_dev, "[re]calibrating local timer\n"); |
646 | | | 646 | |
647 | /* | | 647 | /* |
648 | * Configure timer to one-shot, interrupt masked, | | 648 | * Configure timer to one-shot, interrupt masked, |
649 | * large positive number. | | 649 | * large positive number. |
650 | */ | | 650 | */ |
651 | x86_disable_intr(); | | 651 | x86_disable_intr(); |
652 | lapic_writereg(LAPIC_LVT_TIMER, LAPIC_LVT_MASKED); | | 652 | lapic_writereg(LAPIC_LVT_TIMER, LAPIC_LVT_MASKED); |
653 | lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1); | | 653 | lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1); |
654 | lapic_writereg(LAPIC_ICR_TIMER, 0x80000000); | | 654 | lapic_writereg(LAPIC_ICR_TIMER, 0x80000000); |
655 | (void)lapic_gettick(); | | 655 | (void)lapic_gettick(); |
656 | | | 656 | |
657 | if (secondpass && cpu_hascounter()) { | | 657 | if (secondpass && cpu_hascounter()) { |
658 | /* | | 658 | /* |
659 | * Second pass calibration, using the TSC which has ideally | | 659 | * Second pass calibration, using the TSC which has ideally |
660 | * been calibrated using the HPET or information gleaned | | 660 | * been calibrated using the HPET or information gleaned |
661 | * from MSRs by this point. | | 661 | * from MSRs by this point. |
662 | */ | | 662 | */ |
663 | uint64_t l0, l1, t0, t1; | | 663 | uint64_t l0, l1, t0, t1; |
664 | | | 664 | |
665 | (void)cpu_counter(); | | 665 | (void)cpu_counter(); |
666 | t0 = cpu_counter(); | | 666 | t0 = cpu_counter(); |
667 | l0 = lapic_gettick(); | | 667 | l0 = lapic_gettick(); |
668 | t0 += cpu_counter(); | | 668 | t0 += cpu_counter(); |
669 | DELAY(50000); | | 669 | DELAY(50000); |
670 | t1 = cpu_counter(); | | 670 | t1 = cpu_counter(); |
671 | l1 = lapic_gettick(); | | 671 | l1 = lapic_gettick(); |
672 | t1 += cpu_counter(); | | 672 | t1 += cpu_counter(); |
673 | | | 673 | |
674 | tmp = (l0 - l1) * cpu_frequency(ci) / ((t1 - t0 + 1) / 2); | | 674 | tmp = (l0 - l1) * cpu_frequency(ci) / ((t1 - t0 + 1) / 2); |
675 | lapic_per_second = rounddown(tmp + 500, 1000); | | 675 | lapic_per_second = rounddown(tmp + 500, 1000); |
676 | } else if (lapic_per_second == 0) { | | 676 | } else if (lapic_per_second == 0) { |
677 | /* | | 677 | /* |
678 | * Inaccurate first pass calibration using the i8254. | | 678 | * Inaccurate first pass calibration using the i8254. |
679 | */ | | 679 | */ |
680 | unsigned int seen, delta, initial_i8254, initial_lapic; | | 680 | unsigned int seen, delta, initial_i8254, initial_lapic; |
681 | unsigned int cur_i8254, cur_lapic; | | 681 | unsigned int cur_i8254, cur_lapic; |
682 | | | 682 | |
683 | (void)gettick(); | | 683 | (void)gettick(); |
684 | initial_lapic = lapic_gettick(); | | 684 | initial_lapic = lapic_gettick(); |
685 | initial_i8254 = gettick(); | | 685 | initial_i8254 = gettick(); |
686 | for (seen = 0; seen < TIMER_FREQ / 100; seen += delta) { | | 686 | for (seen = 0; seen < TIMER_FREQ / 100; seen += delta) { |
687 | cur_i8254 = gettick(); | | 687 | cur_i8254 = gettick(); |
688 | if (cur_i8254 > initial_i8254) | | 688 | if (cur_i8254 > initial_i8254) |
689 | delta = x86_rtclock_tval - (cur_i8254 - initial_i8254); | | 689 | delta = x86_rtclock_tval - (cur_i8254 - initial_i8254); |
690 | else | | 690 | else |
691 | delta = initial_i8254 - cur_i8254; | | 691 | delta = initial_i8254 - cur_i8254; |
692 | initial_i8254 = cur_i8254; | | 692 | initial_i8254 = cur_i8254; |
693 | } | | 693 | } |
694 | cur_lapic = lapic_gettick(); | | 694 | cur_lapic = lapic_gettick(); |
695 | tmp = initial_lapic - cur_lapic; | | 695 | tmp = initial_lapic - cur_lapic; |
696 | lapic_per_second = (tmp * TIMER_FREQ + seen / 2) / seen; | | 696 | lapic_per_second = (tmp * TIMER_FREQ + seen / 2) / seen; |
697 | } | | 697 | } |
698 | x86_enable_intr(); | | 698 | x86_enable_intr(); |
699 | | | 699 | |
700 | humanize_number(tbuf, sizeof(tbuf), lapic_per_second, "Hz", 1000); | | 700 | humanize_number(tbuf, sizeof(tbuf), lapic_per_second, "Hz", 1000); |
701 | aprint_debug_dev(ci->ci_dev, "apic clock running at %s\n", tbuf); | | 701 | aprint_debug_dev(ci->ci_dev, "apic clock running at %s\n", tbuf); |
702 | | | 702 | |
703 | if (lapic_per_second != 0) { | | 703 | if (lapic_per_second != 0) { |
704 | /* | | 704 | /* |
705 | * reprogram the apic timer to run in periodic mode. | | 705 | * reprogram the apic timer to run in periodic mode. |
706 | * XXX need to program timer on other CPUs, too. | | 706 | * XXX need to program timer on other CPUs, too. |
707 | */ | | 707 | */ |
708 | lapic_tval = (lapic_per_second * 2) / hz; | | 708 | lapic_tval = (lapic_per_second * 2) / hz; |
709 | lapic_tval = (lapic_tval / 2) + (lapic_tval & 0x1); | | 709 | lapic_tval = (lapic_tval / 2) + (lapic_tval & 0x1); |
710 | | | 710 | |
711 | lapic_writereg(LAPIC_LVT_TIMER, LAPIC_LVT_TMM_PERIODIC | | 711 | lapic_writereg(LAPIC_LVT_TIMER, LAPIC_LVT_TMM_PERIODIC |
712 | | LAPIC_LVT_MASKED | LAPIC_TIMER_VECTOR); | | 712 | | LAPIC_LVT_MASKED | LAPIC_TIMER_VECTOR); |
713 | lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1); | | 713 | lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1); |
714 | lapic_writereg(LAPIC_ICR_TIMER, lapic_tval); | | 714 | lapic_writereg(LAPIC_ICR_TIMER, lapic_tval); |
715 | | | 715 | |
716 | /* | | 716 | /* |
717 | * Compute fixed-point ratios between cycles and | | 717 | * Compute fixed-point ratios between cycles and |
718 | * microseconds to avoid having to do any division | | 718 | * microseconds to avoid having to do any division |
719 | * in lapic_delay. | | 719 | * in lapic_delay. |
720 | */ | | 720 | */ |
721 | | | 721 | |
722 | tmp = (1000000 * (uint64_t)1 << 32) / lapic_per_second; | | 722 | tmp = (1000000 * (uint64_t)1 << 32) / lapic_per_second; |
723 | lapic_frac_usec_per_cycle = tmp; | | 723 | lapic_frac_usec_per_cycle = tmp; |
724 | | | 724 | |
725 | tmp = (lapic_per_second * (uint64_t)1 << 32) / 1000000; | | 725 | tmp = (lapic_per_second * (uint64_t)1 << 32) / 1000000; |
726 | | | 726 | |
727 | lapic_frac_cycle_per_usec = tmp; | | 727 | lapic_frac_cycle_per_usec = tmp; |
728 | | | 728 | |
729 | /* | | 729 | /* |
730 | * Compute delay in cycles for likely short delays in usec. | | 730 | * Compute delay in cycles for likely short delays in usec. |
731 | */ | | 731 | */ |
732 | for (i = 0; i < 26; i++) | | 732 | for (i = 0; i < 26; i++) |
733 | lapic_delaytab[i] = (lapic_frac_cycle_per_usec * i) >> | | 733 | lapic_delaytab[i] = (lapic_frac_cycle_per_usec * i) >> |
734 | 32; | | 734 | 32; |
735 | | | 735 | |
736 | /* | | 736 | /* |
737 | * Apply workaround for broken periodic timer under KVM | | 737 | * Apply workaround for broken periodic timer under KVM |
738 | */ | | 738 | */ |
739 | if (vm_guest == VM_GUEST_KVM) { | | 739 | if (vm_guest == VM_GUEST_KVM) { |
740 | lapic_broken_periodic = true; | | 740 | lapic_broken_periodic = true; |
741 | lapic_timecounter.tc_quality = -100; | | 741 | lapic_timecounter.tc_quality = -100; |
742 | aprint_debug_dev(ci->ci_dev, | | 742 | aprint_debug_dev(ci->ci_dev, |
743 | "applying KVM timer workaround\n"); | | 743 | "applying KVM timer workaround\n"); |
744 | } | | 744 | } |
745 | | | 745 | |
746 | /* | | 746 | /* |
747 | * Now that the timer's calibrated, use the apic timer routines | | 747 | * Now that the timer's calibrated, use the apic timer routines |
748 | * for all our timing needs.. | | 748 | * for all our timing needs.. |
749 | */ | | 749 | */ |
750 | if (!secondpass) { | | 750 | if (!secondpass) { |
751 | delay_func = lapic_delay; | | 751 | delay_func = lapic_delay; |
752 | x86_initclock_func = lapic_initclock; | | 752 | x86_initclock_func = lapic_initclock; |
753 | initrtclock(0); | | 753 | initrtclock(0); |
754 | } | | 754 | } |
755 | } | | 755 | } |
756 | } | | 756 | } |
757 | | | 757 | |
758 | /* | | 758 | /* |
759 | * delay for N usec. | | 759 | * delay for N usec. |
760 | */ | | 760 | */ |
761 | | | 761 | |
762 | static void | | 762 | static void |
763 | lapic_delay(unsigned int usec) | | 763 | lapic_delay(unsigned int usec) |
764 | { | | 764 | { |
765 | int32_t xtick, otick; | | 765 | int32_t xtick, otick; |
766 | int64_t deltat; | | 766 | int64_t deltat; |
767 | | | 767 | |
768 | /* XXX Bad to disable preemption, but it's tied to the cpu. */ | | 768 | /* XXX Bad to disable preemption, but it's tied to the cpu. */ |
769 | kpreempt_disable(); | | 769 | kpreempt_disable(); |
770 | otick = lapic_gettick(); | | 770 | otick = lapic_gettick(); |
771 | | | 771 | |
772 | if (usec <= 0) { | | 772 | if (usec <= 0) { |
773 | kpreempt_enable(); | | 773 | kpreempt_enable(); |
774 | return; | | 774 | return; |
775 | } | | 775 | } |
776 | | | 776 | |
777 | if (usec <= 25) | | 777 | if (usec <= 25) |
778 | deltat = lapic_delaytab[usec]; | | 778 | deltat = lapic_delaytab[usec]; |
779 | else | | 779 | else |
780 | deltat = (lapic_frac_cycle_per_usec * usec) >> 32; | | 780 | deltat = (lapic_frac_cycle_per_usec * usec) >> 32; |
781 | | | 781 | |
782 | while (deltat > 0) { | | 782 | while (deltat > 0) { |
783 | xtick = lapic_gettick(); | | 783 | xtick = lapic_gettick(); |
784 | if (lapic_broken_periodic && xtick == 0 && otick == 0) { | | 784 | if (lapic_broken_periodic && xtick == 0 && otick == 0) { |
785 | lapic_reset(); | | 785 | lapic_reset(); |
786 | xtick = lapic_gettick(); | | 786 | xtick = lapic_gettick(); |
787 | if (xtick == 0) | | 787 | if (xtick == 0) |
788 | panic("lapic timer stopped ticking"); | | 788 | panic("lapic timer stopped ticking"); |
789 | } | | 789 | } |
790 | if (xtick > otick) | | 790 | if (xtick > otick) |
791 | deltat -= lapic_tval - (xtick - otick); | | 791 | deltat -= lapic_tval - (xtick - otick); |
792 | else | | 792 | else |
793 | deltat -= otick - xtick; | | 793 | deltat -= otick - xtick; |
794 | otick = xtick; | | 794 | otick = xtick; |
795 | | | 795 | |
796 | x86_pause(); | | 796 | x86_pause(); |
797 | } | | 797 | } |
798 | kpreempt_enable(); | | 798 | kpreempt_enable(); |
799 | } | | 799 | } |
800 | | | 800 | |
801 | /* | | 801 | /* |
802 | * XXX the following belong mostly or partly elsewhere.. | | 802 | * XXX the following belong mostly or partly elsewhere.. |
803 | */ | | 803 | */ |
804 | | | 804 | |
805 | static void | | 805 | static void |
806 | i82489_icr_wait(void) | | 806 | i82489_icr_wait(void) |
807 | { | | 807 | { |
808 | #ifdef DIAGNOSTIC | | 808 | #ifdef DIAGNOSTIC |
809 | unsigned j = 100000; | | 809 | unsigned j = 100000; |
810 | #endif /* DIAGNOSTIC */ | | 810 | #endif /* DIAGNOSTIC */ |
811 | | | 811 | |
812 | while ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0) { | | 812 | while ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0) { |
813 | x86_pause(); | | 813 | x86_pause(); |
814 | #ifdef DIAGNOSTIC | | 814 | #ifdef DIAGNOSTIC |
815 | j--; | | 815 | j--; |
816 | if (j == 0) | | 816 | if (j == 0) |
817 | panic("i82489_icr_wait: busy"); | | 817 | panic("i82489_icr_wait: busy"); |
818 | #endif /* DIAGNOSTIC */ | | 818 | #endif /* DIAGNOSTIC */ |
819 | } | | 819 | } |
820 | } | | 820 | } |
821 | | | 821 | |
822 | static int | | 822 | static int |
823 | i82489_ipi_init(int target) | | 823 | i82489_ipi_init(int target) |
824 | { | | 824 | { |
825 | uint32_t esr; | | 825 | uint32_t esr; |
826 | | | 826 | |
827 | i82489_writereg(LAPIC_ESR, 0); | | 827 | i82489_writereg(LAPIC_ESR, 0); |
828 | (void)i82489_readreg(LAPIC_ESR); | | 828 | (void)i82489_readreg(LAPIC_ESR); |
829 | | | 829 | |
830 | i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT); | | 830 | i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT); |
831 | | | 831 | |
832 | i82489_writereg(LAPIC_ICRLO, LAPIC_DLMODE_INIT | LAPIC_LEVEL_ASSERT); | | 832 | i82489_writereg(LAPIC_ICRLO, LAPIC_DLMODE_INIT | LAPIC_LEVEL_ASSERT); |
833 | i82489_icr_wait(); | | 833 | i82489_icr_wait(); |
834 | delay_func(10000); | | 834 | delay_func(10000); |
835 | i82489_writereg(LAPIC_ICRLO, | | 835 | i82489_writereg(LAPIC_ICRLO, |
836 | LAPIC_DLMODE_INIT | LAPIC_TRIGMODE_LEVEL | LAPIC_LEVEL_DEASSERT); | | 836 | LAPIC_DLMODE_INIT | LAPIC_TRIGMODE_LEVEL | LAPIC_LEVEL_DEASSERT); |
837 | i82489_icr_wait(); | | 837 | i82489_icr_wait(); |
838 | | | 838 | |
839 | if ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0) | | 839 | if ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0) |
840 | return EBUSY; | | 840 | return EBUSY; |
841 | | | 841 | |
842 | esr = i82489_readreg(LAPIC_ESR); | | 842 | esr = i82489_readreg(LAPIC_ESR); |
843 | if (esr != 0) | | 843 | if (esr != 0) |
844 | aprint_debug("%s: ESR %08x\n", __func__, esr); | | 844 | aprint_debug("%s: ESR %08x\n", __func__, esr); |
845 | | | 845 | |
846 | return 0; | | 846 | return 0; |
847 | } | | 847 | } |
848 | | | 848 | |
849 | static int | | 849 | static int |
850 | i82489_ipi_startup(int target, int vec) | | 850 | i82489_ipi_startup(int target, int vec) |
851 | { | | 851 | { |
852 | uint32_t esr; | | 852 | uint32_t esr; |
853 | | | 853 | |
854 | i82489_writereg(LAPIC_ESR, 0); | | 854 | i82489_writereg(LAPIC_ESR, 0); |
855 | (void)i82489_readreg(LAPIC_ESR); | | 855 | (void)i82489_readreg(LAPIC_ESR); |
856 | | | 856 | |
857 | i82489_icr_wait(); | | 857 | i82489_icr_wait(); |
858 | i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT); | | 858 | i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT); |
859 | i82489_writereg(LAPIC_ICRLO, vec | LAPIC_DLMODE_STARTUP | | | 859 | i82489_writereg(LAPIC_ICRLO, vec | LAPIC_DLMODE_STARTUP | |
860 | LAPIC_LEVEL_ASSERT); | | 860 | LAPIC_LEVEL_ASSERT); |
861 | i82489_icr_wait(); | | 861 | i82489_icr_wait(); |
862 | | | 862 | |
863 | if ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0) | | 863 | if ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0) |
864 | return EBUSY; | | 864 | return EBUSY; |
865 | | | 865 | |
866 | esr = i82489_readreg(LAPIC_ESR); | | 866 | esr = i82489_readreg(LAPIC_ESR); |
867 | if (esr != 0) | | 867 | if (esr != 0) |
868 | aprint_debug("%s: ESR %08x\n", __func__, esr); | | 868 | aprint_debug("%s: ESR %08x\n", __func__, esr); |
869 | | | 869 | |
870 | return 0; | | 870 | return 0; |
871 | } | | 871 | } |
872 | | | 872 | |
873 | static int | | 873 | static int |
874 | i82489_ipi(int vec, int target, int dl) | | 874 | i82489_ipi(int vec, int target, int dl) |
875 | { | | 875 | { |
876 | int result, s; | | 876 | int result, s; |
877 | | | 877 | |
878 | s = splhigh(); | | 878 | s = splhigh(); |
879 | | | 879 | |
880 | i82489_icr_wait(); | | 880 | i82489_icr_wait(); |
881 | | | 881 | |
882 | if ((target & LAPIC_DEST_MASK) == 0) | | 882 | if ((target & LAPIC_DEST_MASK) == 0) |
883 | i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT); | | 883 | i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT); |
884 | | | 884 | |
885 | i82489_writereg(LAPIC_ICRLO, | | 885 | i82489_writereg(LAPIC_ICRLO, |
886 | (target & LAPIC_DEST_MASK) | vec | dl | LAPIC_LEVEL_ASSERT); | | 886 | (target & LAPIC_DEST_MASK) | vec | dl | LAPIC_LEVEL_ASSERT); |
887 | | | 887 | |
888 | #ifdef DIAGNOSTIC | | 888 | #ifdef DIAGNOSTIC |
889 | i82489_icr_wait(); | | 889 | i82489_icr_wait(); |
890 | result = (i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) ? EBUSY : 0; | | 890 | result = (i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) ? EBUSY : 0; |
891 | #else | | 891 | #else |
892 | /* Don't wait - if it doesn't go, we're in big trouble anyway. */ | | 892 | /* Don't wait - if it doesn't go, we're in big trouble anyway. */ |
893 | result = 0; | | 893 | result = 0; |
894 | #endif | | 894 | #endif |
895 | splx(s); | | 895 | splx(s); |
896 | | | 896 | |
897 | return result; | | 897 | return result; |
898 | } | | 898 | } |
899 | | | 899 | |
900 | static int | | 900 | static int |
901 | x2apic_ipi_init(int target) | | 901 | x2apic_ipi_init(int target) |
902 | { | | 902 | { |
903 | | | 903 | |
904 | x2apic_write_icr(target, LAPIC_DLMODE_INIT | LAPIC_LEVEL_ASSERT); | | 904 | x2apic_write_icr(target, LAPIC_DLMODE_INIT | LAPIC_LEVEL_ASSERT); |
905 | | | 905 | |
906 | delay_func(10000); | | 906 | delay_func(10000); |
907 | | | 907 | |
908 | x2apic_write_icr(0, | | 908 | x2apic_write_icr(0, |
909 | LAPIC_DLMODE_INIT | LAPIC_TRIGMODE_LEVEL | LAPIC_LEVEL_DEASSERT); | | 909 | LAPIC_DLMODE_INIT | LAPIC_TRIGMODE_LEVEL | LAPIC_LEVEL_DEASSERT); |
910 | | | 910 | |
911 | return 0; | | 911 | return 0; |
912 | } | | 912 | } |
913 | | | 913 | |
914 | static int | | 914 | static int |
915 | x2apic_ipi_startup(int target, int vec) | | 915 | x2apic_ipi_startup(int target, int vec) |
916 | { | | 916 | { |
917 | | | 917 | |
918 | x2apic_write_icr(target, | | 918 | x2apic_write_icr(target, |
919 | vec | LAPIC_DLMODE_STARTUP | LAPIC_LEVEL_ASSERT); | | 919 | vec | LAPIC_DLMODE_STARTUP | LAPIC_LEVEL_ASSERT); |
920 | | | 920 | |
921 | return 0; | | 921 | return 0; |
922 | } | | 922 | } |
923 | | | 923 | |
924 | static int | | 924 | static int |
925 | x2apic_ipi(int vec, int target, int dl) | | 925 | x2apic_ipi(int vec, int target, int dl) |
926 | { | | 926 | { |
927 | uint32_t dest_id = 0; | | 927 | uint32_t dest_id = 0; |
928 | | | 928 | |
929 | if ((target & LAPIC_DEST_MASK) == 0) | | 929 | if ((target & LAPIC_DEST_MASK) == 0) |
930 | dest_id = target; | | 930 | dest_id = target; |
931 | | | 931 | |
932 | x2apic_write_icr(dest_id, | | 932 | x2apic_write_icr(dest_id, |
933 | (target & LAPIC_DEST_MASK) | vec | dl | LAPIC_LEVEL_ASSERT); | | 933 | (target & LAPIC_DEST_MASK) | vec | dl | LAPIC_LEVEL_ASSERT); |
934 | | | 934 | |
935 | return 0; | | 935 | return 0; |
936 | } | | 936 | } |
937 | | | 937 | |
938 | int | | 938 | int |
939 | x86_ipi_init(int target) | | 939 | x86_ipi_init(int target) |
940 | { | | 940 | { |
941 | if (x2apic_mode) | | 941 | if (x2apic_mode) |
942 | return x2apic_ipi_init(target); | | 942 | return x2apic_ipi_init(target); |
943 | return i82489_ipi_init(target); | | 943 | return i82489_ipi_init(target); |
944 | } | | 944 | } |
945 | | | 945 | |
946 | int | | 946 | int |
947 | x86_ipi_startup(int target, int vec) | | 947 | x86_ipi_startup(int target, int vec) |
948 | { | | 948 | { |
949 | if (x2apic_mode) | | 949 | if (x2apic_mode) |
950 | return x2apic_ipi_startup(target, vec); | | 950 | return x2apic_ipi_startup(target, vec); |
951 | return i82489_ipi_startup(target, vec); | | 951 | return i82489_ipi_startup(target, vec); |
952 | } | | 952 | } |
953 | | | 953 | |
954 | /* | | 954 | /* |
955 | * Using 'pin numbers' as: | | 955 | * Using 'pin numbers' as: |
956 | * 0 - timer | | 956 | * 0 - timer |
957 | * 1 - thermal | | 957 | * 1 - thermal |
958 | * 2 - PCINT | | 958 | * 2 - PCINT |
959 | * 3 - LVINT0 | | 959 | * 3 - LVINT0 |
960 | * 4 - LVINT1 | | 960 | * 4 - LVINT1 |
961 | * 5 - LVERR | | 961 | * 5 - LVERR |
962 | */ | | 962 | */ |
963 | | | 963 | |
964 | static void | | 964 | static void |
965 | lapic_hwmask(struct pic *pic, int pin) | | 965 | lapic_hwmask(struct pic *pic, int pin) |
966 | { | | 966 | { |
967 | int reg; | | 967 | int reg; |
968 | uint32_t val; | | 968 | uint32_t val; |
969 | | | 969 | |
970 | reg = LAPIC_LVT_TIMER + (pin << 4); | | 970 | reg = LAPIC_LVT_TIMER + (pin << 4); |
971 | val = lapic_readreg(reg); | | 971 | val = lapic_readreg(reg); |
972 | val |= LAPIC_LVT_MASKED; | | 972 | val |= LAPIC_LVT_MASKED; |
973 | lapic_writereg(reg, val); | | 973 | lapic_writereg(reg, val); |
974 | } | | 974 | } |
975 | | | 975 | |
976 | static void | | 976 | static void |
977 | lapic_hwunmask(struct pic *pic, int pin) | | 977 | lapic_hwunmask(struct pic *pic, int pin) |
978 | { | | 978 | { |
979 | int reg; | | 979 | int reg; |
980 | uint32_t val; | | 980 | uint32_t val; |
981 | | | 981 | |
982 | reg = LAPIC_LVT_TIMER + (pin << 4); | | 982 | reg = LAPIC_LVT_TIMER + (pin << 4); |
983 | val = lapic_readreg(reg); | | 983 | val = lapic_readreg(reg); |
984 | val &= ~LAPIC_LVT_MASKED; | | 984 | val &= ~LAPIC_LVT_MASKED; |
985 | lapic_writereg(reg, val); | | 985 | lapic_writereg(reg, val); |
986 | } | | 986 | } |
987 | | | 987 | |
988 | static void | | 988 | static void |
989 | lapic_setup(struct pic *pic, struct cpu_info *ci, | | 989 | lapic_setup(struct pic *pic, struct cpu_info *ci, |
990 | int pin, int idtvec, int type) | | 990 | int pin, int idtvec, int type) |
991 | { | | 991 | { |
992 | } | | 992 | } |
993 | | | 993 | |
994 | void | | 994 | void |
995 | lapic_dump(void) | | 995 | lapic_dump(void) |
996 | { | | 996 | { |
997 | struct cpu_info *ci = curcpu(); | | 997 | struct cpu_info *ci = curcpu(); |
998 | | | 998 | |
999 | #define APIC_LVT_PRINT(ci, where, idx, lvtreg) \ | | 999 | #define APIC_LVT_PRINT(ci, where, idx, lvtreg) \ |
1000 | apic_format_redir(device_xname(ci->ci_dev), where, (idx), \ | | 1000 | apic_format_redir(device_xname(ci->ci_dev), where, (idx), \ |
1001 | APIC_VECTYPE_LAPIC_LVT, 0, lapic_readreg(lvtreg)) | | 1001 | APIC_VECTYPE_LAPIC_LVT, 0, lapic_readreg(lvtreg)) |
1002 | | | 1002 | |
1003 | APIC_LVT_PRINT(ci, "cmci", 0, LAPIC_LVT_CMCI); | | 1003 | APIC_LVT_PRINT(ci, "cmci", 0, LAPIC_LVT_CMCI); |
1004 | APIC_LVT_PRINT(ci, "timer", 0, LAPIC_LVT_TIMER); | | 1004 | APIC_LVT_PRINT(ci, "timer", 0, LAPIC_LVT_TIMER); |
1005 | APIC_LVT_PRINT(ci, "thermal", 0, LAPIC_LVT_THERM); | | 1005 | APIC_LVT_PRINT(ci, "thermal", 0, LAPIC_LVT_THERM); |
1006 | APIC_LVT_PRINT(ci, "pcint", 0, LAPIC_LVT_PCINT); | | 1006 | APIC_LVT_PRINT(ci, "pcint", 0, LAPIC_LVT_PCINT); |
1007 | APIC_LVT_PRINT(ci, "lint", 0, LAPIC_LVT_LINT0); | | 1007 | APIC_LVT_PRINT(ci, "lint", 0, LAPIC_LVT_LINT0); |
1008 | APIC_LVT_PRINT(ci, "lint", 1, LAPIC_LVT_LINT1); | | 1008 | APIC_LVT_PRINT(ci, "lint", 1, LAPIC_LVT_LINT1); |
1009 | APIC_LVT_PRINT(ci, "err", 0, LAPIC_LVT_ERR); | | 1009 | APIC_LVT_PRINT(ci, "err", 0, LAPIC_LVT_ERR); |
1010 | | | 1010 | |
1011 | #undef APIC_LVT_PRINT | | 1011 | #undef APIC_LVT_PRINT |
1012 | } | | 1012 | } |
1013 | #else /* XENPV */ | | 1013 | #else /* XENPV */ |
1014 | void | | 1014 | void |
1015 | lapic_boot_init(paddr_t lapic_base) | | 1015 | lapic_boot_init(paddr_t lapic_base) |
1016 | { | | 1016 | { |
1017 | } | | 1017 | } |
1018 | #endif /* XENPV */ | | 1018 | #endif /* XENPV */ |