Sun Feb 25 18:27:54 2024 UTC (110d)
s/asynchronious/asynchronous/ in comment.


(andvar)
diff -r1.89 -r1.90 src/sys/arch/x86/x86/lapic.c

cvs diff -r1.89 -r1.90 src/sys/arch/x86/x86/lapic.c (switch to unified diff)

--- src/sys/arch/x86/x86/lapic.c 2022/09/07 00:40:19 1.89
+++ src/sys/arch/x86/x86/lapic.c 2024/02/25 18:27:54 1.90
@@ -1,1018 +1,1018 @@ @@ -1,1018 +1,1018 @@
1/* $NetBSD: lapic.c,v 1.89 2022/09/07 00:40:19 knakahara Exp $ */ 1/* $NetBSD: lapic.c,v 1.90 2024/02/25 18:27:54 andvar Exp $ */
2 2
3/*- 3/*-
4 * Copyright (c) 2000, 2008, 2020 The NetBSD Foundation, Inc. 4 * Copyright (c) 2000, 2008, 2020 The NetBSD Foundation, Inc.
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * This code is derived from software contributed to The NetBSD Foundation 7 * This code is derived from software contributed to The NetBSD Foundation
8 * by RedBack Networks Inc. 8 * by RedBack Networks Inc.
9 * 9 *
10 * Author: Bill Sommerfeld 10 * Author: Bill Sommerfeld
11 * 11 *
12 * Redistribution and use in source and binary forms, with or without 12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions 13 * modification, are permitted provided that the following conditions
14 * are met: 14 * are met:
15 * 1. Redistributions of source code must retain the above copyright 15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer. 16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright 17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the 18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution. 19 * documentation and/or other materials provided with the distribution.
20 * 20 *
21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE. 31 * POSSIBILITY OF SUCH DAMAGE.
32 */ 32 */
33 33
34#include <sys/cdefs.h> 34#include <sys/cdefs.h>
35__KERNEL_RCSID(0, "$NetBSD: lapic.c,v 1.89 2022/09/07 00:40:19 knakahara Exp $"); 35__KERNEL_RCSID(0, "$NetBSD: lapic.c,v 1.90 2024/02/25 18:27:54 andvar Exp $");
36 36
37#include "acpica.h" 37#include "acpica.h"
38#include "ioapic.h" 38#include "ioapic.h"
39#include "opt_acpi.h" 39#include "opt_acpi.h"
40#include "opt_ddb.h" 40#include "opt_ddb.h"
41#include "opt_mpbios.h" /* for MPDEBUG */ 41#include "opt_mpbios.h" /* for MPDEBUG */
42#include "opt_multiprocessor.h" 42#include "opt_multiprocessor.h"
43#include "opt_ntp.h" 43#include "opt_ntp.h"
44#include "opt_xen.h" 44#include "opt_xen.h"
45 45
46 46
47#include <sys/param.h> 47#include <sys/param.h>
48#include <sys/proc.h> 48#include <sys/proc.h>
49#include <sys/systm.h> 49#include <sys/systm.h>
50#include <sys/device.h> 50#include <sys/device.h>
51#include <sys/timetc.h> 51#include <sys/timetc.h>
52 52
53#include <uvm/uvm_extern.h> 53#include <uvm/uvm_extern.h>
54 54
55#include <dev/ic/i8253reg.h> 55#include <dev/ic/i8253reg.h>
56 56
57#include <x86/machdep.h> 57#include <x86/machdep.h>
58#include <machine/cpu.h> 58#include <machine/cpu.h>
59#include <machine/cpu_counter.h> 59#include <machine/cpu_counter.h>
60#include <machine/cpufunc.h> 60#include <machine/cpufunc.h>
61#include <machine/cpuvar.h> 61#include <machine/cpuvar.h>
62#include <machine/pmap.h> 62#include <machine/pmap.h>
63#include <machine/vmparam.h> 63#include <machine/vmparam.h>
64#include <machine/mpacpi.h> 64#include <machine/mpacpi.h>
65#include <machine/mpbiosvar.h> 65#include <machine/mpbiosvar.h>
66#include <machine/pcb.h> 66#include <machine/pcb.h>
67#include <machine/pmap_private.h> 67#include <machine/pmap_private.h>
68#include <machine/specialreg.h> 68#include <machine/specialreg.h>
69#include <machine/segments.h> 69#include <machine/segments.h>
70#include <x86/x86/tsc.h> 70#include <x86/x86/tsc.h>
71#include <x86/i82093var.h> 71#include <x86/i82093var.h>
72 72
73#include <machine/apicvar.h> 73#include <machine/apicvar.h>
74#include <machine/i82489reg.h> 74#include <machine/i82489reg.h>
75#include <machine/i82489var.h> 75#include <machine/i82489var.h>
76 76
77#ifndef XENPV 77#ifndef XENPV
78#if NACPICA > 0 78#if NACPICA > 0
79#include <dev/acpi/acpica.h> 79#include <dev/acpi/acpica.h>
80#include <dev/acpi/acpivar.h> 80#include <dev/acpi/acpivar.h>
81#endif 81#endif
82 82
83#ifdef DDB 83#ifdef DDB
84#include <machine/db_machdep.h> 84#include <machine/db_machdep.h>
85#ifdef MULTIPROCESSOR 85#ifdef MULTIPROCESSOR
86#ifdef __x86_64__ 86#ifdef __x86_64__
87typedef void (vector)(void); 87typedef void (vector)(void);
88extern vector Xintr_x2apic_ddbipi; 88extern vector Xintr_x2apic_ddbipi;
89extern int ddb_vec; 89extern int ddb_vec;
90#endif 90#endif
91#endif 91#endif
92#endif 92#endif
93 93
94#include <dev/vmt/vmtreg.h> /* for vmt_hvcall() */ 94#include <dev/vmt/vmtreg.h> /* for vmt_hvcall() */
95#include <dev/vmt/vmtvar.h> /* for vmt_hvcall() */ 95#include <dev/vmt/vmtvar.h> /* for vmt_hvcall() */
96 96
97/* Referenced from vector.S */ 97/* Referenced from vector.S */
98void lapic_clockintr(void *, struct intrframe *); 98void lapic_clockintr(void *, struct intrframe *);
99 99
100static void lapic_delay(unsigned int); 100static void lapic_delay(unsigned int);
101static uint32_t lapic_gettick(void); 101static uint32_t lapic_gettick(void);
102static void lapic_setup_bsp(paddr_t); 102static void lapic_setup_bsp(paddr_t);
103static void lapic_map(paddr_t); 103static void lapic_map(paddr_t);
104 104
105static void lapic_hwmask(struct pic *, int); 105static void lapic_hwmask(struct pic *, int);
106static void lapic_hwunmask(struct pic *, int); 106static void lapic_hwunmask(struct pic *, int);
107static void lapic_setup(struct pic *, struct cpu_info *, int, int, int); 107static void lapic_setup(struct pic *, struct cpu_info *, int, int, int);
108/* Make it public to call via ddb */ 108/* Make it public to call via ddb */
109void lapic_dump(void); 109void lapic_dump(void);
110 110
111struct pic local_pic = { 111struct pic local_pic = {
112 .pic_name = "lapic", 112 .pic_name = "lapic",
113 .pic_type = PIC_LAPIC, 113 .pic_type = PIC_LAPIC,
114 .pic_lock = __SIMPLELOCK_UNLOCKED, 114 .pic_lock = __SIMPLELOCK_UNLOCKED,
115 .pic_hwmask = lapic_hwmask, 115 .pic_hwmask = lapic_hwmask,
116 .pic_hwunmask = lapic_hwunmask, 116 .pic_hwunmask = lapic_hwunmask,
117 .pic_addroute = lapic_setup, 117 .pic_addroute = lapic_setup,
118 .pic_delroute = lapic_setup, 118 .pic_delroute = lapic_setup,
119 .pic_intr_get_devname = x86_intr_get_devname, 119 .pic_intr_get_devname = x86_intr_get_devname,
120 .pic_intr_get_assigned = x86_intr_get_assigned, 120 .pic_intr_get_assigned = x86_intr_get_assigned,
121 .pic_intr_get_count = x86_intr_get_count, 121 .pic_intr_get_count = x86_intr_get_count,
122}; 122};
123 123
124static int i82489_ipi(int vec, int target, int dl); 124static int i82489_ipi(int vec, int target, int dl);
125static int x2apic_ipi(int vec, int target, int dl); 125static int x2apic_ipi(int vec, int target, int dl);
126int (*x86_ipi)(int, int, int) = i82489_ipi; 126int (*x86_ipi)(int, int, int) = i82489_ipi;
127 127
128bool x2apic_mode __read_mostly; 128bool x2apic_mode __read_mostly;
129#ifdef LAPIC_ENABLE_X2APIC 129#ifdef LAPIC_ENABLE_X2APIC
130bool x2apic_enable = true; 130bool x2apic_enable = true;
131#else 131#else
132bool x2apic_enable = false; 132bool x2apic_enable = false;
133#endif 133#endif
134 134
135static bool lapic_broken_periodic __read_mostly; 135static bool lapic_broken_periodic __read_mostly;
136 136
137static uint32_t 137static uint32_t
138i82489_readreg(u_int reg) 138i82489_readreg(u_int reg)
139{ 139{
140 return *((volatile uint32_t *)(local_apic_va + reg)); 140 return *((volatile uint32_t *)(local_apic_va + reg));
141} 141}
142 142
143static void 143static void
144i82489_writereg(u_int reg, uint32_t val) 144i82489_writereg(u_int reg, uint32_t val)
145{ 145{
146 *((volatile uint32_t *)(local_apic_va + reg)) = val; 146 *((volatile uint32_t *)(local_apic_va + reg)) = val;
147} 147}
148 148
149static uint32_t 149static uint32_t
150i82489_cpu_number(void) 150i82489_cpu_number(void)
151{ 151{
152 return i82489_readreg(LAPIC_ID) >> LAPIC_ID_SHIFT; 152 return i82489_readreg(LAPIC_ID) >> LAPIC_ID_SHIFT;
153} 153}
154 154
155static uint32_t 155static uint32_t
156x2apic_readreg(u_int reg) 156x2apic_readreg(u_int reg)
157{ 157{
158 return rdmsr(MSR_X2APIC_BASE + (reg >> 4)); 158 return rdmsr(MSR_X2APIC_BASE + (reg >> 4));
159} 159}
160 160
161static void 161static void
162x2apic_writereg(u_int reg, uint32_t val) 162x2apic_writereg(u_int reg, uint32_t val)
163{ 163{
164 x86_mfence(); 164 x86_mfence();
165 wrmsr(MSR_X2APIC_BASE + (reg >> 4), val); 165 wrmsr(MSR_X2APIC_BASE + (reg >> 4), val);
166} 166}
167 167
168static void 168static void
169x2apic_writereg64(u_int reg, uint64_t val) 169x2apic_writereg64(u_int reg, uint64_t val)
170{ 170{
171 KDASSERT(reg == LAPIC_ICRLO); 171 KDASSERT(reg == LAPIC_ICRLO);
172 x86_mfence(); 172 x86_mfence();
173 wrmsr(MSR_X2APIC_BASE + (reg >> 4), val); 173 wrmsr(MSR_X2APIC_BASE + (reg >> 4), val);
174} 174}
175 175
176static void 176static void
177x2apic_write_icr(uint32_t hi, uint32_t lo) 177x2apic_write_icr(uint32_t hi, uint32_t lo)
178{ 178{
179 x2apic_writereg64(LAPIC_ICRLO, ((uint64_t)hi << 32) | lo); 179 x2apic_writereg64(LAPIC_ICRLO, ((uint64_t)hi << 32) | lo);
180} 180}
181 181
182static uint32_t 182static uint32_t
183x2apic_cpu_number(void) 183x2apic_cpu_number(void)
184{ 184{
185 return x2apic_readreg(LAPIC_ID); 185 return x2apic_readreg(LAPIC_ID);
186} 186}
187 187
188uint32_t 188uint32_t
189lapic_readreg(u_int reg) 189lapic_readreg(u_int reg)
190{ 190{
191 if (x2apic_mode) 191 if (x2apic_mode)
192 return x2apic_readreg(reg); 192 return x2apic_readreg(reg);
193 return i82489_readreg(reg); 193 return i82489_readreg(reg);
194} 194}
195 195
196void 196void
197lapic_writereg(u_int reg, uint32_t val) 197lapic_writereg(u_int reg, uint32_t val)
198{ 198{
199 if (x2apic_mode) 199 if (x2apic_mode)
200 x2apic_writereg(reg, val); 200 x2apic_writereg(reg, val);
201 else 201 else
202 i82489_writereg(reg, val); 202 i82489_writereg(reg, val);
203} 203}
204 204
205void 205void
206lapic_write_tpri(uint32_t val) 206lapic_write_tpri(uint32_t val)
207{ 207{
208 208
209 val &= LAPIC_TPRI_MASK; 209 val &= LAPIC_TPRI_MASK;
210#ifdef i386 210#ifdef i386
211 lapic_writereg(LAPIC_TPRI, val); 211 lapic_writereg(LAPIC_TPRI, val);
212#else 212#else
213 lcr8(val >> 4); 213 lcr8(val >> 4);
214#endif 214#endif
215} 215}
216 216
217uint32_t 217uint32_t
218lapic_cpu_number(void) 218lapic_cpu_number(void)
219{ 219{
220 if (x2apic_mode) 220 if (x2apic_mode)
221 return x2apic_cpu_number(); 221 return x2apic_cpu_number();
222 return i82489_cpu_number(); 222 return i82489_cpu_number();
223} 223}
224 224
225static void 225static void
226lapic_enable_x2apic(void) 226lapic_enable_x2apic(void)
227{ 227{
228 uint64_t apicbase; 228 uint64_t apicbase;
229 229
230 apicbase = rdmsr(MSR_APICBASE); 230 apicbase = rdmsr(MSR_APICBASE);
231 if (!ISSET(apicbase, APICBASE_EN)) { 231 if (!ISSET(apicbase, APICBASE_EN)) {
232 apicbase |= APICBASE_EN; 232 apicbase |= APICBASE_EN;
233 wrmsr(MSR_APICBASE, apicbase); 233 wrmsr(MSR_APICBASE, apicbase);
234 } 234 }
235 apicbase |= APICBASE_EXTD; 235 apicbase |= APICBASE_EXTD;
236 wrmsr(MSR_APICBASE, apicbase); 236 wrmsr(MSR_APICBASE, apicbase);
237} 237}
238 238
239bool 239bool
240lapic_is_x2apic(void) 240lapic_is_x2apic(void)
241{ 241{
242 uint64_t msr; 242 uint64_t msr;
243 243
244 if (!ISSET(cpu_feature[0], CPUID_APIC) || 244 if (!ISSET(cpu_feature[0], CPUID_APIC) ||
245 rdmsr_safe(MSR_APICBASE, &msr) == EFAULT) 245 rdmsr_safe(MSR_APICBASE, &msr) == EFAULT)
246 return false; 246 return false;
247 return (msr & (APICBASE_EN | APICBASE_EXTD)) == 247 return (msr & (APICBASE_EN | APICBASE_EXTD)) ==
248 (APICBASE_EN | APICBASE_EXTD); 248 (APICBASE_EN | APICBASE_EXTD);
249} 249}
250 250
251/* 251/*
252 * Initialize the local APIC on the BSP. 252 * Initialize the local APIC on the BSP.
253 */ 253 */
254static void 254static void
255lapic_setup_bsp(paddr_t lapic_base) 255lapic_setup_bsp(paddr_t lapic_base)
256{ 256{
257 u_int regs[6]; 257 u_int regs[6];
258 const char *reason = NULL; 258 const char *reason = NULL;
259 const char *hw_vendor; 259 const char *hw_vendor;
260 bool bios_x2apic; 260 bool bios_x2apic;
261 261
262 if (ISSET(cpu_feature[1], CPUID2_X2APIC)) { 262 if (ISSET(cpu_feature[1], CPUID2_X2APIC)) {
263#if NACPICA > 0 263#if NACPICA > 0
264 if (acpi_present) { 264 if (acpi_present) {
265 ACPI_TABLE_DMAR *dmar; 265 ACPI_TABLE_DMAR *dmar;
266 ACPI_STATUS status; 266 ACPI_STATUS status;
267 267
268 /* 268 /*
269 * Automatically detect several configurations where 269 * Automatically detect several configurations where
270 * x2APIC mode is known to cause troubles. User can 270 * x2APIC mode is known to cause troubles. User can
271 * override the setting with hw.x2apic_enable tunable. 271 * override the setting with hw.x2apic_enable tunable.
272 */ 272 */
273 status = AcpiGetTable(ACPI_SIG_DMAR, 1, 273 status = AcpiGetTable(ACPI_SIG_DMAR, 1,
274 (ACPI_TABLE_HEADER **)&dmar); 274 (ACPI_TABLE_HEADER **)&dmar);
275 if (ACPI_SUCCESS(status)) { 275 if (ACPI_SUCCESS(status)) {
276 if (ISSET(dmar->Flags, ACPI_DMAR_X2APIC_OPT_OUT)) { 276 if (ISSET(dmar->Flags, ACPI_DMAR_X2APIC_OPT_OUT)) {
277 reason = "by DMAR table"; 277 reason = "by DMAR table";
278 } 278 }
279 AcpiPutTable(&dmar->Header); 279 AcpiPutTable(&dmar->Header);
280 } 280 }
281 } 281 }
282#endif /* NACPICA > 0 */ 282#endif /* NACPICA > 0 */
283 if (vm_guest == VM_GUEST_VMWARE) { 283 if (vm_guest == VM_GUEST_VMWARE) {
284 vmt_hvcall(VM_CMD_GET_VCPU_INFO, regs); 284 vmt_hvcall(VM_CMD_GET_VCPU_INFO, regs);
285 if (ISSET(regs[0], VCPUINFO_VCPU_RESERVED) || 285 if (ISSET(regs[0], VCPUINFO_VCPU_RESERVED) ||
286 !ISSET(regs[0], VCPUINFO_LEGACY_X2APIC)) 286 !ISSET(regs[0], VCPUINFO_LEGACY_X2APIC))
287 reason = "inside VMWare without intr " 287 reason = "inside VMWare without intr "
288 "redirection"; 288 "redirection";
289 } else if (vm_guest == VM_GUEST_XENHVM) { 289 } else if (vm_guest == VM_GUEST_XENHVM) {
290 reason = "due to running under XEN"; 290 reason = "due to running under XEN";
291 } else if (vm_guest == VM_GUEST_NO && 291 } else if (vm_guest == VM_GUEST_NO &&
292 CPUID_TO_FAMILY(curcpu()->ci_signature) == 6 && 292 CPUID_TO_FAMILY(curcpu()->ci_signature) == 6 &&
293 CPUID_TO_MODEL(curcpu()->ci_signature) == 0x2a) { 293 CPUID_TO_MODEL(curcpu()->ci_signature) == 0x2a) {
294 hw_vendor = pmf_get_platform("board-vendor"); 294 hw_vendor = pmf_get_platform("board-vendor");
295 if (hw_vendor != NULL) { 295 if (hw_vendor != NULL) {
296 /* 296 /*
297 * It seems that some Lenovo and ASUS 297 * It seems that some Lenovo and ASUS
298 * SandyBridge-based notebook BIOSes have a bug 298 * SandyBridge-based notebook BIOSes have a bug
299 * which prevents booting AP in x2APIC mode. 299 * which prevents booting AP in x2APIC mode.
300 * Since the only way to detect mobile CPU is 300 * Since the only way to detect mobile CPU is
301 * to check northbridge pci id, which cannot be 301 * to check northbridge pci id, which cannot be
302 * done that early, disable x2APIC for all 302 * done that early, disable x2APIC for all
303 * Lenovo and ASUS SandyBridge machines. 303 * Lenovo and ASUS SandyBridge machines.
304 */ 304 */
305 if (strcmp(hw_vendor, "LENOVO") == 0 || 305 if (strcmp(hw_vendor, "LENOVO") == 0 ||
306 strcmp(hw_vendor, "ASUSTeK Computer Inc.") == 0) { 306 strcmp(hw_vendor, "ASUSTeK Computer Inc.") == 0) {
307 reason = "for a suspected SandyBridge " 307 reason = "for a suspected SandyBridge "
308 "BIOS bug"; 308 "BIOS bug";
309 } 309 }
310 } 310 }
311 } 311 }
312 bios_x2apic = lapic_is_x2apic(); 312 bios_x2apic = lapic_is_x2apic();
313 if (reason != NULL && bios_x2apic) { 313 if (reason != NULL && bios_x2apic) {
314 aprint_verbose("x2APIC should be disabled %s but " 314 aprint_verbose("x2APIC should be disabled %s but "
315 "already enabled by BIOS; enabling.\n", reason); 315 "already enabled by BIOS; enabling.\n", reason);
316 reason = NULL; 316 reason = NULL;
317 } 317 }
318 if (reason == NULL) 318 if (reason == NULL)
319 x2apic_mode = true; 319 x2apic_mode = true;
320 else 320 else
321 aprint_verbose("x2APIC available but disabled %s\n", 321 aprint_verbose("x2APIC available but disabled %s\n",
322 reason); 322 reason);
323 if (x2apic_enable != x2apic_mode) { 323 if (x2apic_enable != x2apic_mode) {
324 if (bios_x2apic && !x2apic_enable) 324 if (bios_x2apic && !x2apic_enable)
325 aprint_verbose("x2APIC disabled by user and " 325 aprint_verbose("x2APIC disabled by user and "
326 "enabled by BIOS; ignoring user setting.\n"); 326 "enabled by BIOS; ignoring user setting.\n");
327 else 327 else
328 x2apic_mode = x2apic_enable; 328 x2apic_mode = x2apic_enable;
329 } 329 }
330 } 330 }
331 if (x2apic_mode) { 331 if (x2apic_mode) {
332 x86_ipi = x2apic_ipi; 332 x86_ipi = x2apic_ipi;
333#if NIOAPIC > 0 333#if NIOAPIC > 0
334 struct ioapic_softc *ioapic; 334 struct ioapic_softc *ioapic;
335 for (ioapic = ioapics; ioapic != NULL; ioapic = ioapic->sc_next) { 335 for (ioapic = ioapics; ioapic != NULL; ioapic = ioapic->sc_next) {
336 ioapic->sc_pic.pic_edge_stubs = x2apic_edge_stubs; 336 ioapic->sc_pic.pic_edge_stubs = x2apic_edge_stubs;
337 ioapic->sc_pic.pic_level_stubs = x2apic_level_stubs; 337 ioapic->sc_pic.pic_level_stubs = x2apic_level_stubs;
338 } 338 }
339#endif 339#endif
340#if defined(DDB) && defined(MULTIPROCESSOR) 340#if defined(DDB) && defined(MULTIPROCESSOR)
341#ifdef __x86_64__ 341#ifdef __x86_64__
342 struct idt_vec *iv = &(cpu_info_primary.ci_idtvec); 342 struct idt_vec *iv = &(cpu_info_primary.ci_idtvec);
343 idt_descriptor_t *idt = iv->iv_idt; 343 idt_descriptor_t *idt = iv->iv_idt;
344 set_idtgate(&idt[ddb_vec], &Xintr_x2apic_ddbipi, 1, 344 set_idtgate(&idt[ddb_vec], &Xintr_x2apic_ddbipi, 1,
345 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 345 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
346#else 346#else
347 /* 347 /*
348 * Set DDB IPI handler in cpu_set_tss_gates() when cpu0 is 348 * Set DDB IPI handler in cpu_set_tss_gates() when cpu0 is
349 * attached. 349 * attached.
350 */ 350 */
351#endif 351#endif
352#endif 352#endif
353 353
354 x86_disable_intr(); 354 x86_disable_intr();
355 lapic_enable_x2apic(); 355 lapic_enable_x2apic();
356#ifdef MULTIPROCESSOR 356#ifdef MULTIPROCESSOR
357 cpu_init_first(); /* Catch up to changed cpu_number() */ 357 cpu_init_first(); /* Catch up to changed cpu_number() */
358#endif 358#endif
359 lapic_write_tpri(0); 359 lapic_write_tpri(0);
360 x86_enable_intr(); 360 x86_enable_intr();
361 } else 361 } else
362 lapic_map(lapic_base); 362 lapic_map(lapic_base);
363} 363}
364 364
365static void 365static void
366lapic_map(paddr_t lapic_base) 366lapic_map(paddr_t lapic_base)
367{ 367{
368 pt_entry_t *pte; 368 pt_entry_t *pte;
369 vaddr_t va = local_apic_va; 369 vaddr_t va = local_apic_va;
370 370
371 /* 371 /*
372 * If the CPU has an APIC MSR, use it and ignore the supplied value: 372 * If the CPU has an APIC MSR, use it and ignore the supplied value:
373 * some ACPI implementations have been observed to pass bad values. 373 * some ACPI implementations have been observed to pass bad values.
374 * Additionally, ensure that the lapic is enabled as we are committed 374 * Additionally, ensure that the lapic is enabled as we are committed
375 * to using it at this point. Be conservative and assume that the MSR 375 * to using it at this point. Be conservative and assume that the MSR
376 * is not present on the Pentium (is it?). 376 * is not present on the Pentium (is it?).
377 */ 377 */
378 if (CPUID_TO_FAMILY(curcpu()->ci_signature) >= 6) { 378 if (CPUID_TO_FAMILY(curcpu()->ci_signature) >= 6) {
379 lapic_base = (paddr_t)rdmsr(MSR_APICBASE); 379 lapic_base = (paddr_t)rdmsr(MSR_APICBASE);
380 if ((lapic_base & APICBASE_PHYSADDR) == 0) { 380 if ((lapic_base & APICBASE_PHYSADDR) == 0) {
381 lapic_base |= LAPIC_BASE; 381 lapic_base |= LAPIC_BASE;
382 } 382 }
383 wrmsr(MSR_APICBASE, lapic_base | APICBASE_EN); 383 wrmsr(MSR_APICBASE, lapic_base | APICBASE_EN);
384 lapic_base &= APICBASE_PHYSADDR; 384 lapic_base &= APICBASE_PHYSADDR;
385 } 385 }
386 386
387 x86_disable_intr(); 387 x86_disable_intr();
388 388
389 /* 389 /*
390 * Map local apic. If we have a local apic, it's safe to assume 390 * Map local apic. If we have a local apic, it's safe to assume
391 * we're on a 486 or better and can use invlpg and non-cacheable PTE's 391 * we're on a 486 or better and can use invlpg and non-cacheable PTE's
392 * 392 *
393 * Whap the PTE "by hand" rather than calling pmap_kenter_pa because 393 * Whap the PTE "by hand" rather than calling pmap_kenter_pa because
394 * the latter will attempt to invoke TLB shootdown code just as we 394 * the latter will attempt to invoke TLB shootdown code just as we
395 * might have changed the value of cpu_number().. 395 * might have changed the value of cpu_number()..
396 */ 396 */
397 397
398 pte = kvtopte(va); 398 pte = kvtopte(va);
399 *pte = lapic_base | PTE_W | PTE_P | PTE_PCD | pmap_pg_g | pmap_pg_nx; 399 *pte = lapic_base | PTE_W | PTE_P | PTE_PCD | pmap_pg_g | pmap_pg_nx;
400 invlpg(va); 400 invlpg(va);
401 401
402#ifdef MULTIPROCESSOR 402#ifdef MULTIPROCESSOR
403 cpu_init_first(); /* Catch up to changed cpu_number() */ 403 cpu_init_first(); /* Catch up to changed cpu_number() */
404#endif 404#endif
405 405
406 lapic_write_tpri(0); 406 lapic_write_tpri(0);
407 x86_enable_intr(); 407 x86_enable_intr();
408} 408}
409 409
410/* 410/*
411 * enable local apic 411 * enable local apic
412 */ 412 */
413void 413void
414lapic_enable(void) 414lapic_enable(void)
415{ 415{
416 lapic_writereg(LAPIC_SVR, LAPIC_SVR_ENABLE | LAPIC_SPURIOUS_VECTOR); 416 lapic_writereg(LAPIC_SVR, LAPIC_SVR_ENABLE | LAPIC_SPURIOUS_VECTOR);
417} 417}
418 418
419void 419void
420lapic_set_lvt(void) 420lapic_set_lvt(void)
421{ 421{
422 struct cpu_info *ci = curcpu(); 422 struct cpu_info *ci = curcpu();
423 int i; 423 int i;
424 struct mp_intr_map *mpi; 424 struct mp_intr_map *mpi;
425 uint32_t lint0, lint1; 425 uint32_t lint0, lint1;
426 426
427#ifdef MULTIPROCESSOR 427#ifdef MULTIPROCESSOR
428 if (mp_verbose) { 428 if (mp_verbose) {
429 apic_format_redir(device_xname(ci->ci_dev), "prelint", 0, 429 apic_format_redir(device_xname(ci->ci_dev), "prelint", 0,
430 APIC_VECTYPE_LAPIC_LVT, 0, lapic_readreg(LAPIC_LVT_LINT0)); 430 APIC_VECTYPE_LAPIC_LVT, 0, lapic_readreg(LAPIC_LVT_LINT0));
431 apic_format_redir(device_xname(ci->ci_dev), "prelint", 1, 431 apic_format_redir(device_xname(ci->ci_dev), "prelint", 1,
432 APIC_VECTYPE_LAPIC_LVT, 0, lapic_readreg(LAPIC_LVT_LINT1)); 432 APIC_VECTYPE_LAPIC_LVT, 0, lapic_readreg(LAPIC_LVT_LINT1));
433 } 433 }
434#endif 434#endif
435 435
436 /* 436 /*
437 * If an I/O APIC has been attached, assume that it is used instead of 437 * If an I/O APIC has been attached, assume that it is used instead of
438 * the 8259A for interrupt delivery. Otherwise request the LAPIC to 438 * the 8259A for interrupt delivery. Otherwise request the LAPIC to
439 * get external interrupts via LINT0 for the primary CPU. 439 * get external interrupts via LINT0 for the primary CPU.
440 */ 440 */
441 lint0 = LAPIC_DLMODE_EXTINT; 441 lint0 = LAPIC_DLMODE_EXTINT;
442 if (nioapics > 0 || !CPU_IS_PRIMARY(curcpu())) 442 if (nioapics > 0 || !CPU_IS_PRIMARY(curcpu()))
443 lint0 |= LAPIC_LVT_MASKED; 443 lint0 |= LAPIC_LVT_MASKED;
444 lapic_writereg(LAPIC_LVT_LINT0, lint0); 444 lapic_writereg(LAPIC_LVT_LINT0, lint0);
445 445
446 /* 446 /*
447 * Non Maskable Interrupts are to be delivered to the primary CPU. 447 * Non Maskable Interrupts are to be delivered to the primary CPU.
448 */ 448 */
449 lint1 = LAPIC_DLMODE_NMI; 449 lint1 = LAPIC_DLMODE_NMI;
450 if (!CPU_IS_PRIMARY(curcpu())) 450 if (!CPU_IS_PRIMARY(curcpu()))
451 lint1 |= LAPIC_LVT_MASKED; 451 lint1 |= LAPIC_LVT_MASKED;
452 lapic_writereg(LAPIC_LVT_LINT1, lint1); 452 lapic_writereg(LAPIC_LVT_LINT1, lint1);
453 453
454 for (i = 0; i < mp_nintr; i++) { 454 for (i = 0; i < mp_nintr; i++) {
455 mpi = &mp_intrs[i]; 455 mpi = &mp_intrs[i];
456 if (mpi->ioapic == NULL && (mpi->cpu_id == MPS_ALL_APICS || 456 if (mpi->ioapic == NULL && (mpi->cpu_id == MPS_ALL_APICS ||
457 mpi->cpu_id == ci->ci_cpuid)) { 457 mpi->cpu_id == ci->ci_cpuid)) {
458 if (mpi->ioapic_pin > 1) 458 if (mpi->ioapic_pin > 1)
459 aprint_error_dev(ci->ci_dev, 459 aprint_error_dev(ci->ci_dev,
460 "%s: WARNING: bad pin value %d\n", 460 "%s: WARNING: bad pin value %d\n",
461 __func__, mpi->ioapic_pin); 461 __func__, mpi->ioapic_pin);
462 if (mpi->ioapic_pin == 0) 462 if (mpi->ioapic_pin == 0)
463 lapic_writereg(LAPIC_LVT_LINT0, mpi->redir); 463 lapic_writereg(LAPIC_LVT_LINT0, mpi->redir);
464 else 464 else
465 lapic_writereg(LAPIC_LVT_LINT1, mpi->redir); 465 lapic_writereg(LAPIC_LVT_LINT1, mpi->redir);
466 } 466 }
467 } 467 }
468 468
469#ifdef MULTIPROCESSOR 469#ifdef MULTIPROCESSOR
470 if (mp_verbose) 470 if (mp_verbose)
471 lapic_dump(); 471 lapic_dump();
472#endif 472#endif
473} 473}
474 474
475/* 475/*
476 * Initialize fixed idt vectors for use by local apic. 476 * Initialize fixed idt vectors for use by local apic.
477 */ 477 */
478void 478void
479lapic_boot_init(paddr_t lapic_base) 479lapic_boot_init(paddr_t lapic_base)
480{ 480{
481 struct idt_vec *iv = &(cpu_info_primary.ci_idtvec); 481 struct idt_vec *iv = &(cpu_info_primary.ci_idtvec);
482 482
483 lapic_setup_bsp(lapic_base); 483 lapic_setup_bsp(lapic_base);
484 484
485#ifdef MULTIPROCESSOR 485#ifdef MULTIPROCESSOR
486 idt_vec_reserve(iv, LAPIC_IPI_VECTOR); 486 idt_vec_reserve(iv, LAPIC_IPI_VECTOR);
487 idt_vec_set(iv, LAPIC_IPI_VECTOR, 487 idt_vec_set(iv, LAPIC_IPI_VECTOR,
488 x2apic_mode ? Xintr_x2apic_ipi : Xintr_lapic_ipi); 488 x2apic_mode ? Xintr_x2apic_ipi : Xintr_lapic_ipi);
489 489
490 idt_vec_reserve(iv, LAPIC_TLB_VECTOR); 490 idt_vec_reserve(iv, LAPIC_TLB_VECTOR);
491 idt_vec_set(iv, LAPIC_TLB_VECTOR, 491 idt_vec_set(iv, LAPIC_TLB_VECTOR,
492 x2apic_mode ? Xintr_x2apic_tlb : Xintr_lapic_tlb); 492 x2apic_mode ? Xintr_x2apic_tlb : Xintr_lapic_tlb);
493#endif 493#endif
494 idt_vec_reserve(iv, LAPIC_SPURIOUS_VECTOR); 494 idt_vec_reserve(iv, LAPIC_SPURIOUS_VECTOR);
495 idt_vec_set(iv, LAPIC_SPURIOUS_VECTOR, Xintrspurious); 495 idt_vec_set(iv, LAPIC_SPURIOUS_VECTOR, Xintrspurious);
496 496
497 idt_vec_reserve(iv, LAPIC_TIMER_VECTOR); 497 idt_vec_reserve(iv, LAPIC_TIMER_VECTOR);
498 idt_vec_set(iv, LAPIC_TIMER_VECTOR, 498 idt_vec_set(iv, LAPIC_TIMER_VECTOR,
499 x2apic_mode ? Xintr_x2apic_ltimer : Xintr_lapic_ltimer); 499 x2apic_mode ? Xintr_x2apic_ltimer : Xintr_lapic_ltimer);
500} 500}
501 501
502static uint32_t 502static uint32_t
503lapic_gettick(void) 503lapic_gettick(void)
504{ 504{
505 return lapic_readreg(LAPIC_CCR_TIMER); 505 return lapic_readreg(LAPIC_CCR_TIMER);
506} 506}
507 507
508#include <sys/kernel.h> /* for hz */ 508#include <sys/kernel.h> /* for hz */
509 509
510uint32_t lapic_tval; 510uint32_t lapic_tval;
511 511
512/* 512/*
513 * this gets us up to a 4GHz busclock.... 513 * this gets us up to a 4GHz busclock....
514 */ 514 */
515uint32_t lapic_per_second; 515uint32_t lapic_per_second;
516uint32_t lapic_frac_usec_per_cycle; 516uint32_t lapic_frac_usec_per_cycle;
517uint64_t lapic_frac_cycle_per_usec; 517uint64_t lapic_frac_cycle_per_usec;
518uint32_t lapic_delaytab[26]; 518uint32_t lapic_delaytab[26];
519 519
520static u_int 520static u_int
521lapic_get_timecount(struct timecounter *tc) 521lapic_get_timecount(struct timecounter *tc)
522{ 522{
523 struct cpu_info *ci; 523 struct cpu_info *ci;
524 uint32_t cur_timer; 524 uint32_t cur_timer;
525 int s; 525 int s;
526 526
527 s = splhigh(); 527 s = splhigh();
528 ci = curcpu(); 528 ci = curcpu();
529 529
530 /* 530 /*
531 * Check for a race against the clockinterrupt. 531 * Check for a race against the clockinterrupt.
532 * The update of ci_lapic_counter is blocked by splhigh() and 532 * The update of ci_lapic_counter is blocked by splhigh() and
533 * the check for a pending clockinterrupt compensates for that. 533 * the check for a pending clockinterrupt compensates for that.
534 * 534 *
535 * If the current tick is almost the Initial Counter, explicitly 535 * If the current tick is almost the Initial Counter, explicitly
536 * check for the pending interrupt bit as the interrupt delivery 536 * check for the pending interrupt bit as the interrupt delivery
537 * could be asynchronious and compensate as well. 537 * could be asynchronous and compensate as well.
538 * 538 *
539 * This can't be done without splhigh() as the calling code might 539 * This can't be done without splhigh() as the calling code might
540 * have masked the clockinterrupt already. 540 * have masked the clockinterrupt already.
541 * 541 *
542 * This code assumes that clockinterrupts are not missed. 542 * This code assumes that clockinterrupts are not missed.
543 */ 543 */
544 cur_timer = lapic_gettick(); 544 cur_timer = lapic_gettick();
545 if (cur_timer >= lapic_tval - 1) { 545 if (cur_timer >= lapic_tval - 1) {
546 uint16_t reg = LAPIC_IRR + LAPIC_TIMER_VECTOR / 32 * 16; 546 uint16_t reg = LAPIC_IRR + LAPIC_TIMER_VECTOR / 32 * 16;
547 547
548 if (lapic_readreg(reg) & (1 << (LAPIC_TIMER_VECTOR % 32))) { 548 if (lapic_readreg(reg) & (1 << (LAPIC_TIMER_VECTOR % 32))) {
549 cur_timer -= lapic_tval; 549 cur_timer -= lapic_tval;
550 } 550 }
551 } else if (ci->ci_ipending & (1ULL << LIR_TIMER)) 551 } else if (ci->ci_ipending & (1ULL << LIR_TIMER))
552 cur_timer = lapic_gettick() - lapic_tval; 552 cur_timer = lapic_gettick() - lapic_tval;
553 cur_timer = ci->ci_lapic_counter - cur_timer; 553 cur_timer = ci->ci_lapic_counter - cur_timer;
554 splx(s); 554 splx(s);
555 555
556 return cur_timer; 556 return cur_timer;
557} 557}
558 558
559static struct timecounter lapic_timecounter = { 559static struct timecounter lapic_timecounter = {
560 .tc_get_timecount = lapic_get_timecount, 560 .tc_get_timecount = lapic_get_timecount,
561 .tc_counter_mask = ~0u, 561 .tc_counter_mask = ~0u,
562 .tc_name = "lapic", 562 .tc_name = "lapic",
563 .tc_quality = 563 .tc_quality =
564#ifndef MULTIPROCESSOR 564#ifndef MULTIPROCESSOR
565 2100, 565 2100,
566#else 566#else
567 -100, /* per CPU state */ 567 -100, /* per CPU state */
568#endif 568#endif
569}; 569};
570 570
571extern u_int i8254_get_timecount(struct timecounter *); 571extern u_int i8254_get_timecount(struct timecounter *);
572 572
573void 573void
574lapic_clockintr(void *arg, struct intrframe *frame) 574lapic_clockintr(void *arg, struct intrframe *frame)
575{ 575{
576 struct cpu_info *ci = curcpu(); 576 struct cpu_info *ci = curcpu();
577 577
578 ci->ci_lapic_counter += lapic_tval; 578 ci->ci_lapic_counter += lapic_tval;
579 ci->ci_isources[LIR_TIMER]->is_evcnt.ev_count++; 579 ci->ci_isources[LIR_TIMER]->is_evcnt.ev_count++;
580 hardclock((struct clockframe *)frame); 580 hardclock((struct clockframe *)frame);
581} 581}
582 582
583void 583void
584lapic_reset(void) 584lapic_reset(void)
585{ 585{
586 586
587 /* 587 /*
588 * Mask the clock interrupt and set mode, 588 * Mask the clock interrupt and set mode,
589 * then set divisor, 589 * then set divisor,
590 * then unmask and set the vector. 590 * then unmask and set the vector.
591 */ 591 */
592 lapic_writereg(LAPIC_LVT_TIMER, 592 lapic_writereg(LAPIC_LVT_TIMER,
593 LAPIC_LVT_TMM_PERIODIC | LAPIC_LVT_MASKED); 593 LAPIC_LVT_TMM_PERIODIC | LAPIC_LVT_MASKED);
594 lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1); 594 lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1);
595 lapic_writereg(LAPIC_ICR_TIMER, lapic_tval); 595 lapic_writereg(LAPIC_ICR_TIMER, lapic_tval);
596 lapic_writereg(LAPIC_LVT_TIMER, 596 lapic_writereg(LAPIC_LVT_TIMER,
597 LAPIC_LVT_TMM_PERIODIC | LAPIC_TIMER_VECTOR); 597 LAPIC_LVT_TMM_PERIODIC | LAPIC_TIMER_VECTOR);
598 lapic_writereg(LAPIC_EOI, 0); 598 lapic_writereg(LAPIC_EOI, 0);
599} 599}
600 600
601static void 601static void
602lapic_initclock(void) 602lapic_initclock(void)
603{ 603{
604 604
605 if (curcpu() == &cpu_info_primary) { 605 if (curcpu() == &cpu_info_primary) {
606 /* 606 /*
607 * Recalibrate the timer using the cycle counter, now that 607 * Recalibrate the timer using the cycle counter, now that
608 * the cycle counter itself has been recalibrated. 608 * the cycle counter itself has been recalibrated.
609 */ 609 */
610 lapic_calibrate_timer(true); 610 lapic_calibrate_timer(true);
611 611
612 /* 612 /*
613 * Hook up time counter. This assume that all LAPICs have 613 * Hook up time counter. This assume that all LAPICs have
614 * the same frequency. 614 * the same frequency.
615 */ 615 */
616 lapic_timecounter.tc_frequency = lapic_per_second; 616 lapic_timecounter.tc_frequency = lapic_per_second;
617 tc_init(&lapic_timecounter); 617 tc_init(&lapic_timecounter);
618 } 618 }
619 619
620 /* Start local apic countdown timer running, in repeated mode. */ 620 /* Start local apic countdown timer running, in repeated mode. */
621 lapic_reset(); 621 lapic_reset();
622} 622}
623 623
624/* 624/*
625 * Calibrate the local apic count-down timer (which is running at 625 * Calibrate the local apic count-down timer (which is running at
626 * bus-clock speed) vs. the i8254 counter/timer (which is running at 626 * bus-clock speed) vs. the i8254 counter/timer (which is running at
627 * a fixed rate). 627 * a fixed rate).
628 * 628 *
629 * The Intel MP spec says: "An MP operating system may use the IRQ8 629 * The Intel MP spec says: "An MP operating system may use the IRQ8
630 * real-time clock as a reference to determine the actual APIC timer clock 630 * real-time clock as a reference to determine the actual APIC timer clock
631 * speed." 631 * speed."
632 * 632 *
633 * We're actually using the IRQ0 timer. Hmm. 633 * We're actually using the IRQ0 timer. Hmm.
634 */ 634 */
635void 635void
636lapic_calibrate_timer(bool secondpass) 636lapic_calibrate_timer(bool secondpass)
637{ 637{
638 struct cpu_info *ci = curcpu(); 638 struct cpu_info *ci = curcpu();
639 uint64_t tmp; 639 uint64_t tmp;
640 int i; 640 int i;
641 char tbuf[9]; 641 char tbuf[9];
642 642
643 KASSERT(ci == &cpu_info_primary); 643 KASSERT(ci == &cpu_info_primary);
644 644
645 aprint_debug_dev(ci->ci_dev, "[re]calibrating local timer\n"); 645 aprint_debug_dev(ci->ci_dev, "[re]calibrating local timer\n");
646 646
647 /* 647 /*
648 * Configure timer to one-shot, interrupt masked, 648 * Configure timer to one-shot, interrupt masked,
649 * large positive number. 649 * large positive number.
650 */ 650 */
651 x86_disable_intr(); 651 x86_disable_intr();
652 lapic_writereg(LAPIC_LVT_TIMER, LAPIC_LVT_MASKED); 652 lapic_writereg(LAPIC_LVT_TIMER, LAPIC_LVT_MASKED);
653 lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1); 653 lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1);
654 lapic_writereg(LAPIC_ICR_TIMER, 0x80000000); 654 lapic_writereg(LAPIC_ICR_TIMER, 0x80000000);
655 (void)lapic_gettick(); 655 (void)lapic_gettick();
656 656
657 if (secondpass && cpu_hascounter()) { 657 if (secondpass && cpu_hascounter()) {
658 /* 658 /*
659 * Second pass calibration, using the TSC which has ideally 659 * Second pass calibration, using the TSC which has ideally
660 * been calibrated using the HPET or information gleaned 660 * been calibrated using the HPET or information gleaned
661 * from MSRs by this point. 661 * from MSRs by this point.
662 */ 662 */
663 uint64_t l0, l1, t0, t1; 663 uint64_t l0, l1, t0, t1;
664 664
665 (void)cpu_counter(); 665 (void)cpu_counter();
666 t0 = cpu_counter(); 666 t0 = cpu_counter();
667 l0 = lapic_gettick(); 667 l0 = lapic_gettick();
668 t0 += cpu_counter(); 668 t0 += cpu_counter();
669 DELAY(50000); 669 DELAY(50000);
670 t1 = cpu_counter(); 670 t1 = cpu_counter();
671 l1 = lapic_gettick(); 671 l1 = lapic_gettick();
672 t1 += cpu_counter(); 672 t1 += cpu_counter();
673 673
674 tmp = (l0 - l1) * cpu_frequency(ci) / ((t1 - t0 + 1) / 2); 674 tmp = (l0 - l1) * cpu_frequency(ci) / ((t1 - t0 + 1) / 2);
675 lapic_per_second = rounddown(tmp + 500, 1000); 675 lapic_per_second = rounddown(tmp + 500, 1000);
676 } else if (lapic_per_second == 0) { 676 } else if (lapic_per_second == 0) {
677 /* 677 /*
678 * Inaccurate first pass calibration using the i8254. 678 * Inaccurate first pass calibration using the i8254.
679 */ 679 */
680 unsigned int seen, delta, initial_i8254, initial_lapic; 680 unsigned int seen, delta, initial_i8254, initial_lapic;
681 unsigned int cur_i8254, cur_lapic; 681 unsigned int cur_i8254, cur_lapic;
682 682
683 (void)gettick(); 683 (void)gettick();
684 initial_lapic = lapic_gettick(); 684 initial_lapic = lapic_gettick();
685 initial_i8254 = gettick(); 685 initial_i8254 = gettick();
686 for (seen = 0; seen < TIMER_FREQ / 100; seen += delta) { 686 for (seen = 0; seen < TIMER_FREQ / 100; seen += delta) {
687 cur_i8254 = gettick(); 687 cur_i8254 = gettick();
688 if (cur_i8254 > initial_i8254) 688 if (cur_i8254 > initial_i8254)
689 delta = x86_rtclock_tval - (cur_i8254 - initial_i8254); 689 delta = x86_rtclock_tval - (cur_i8254 - initial_i8254);
690 else 690 else
691 delta = initial_i8254 - cur_i8254; 691 delta = initial_i8254 - cur_i8254;
692 initial_i8254 = cur_i8254; 692 initial_i8254 = cur_i8254;
693 } 693 }
694 cur_lapic = lapic_gettick(); 694 cur_lapic = lapic_gettick();
695 tmp = initial_lapic - cur_lapic; 695 tmp = initial_lapic - cur_lapic;
696 lapic_per_second = (tmp * TIMER_FREQ + seen / 2) / seen; 696 lapic_per_second = (tmp * TIMER_FREQ + seen / 2) / seen;
697 } 697 }
698 x86_enable_intr(); 698 x86_enable_intr();
699 699
700 humanize_number(tbuf, sizeof(tbuf), lapic_per_second, "Hz", 1000); 700 humanize_number(tbuf, sizeof(tbuf), lapic_per_second, "Hz", 1000);
701 aprint_debug_dev(ci->ci_dev, "apic clock running at %s\n", tbuf); 701 aprint_debug_dev(ci->ci_dev, "apic clock running at %s\n", tbuf);
702 702
703 if (lapic_per_second != 0) { 703 if (lapic_per_second != 0) {
704 /* 704 /*
705 * reprogram the apic timer to run in periodic mode. 705 * reprogram the apic timer to run in periodic mode.
706 * XXX need to program timer on other CPUs, too. 706 * XXX need to program timer on other CPUs, too.
707 */ 707 */
708 lapic_tval = (lapic_per_second * 2) / hz; 708 lapic_tval = (lapic_per_second * 2) / hz;
709 lapic_tval = (lapic_tval / 2) + (lapic_tval & 0x1); 709 lapic_tval = (lapic_tval / 2) + (lapic_tval & 0x1);
710 710
711 lapic_writereg(LAPIC_LVT_TIMER, LAPIC_LVT_TMM_PERIODIC 711 lapic_writereg(LAPIC_LVT_TIMER, LAPIC_LVT_TMM_PERIODIC
712 | LAPIC_LVT_MASKED | LAPIC_TIMER_VECTOR); 712 | LAPIC_LVT_MASKED | LAPIC_TIMER_VECTOR);
713 lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1); 713 lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1);
714 lapic_writereg(LAPIC_ICR_TIMER, lapic_tval); 714 lapic_writereg(LAPIC_ICR_TIMER, lapic_tval);
715 715
716 /* 716 /*
717 * Compute fixed-point ratios between cycles and 717 * Compute fixed-point ratios between cycles and
718 * microseconds to avoid having to do any division 718 * microseconds to avoid having to do any division
719 * in lapic_delay. 719 * in lapic_delay.
720 */ 720 */
721 721
722 tmp = (1000000 * (uint64_t)1 << 32) / lapic_per_second; 722 tmp = (1000000 * (uint64_t)1 << 32) / lapic_per_second;
723 lapic_frac_usec_per_cycle = tmp; 723 lapic_frac_usec_per_cycle = tmp;
724 724
725 tmp = (lapic_per_second * (uint64_t)1 << 32) / 1000000; 725 tmp = (lapic_per_second * (uint64_t)1 << 32) / 1000000;
726 726
727 lapic_frac_cycle_per_usec = tmp; 727 lapic_frac_cycle_per_usec = tmp;
728 728
729 /* 729 /*
730 * Compute delay in cycles for likely short delays in usec. 730 * Compute delay in cycles for likely short delays in usec.
731 */ 731 */
732 for (i = 0; i < 26; i++) 732 for (i = 0; i < 26; i++)
733 lapic_delaytab[i] = (lapic_frac_cycle_per_usec * i) >> 733 lapic_delaytab[i] = (lapic_frac_cycle_per_usec * i) >>
734 32; 734 32;
735 735
736 /* 736 /*
737 * Apply workaround for broken periodic timer under KVM 737 * Apply workaround for broken periodic timer under KVM
738 */ 738 */
739 if (vm_guest == VM_GUEST_KVM) { 739 if (vm_guest == VM_GUEST_KVM) {
740 lapic_broken_periodic = true; 740 lapic_broken_periodic = true;
741 lapic_timecounter.tc_quality = -100; 741 lapic_timecounter.tc_quality = -100;
742 aprint_debug_dev(ci->ci_dev, 742 aprint_debug_dev(ci->ci_dev,
743 "applying KVM timer workaround\n"); 743 "applying KVM timer workaround\n");
744 } 744 }
745 745
746 /* 746 /*
747 * Now that the timer's calibrated, use the apic timer routines 747 * Now that the timer's calibrated, use the apic timer routines
748 * for all our timing needs.. 748 * for all our timing needs..
749 */ 749 */
750 if (!secondpass) { 750 if (!secondpass) {
751 delay_func = lapic_delay; 751 delay_func = lapic_delay;
752 x86_initclock_func = lapic_initclock; 752 x86_initclock_func = lapic_initclock;
753 initrtclock(0); 753 initrtclock(0);
754 } 754 }
755 } 755 }
756} 756}
757 757
758/* 758/*
759 * delay for N usec. 759 * delay for N usec.
760 */ 760 */
761 761
762static void 762static void
763lapic_delay(unsigned int usec) 763lapic_delay(unsigned int usec)
764{ 764{
765 int32_t xtick, otick; 765 int32_t xtick, otick;
766 int64_t deltat; 766 int64_t deltat;
767 767
768 /* XXX Bad to disable preemption, but it's tied to the cpu. */ 768 /* XXX Bad to disable preemption, but it's tied to the cpu. */
769 kpreempt_disable(); 769 kpreempt_disable();
770 otick = lapic_gettick(); 770 otick = lapic_gettick();
771 771
772 if (usec <= 0) { 772 if (usec <= 0) {
773 kpreempt_enable(); 773 kpreempt_enable();
774 return; 774 return;
775 } 775 }
776 776
777 if (usec <= 25) 777 if (usec <= 25)
778 deltat = lapic_delaytab[usec]; 778 deltat = lapic_delaytab[usec];
779 else 779 else
780 deltat = (lapic_frac_cycle_per_usec * usec) >> 32; 780 deltat = (lapic_frac_cycle_per_usec * usec) >> 32;
781 781
782 while (deltat > 0) { 782 while (deltat > 0) {
783 xtick = lapic_gettick(); 783 xtick = lapic_gettick();
784 if (lapic_broken_periodic && xtick == 0 && otick == 0) { 784 if (lapic_broken_periodic && xtick == 0 && otick == 0) {
785 lapic_reset(); 785 lapic_reset();
786 xtick = lapic_gettick(); 786 xtick = lapic_gettick();
787 if (xtick == 0) 787 if (xtick == 0)
788 panic("lapic timer stopped ticking"); 788 panic("lapic timer stopped ticking");
789 } 789 }
790 if (xtick > otick) 790 if (xtick > otick)
791 deltat -= lapic_tval - (xtick - otick); 791 deltat -= lapic_tval - (xtick - otick);
792 else 792 else
793 deltat -= otick - xtick; 793 deltat -= otick - xtick;
794 otick = xtick; 794 otick = xtick;
795 795
796 x86_pause(); 796 x86_pause();
797 } 797 }
798 kpreempt_enable(); 798 kpreempt_enable();
799} 799}
800 800
801/* 801/*
802 * XXX the following belong mostly or partly elsewhere.. 802 * XXX the following belong mostly or partly elsewhere..
803 */ 803 */
804 804
805static void 805static void
806i82489_icr_wait(void) 806i82489_icr_wait(void)
807{ 807{
808#ifdef DIAGNOSTIC 808#ifdef DIAGNOSTIC
809 unsigned j = 100000; 809 unsigned j = 100000;
810#endif /* DIAGNOSTIC */ 810#endif /* DIAGNOSTIC */
811 811
812 while ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0) { 812 while ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0) {
813 x86_pause(); 813 x86_pause();
814#ifdef DIAGNOSTIC 814#ifdef DIAGNOSTIC
815 j--; 815 j--;
816 if (j == 0) 816 if (j == 0)
817 panic("i82489_icr_wait: busy"); 817 panic("i82489_icr_wait: busy");
818#endif /* DIAGNOSTIC */ 818#endif /* DIAGNOSTIC */
819 } 819 }
820} 820}
821 821
822static int 822static int
823i82489_ipi_init(int target) 823i82489_ipi_init(int target)
824{ 824{
825 uint32_t esr; 825 uint32_t esr;
826 826
827 i82489_writereg(LAPIC_ESR, 0); 827 i82489_writereg(LAPIC_ESR, 0);
828 (void)i82489_readreg(LAPIC_ESR); 828 (void)i82489_readreg(LAPIC_ESR);
829 829
830 i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT); 830 i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT);
831 831
832 i82489_writereg(LAPIC_ICRLO, LAPIC_DLMODE_INIT | LAPIC_LEVEL_ASSERT); 832 i82489_writereg(LAPIC_ICRLO, LAPIC_DLMODE_INIT | LAPIC_LEVEL_ASSERT);
833 i82489_icr_wait(); 833 i82489_icr_wait();
834 delay_func(10000); 834 delay_func(10000);
835 i82489_writereg(LAPIC_ICRLO, 835 i82489_writereg(LAPIC_ICRLO,
836 LAPIC_DLMODE_INIT | LAPIC_TRIGMODE_LEVEL | LAPIC_LEVEL_DEASSERT); 836 LAPIC_DLMODE_INIT | LAPIC_TRIGMODE_LEVEL | LAPIC_LEVEL_DEASSERT);
837 i82489_icr_wait(); 837 i82489_icr_wait();
838 838
839 if ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0) 839 if ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0)
840 return EBUSY; 840 return EBUSY;
841 841
842 esr = i82489_readreg(LAPIC_ESR); 842 esr = i82489_readreg(LAPIC_ESR);
843 if (esr != 0) 843 if (esr != 0)
844 aprint_debug("%s: ESR %08x\n", __func__, esr); 844 aprint_debug("%s: ESR %08x\n", __func__, esr);
845 845
846 return 0; 846 return 0;
847} 847}
848 848
849static int 849static int
850i82489_ipi_startup(int target, int vec) 850i82489_ipi_startup(int target, int vec)
851{ 851{
852 uint32_t esr; 852 uint32_t esr;
853 853
854 i82489_writereg(LAPIC_ESR, 0); 854 i82489_writereg(LAPIC_ESR, 0);
855 (void)i82489_readreg(LAPIC_ESR); 855 (void)i82489_readreg(LAPIC_ESR);
856 856
857 i82489_icr_wait(); 857 i82489_icr_wait();
858 i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT); 858 i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT);
859 i82489_writereg(LAPIC_ICRLO, vec | LAPIC_DLMODE_STARTUP | 859 i82489_writereg(LAPIC_ICRLO, vec | LAPIC_DLMODE_STARTUP |
860 LAPIC_LEVEL_ASSERT); 860 LAPIC_LEVEL_ASSERT);
861 i82489_icr_wait(); 861 i82489_icr_wait();
862 862
863 if ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0) 863 if ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0)
864 return EBUSY; 864 return EBUSY;
865 865
866 esr = i82489_readreg(LAPIC_ESR); 866 esr = i82489_readreg(LAPIC_ESR);
867 if (esr != 0) 867 if (esr != 0)
868 aprint_debug("%s: ESR %08x\n", __func__, esr); 868 aprint_debug("%s: ESR %08x\n", __func__, esr);
869 869
870 return 0; 870 return 0;
871} 871}
872 872
873static int 873static int
874i82489_ipi(int vec, int target, int dl) 874i82489_ipi(int vec, int target, int dl)
875{ 875{
876 int result, s; 876 int result, s;
877 877
878 s = splhigh(); 878 s = splhigh();
879 879
880 i82489_icr_wait(); 880 i82489_icr_wait();
881 881
882 if ((target & LAPIC_DEST_MASK) == 0) 882 if ((target & LAPIC_DEST_MASK) == 0)
883 i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT); 883 i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT);
884 884
885 i82489_writereg(LAPIC_ICRLO, 885 i82489_writereg(LAPIC_ICRLO,
886 (target & LAPIC_DEST_MASK) | vec | dl | LAPIC_LEVEL_ASSERT); 886 (target & LAPIC_DEST_MASK) | vec | dl | LAPIC_LEVEL_ASSERT);
887 887
888#ifdef DIAGNOSTIC 888#ifdef DIAGNOSTIC
889 i82489_icr_wait(); 889 i82489_icr_wait();
890 result = (i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) ? EBUSY : 0; 890 result = (i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) ? EBUSY : 0;
891#else 891#else
892 /* Don't wait - if it doesn't go, we're in big trouble anyway. */ 892 /* Don't wait - if it doesn't go, we're in big trouble anyway. */
893 result = 0; 893 result = 0;
894#endif 894#endif
895 splx(s); 895 splx(s);
896 896
897 return result; 897 return result;
898} 898}
899 899
900static int 900static int
901x2apic_ipi_init(int target) 901x2apic_ipi_init(int target)
902{ 902{
903 903
904 x2apic_write_icr(target, LAPIC_DLMODE_INIT | LAPIC_LEVEL_ASSERT); 904 x2apic_write_icr(target, LAPIC_DLMODE_INIT | LAPIC_LEVEL_ASSERT);
905 905
906 delay_func(10000); 906 delay_func(10000);
907 907
908 x2apic_write_icr(0, 908 x2apic_write_icr(0,
909 LAPIC_DLMODE_INIT | LAPIC_TRIGMODE_LEVEL | LAPIC_LEVEL_DEASSERT); 909 LAPIC_DLMODE_INIT | LAPIC_TRIGMODE_LEVEL | LAPIC_LEVEL_DEASSERT);
910 910
911 return 0; 911 return 0;
912} 912}
913 913
914static int 914static int
915x2apic_ipi_startup(int target, int vec) 915x2apic_ipi_startup(int target, int vec)
916{ 916{
917 917
918 x2apic_write_icr(target, 918 x2apic_write_icr(target,
919 vec | LAPIC_DLMODE_STARTUP | LAPIC_LEVEL_ASSERT); 919 vec | LAPIC_DLMODE_STARTUP | LAPIC_LEVEL_ASSERT);
920 920
921 return 0; 921 return 0;
922} 922}
923 923
924static int 924static int
925x2apic_ipi(int vec, int target, int dl) 925x2apic_ipi(int vec, int target, int dl)
926{ 926{
927 uint32_t dest_id = 0; 927 uint32_t dest_id = 0;
928 928
929 if ((target & LAPIC_DEST_MASK) == 0) 929 if ((target & LAPIC_DEST_MASK) == 0)
930 dest_id = target; 930 dest_id = target;
931 931
932 x2apic_write_icr(dest_id, 932 x2apic_write_icr(dest_id,
933 (target & LAPIC_DEST_MASK) | vec | dl | LAPIC_LEVEL_ASSERT); 933 (target & LAPIC_DEST_MASK) | vec | dl | LAPIC_LEVEL_ASSERT);
934 934
935 return 0; 935 return 0;
936} 936}
937 937
938int 938int
939x86_ipi_init(int target) 939x86_ipi_init(int target)
940{ 940{
941 if (x2apic_mode) 941 if (x2apic_mode)
942 return x2apic_ipi_init(target); 942 return x2apic_ipi_init(target);
943 return i82489_ipi_init(target); 943 return i82489_ipi_init(target);
944} 944}
945 945
946int 946int
947x86_ipi_startup(int target, int vec) 947x86_ipi_startup(int target, int vec)
948{ 948{
949 if (x2apic_mode) 949 if (x2apic_mode)
950 return x2apic_ipi_startup(target, vec); 950 return x2apic_ipi_startup(target, vec);
951 return i82489_ipi_startup(target, vec); 951 return i82489_ipi_startup(target, vec);
952} 952}
953 953
954/* 954/*
955 * Using 'pin numbers' as: 955 * Using 'pin numbers' as:
956 * 0 - timer 956 * 0 - timer
957 * 1 - thermal 957 * 1 - thermal
958 * 2 - PCINT 958 * 2 - PCINT
959 * 3 - LVINT0 959 * 3 - LVINT0
960 * 4 - LVINT1 960 * 4 - LVINT1
961 * 5 - LVERR 961 * 5 - LVERR
962 */ 962 */
963 963
964static void 964static void
965lapic_hwmask(struct pic *pic, int pin) 965lapic_hwmask(struct pic *pic, int pin)
966{ 966{
967 int reg; 967 int reg;
968 uint32_t val; 968 uint32_t val;
969 969
970 reg = LAPIC_LVT_TIMER + (pin << 4); 970 reg = LAPIC_LVT_TIMER + (pin << 4);
971 val = lapic_readreg(reg); 971 val = lapic_readreg(reg);
972 val |= LAPIC_LVT_MASKED; 972 val |= LAPIC_LVT_MASKED;
973 lapic_writereg(reg, val); 973 lapic_writereg(reg, val);
974} 974}
975 975
976static void 976static void
977lapic_hwunmask(struct pic *pic, int pin) 977lapic_hwunmask(struct pic *pic, int pin)
978{ 978{
979 int reg; 979 int reg;
980 uint32_t val; 980 uint32_t val;
981 981
982 reg = LAPIC_LVT_TIMER + (pin << 4); 982 reg = LAPIC_LVT_TIMER + (pin << 4);
983 val = lapic_readreg(reg); 983 val = lapic_readreg(reg);
984 val &= ~LAPIC_LVT_MASKED; 984 val &= ~LAPIC_LVT_MASKED;
985 lapic_writereg(reg, val); 985 lapic_writereg(reg, val);
986} 986}
987 987
988static void 988static void
989lapic_setup(struct pic *pic, struct cpu_info *ci, 989lapic_setup(struct pic *pic, struct cpu_info *ci,
990 int pin, int idtvec, int type) 990 int pin, int idtvec, int type)
991{ 991{
992} 992}
993 993
994void 994void
995lapic_dump(void) 995lapic_dump(void)
996{ 996{
997 struct cpu_info *ci = curcpu(); 997 struct cpu_info *ci = curcpu();
998 998
999#define APIC_LVT_PRINT(ci, where, idx, lvtreg) \ 999#define APIC_LVT_PRINT(ci, where, idx, lvtreg) \
1000 apic_format_redir(device_xname(ci->ci_dev), where, (idx), \ 1000 apic_format_redir(device_xname(ci->ci_dev), where, (idx), \
1001 APIC_VECTYPE_LAPIC_LVT, 0, lapic_readreg(lvtreg)) 1001 APIC_VECTYPE_LAPIC_LVT, 0, lapic_readreg(lvtreg))
1002 1002
1003 APIC_LVT_PRINT(ci, "cmci", 0, LAPIC_LVT_CMCI); 1003 APIC_LVT_PRINT(ci, "cmci", 0, LAPIC_LVT_CMCI);
1004 APIC_LVT_PRINT(ci, "timer", 0, LAPIC_LVT_TIMER); 1004 APIC_LVT_PRINT(ci, "timer", 0, LAPIC_LVT_TIMER);
1005 APIC_LVT_PRINT(ci, "thermal", 0, LAPIC_LVT_THERM); 1005 APIC_LVT_PRINT(ci, "thermal", 0, LAPIC_LVT_THERM);
1006 APIC_LVT_PRINT(ci, "pcint", 0, LAPIC_LVT_PCINT); 1006 APIC_LVT_PRINT(ci, "pcint", 0, LAPIC_LVT_PCINT);
1007 APIC_LVT_PRINT(ci, "lint", 0, LAPIC_LVT_LINT0); 1007 APIC_LVT_PRINT(ci, "lint", 0, LAPIC_LVT_LINT0);
1008 APIC_LVT_PRINT(ci, "lint", 1, LAPIC_LVT_LINT1); 1008 APIC_LVT_PRINT(ci, "lint", 1, LAPIC_LVT_LINT1);
1009 APIC_LVT_PRINT(ci, "err", 0, LAPIC_LVT_ERR); 1009 APIC_LVT_PRINT(ci, "err", 0, LAPIC_LVT_ERR);
1010 1010
1011#undef APIC_LVT_PRINT 1011#undef APIC_LVT_PRINT
1012} 1012}
1013#else /* XENPV */ 1013#else /* XENPV */
1014void 1014void
1015lapic_boot_init(paddr_t lapic_base) 1015lapic_boot_init(paddr_t lapic_base)
1016{ 1016{
1017} 1017}
1018#endif /* XENPV */ 1018#endif /* XENPV */