Sat Oct 12 06:31:04 2019 UTC ()

Rewrite the FPU code on x86. This greatly simplifies the logic and removes
the dependency on IPL_HIGH. NVMM is updated accordingly. Posted on
port-amd64 a week ago.

Bump the kernel version to 9.99.16.


(maxv)

diff -r1.48 -r1.49 src/sys/arch/amd64/amd64/amd64_trap.S
diff -r1.76 -r1.77 src/sys/arch/amd64/amd64/genassym.cf
diff -r1.188 -r1.189 src/sys/arch/amd64/amd64/locore.S
diff -r1.336 -r1.337 src/sys/arch/amd64/amd64/machdep.c
diff -r1.40 -r1.41 src/sys/arch/amd64/amd64/spl.S
diff -r1.44 -r1.45 src/sys/arch/amd64/include/frameasm.h
diff -r1.29 -r1.30 src/sys/arch/amd64/include/pcb.h
diff -r1.22 -r1.23 src/sys/arch/amd64/include/proc.h
diff -r1.113 -r1.114 src/sys/arch/i386/i386/genassym.cf
diff -r1.19 -r1.20 src/sys/arch/i386/i386/i386_trap.S
diff -r1.171 -r1.172 src/sys/arch/i386/i386/locore.S
diff -r1.820 -r1.821 src/sys/arch/i386/i386/machdep.c
diff -r1.48 -r1.49 src/sys/arch/i386/i386/spl.S
diff -r1.28 -r1.29 src/sys/arch/i386/include/frameasm.h
diff -r1.58 -r1.59 src/sys/arch/i386/include/pcb.h
diff -r1.45 -r1.46 src/sys/arch/i386/include/proc.h
diff -r1.50 -r1.51 src/sys/arch/x86/acpi/acpi_wakeup.c
diff -r1.109 -r1.110 src/sys/arch/x86/include/cpu.h
diff -r1.18 -r1.19 src/sys/arch/x86/include/fpu.h
diff -r1.172 -r1.173 src/sys/arch/x86/x86/cpu.c
diff -r1.57 -r1.58 src/sys/arch/x86/x86/fpu.c
diff -r1.27 -r1.28 src/sys/arch/x86/x86/ipi.c
diff -r1.37 -r1.38 src/sys/arch/x86/x86/vm_machdep.c
diff -r1.129 -r1.130 src/sys/arch/xen/x86/cpu.c
diff -r1.32 -r1.33 src/sys/arch/xen/x86/xen_ipi.c
diff -r1.49 -r1.50 src/sys/dev/nvmm/x86/nvmm_x86_svm.c
diff -r1.38 -r1.39 src/sys/dev/nvmm/x86/nvmm_x86_vmx.c
diff -r1.616 -r1.617 src/sys/sys/param.h

--- src/sys/arch/amd64/amd64/amd64_trap.S 2019/05/18 13:32:12	1.48
+++ src/sys/arch/amd64/amd64/amd64_trap.S 2019/10/12 06:31:03	1.49

 @@ -1,14 +1,14 @@
-/*	$NetBSD: amd64_trap.S,v 1.48 2019/05/18 13:32:12 maxv Exp $	*/
+/*	$NetBSD: amd64_trap.S,v 1.49 2019/10/12 06:31:03 maxv Exp $	*/
 /*
  * Copyright (c) 1998, 2007, 2008, 2017 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Charles M. Hannum, by Andrew Doran and by Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -662,26 +662,27 @@ calltrap:
 	/* Check for ASTs on exit to user mode. */
 	CLI(si)
 	CHECK_ASTPENDING(%r14)
 	je	3f
 	CLEAR_ASTPENDING(%r14)
 	STI(si)
 	movl	$T_ASTFLT,TF_TRAPNO(%rsp)
 	movq	%rsp,%rdi
 	incq	CPUVAR(NTRAP)
 	call	_C_LABEL(trap)
 	jmp	.Lalltraps_checkast	/* re-check ASTs */
 :	CHECK_DEFERRED_SWITCH
 	jnz	9f
 	HANDLE_DEFERRED_FPU
 :
 #ifdef DIAGNOSTIC
 	cmpl	CPUVAR(ILEVEL),%ebx
 	jne	.Lspl_error
 #endif
 	INTRFASTEXIT
 :	STI(si)
 	call	_C_LABEL(do_pmap_load)
 	jmp	.Lalltraps_checkast	/* re-check ASTs */
 #ifdef DIAGNOSTIC

--- src/sys/arch/amd64/amd64/genassym.cf 2019/05/29 16:54:41	1.76
+++ src/sys/arch/amd64/amd64/genassym.cf 2019/10/12 06:31:03	1.77

 @@ -1,14 +1,14 @@
-#	$NetBSD: genassym.cf,v 1.76 2019/05/29 16:54:41 maxv Exp $
+#	$NetBSD: genassym.cf,v 1.77 2019/10/12 06:31:03 maxv Exp $
+#
 # Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
 # All rights reserved.
+#
 # This code is derived from software contributed to The NetBSD Foundation
 # by Charles M. Hannum, and by Andrew Doran.
+#
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1. Redistributions of source code must retain the above copyright
 #    notice, this list of conditions and the following disclaimer.
 @@ -154,50 +154,52 @@ define	L_CTXSWTCH		offsetof(struct lwp,
 define	L_NCSW			offsetof(struct lwp, l_ncsw)
 define	L_NOPREEMPT		offsetof(struct lwp, l_nopreempt)
 define	L_DOPREEMPT		offsetof(struct lwp, l_dopreempt)
 define	L_CPU			offsetof(struct lwp, l_cpu)
 define	L_KPRIORITY		offsetof(struct lwp, l_kpriority)
 define	L_MD_REGS		offsetof(struct lwp, l_md.md_regs)
 define	L_MD_FLAGS		offsetof(struct lwp, l_md.md_flags)
 define	L_MD_ASTPENDING		offsetof(struct lwp, l_md.md_astpending)
 define	PAGE_SIZE		PAGE_SIZE
 define	MDL_IRET		MDL_IRET
 define	MDL_COMPAT32		MDL_COMPAT32
 define	MDL_FPU_IN_CPU		MDL_FPU_IN_CPU
 define	P_FLAG			offsetof(struct proc, p_flag)
 define	P_RASLIST		offsetof(struct proc, p_raslist)
 define	P_MD_SYSCALL		offsetof(struct proc, p_md.md_syscall)
 define	LW_SYSTEM		LW_SYSTEM
 define	M_DATA			offsetof(struct mbuf, m_data)
 define	M_LEN			offsetof(struct mbuf, m_len)
 define	M_NEXT			offsetof(struct mbuf, m_next)
 define	IP_SRC			offsetof(struct ip, ip_src)
 define	IP_DST			offsetof(struct ip, ip_dst)
 define	PCB_CR3			offsetof(struct pcb, pcb_cr3)
 define	PCB_RBP			offsetof(struct pcb, pcb_rbp)
 define	PCB_RSP			offsetof(struct pcb, pcb_rsp)
 define	PCB_RSP0		offsetof(struct pcb, pcb_rsp0)
 define	PCB_CR0			offsetof(struct pcb, pcb_cr0)
 define	PCB_ONFAULT		offsetof(struct pcb, pcb_onfault)
 define	PCB_FLAGS		offsetof(struct pcb, pcb_flags)
 define	PCB_COMPAT32		PCB_COMPAT32
 define	PCB_FS			offsetof(struct pcb, pcb_fs)
 define	PCB_GS			offsetof(struct pcb, pcb_gs)
 define	PCB_SAVEFPU		offsetof(struct pcb, pcb_savefpu)
 define	TF_RDI			offsetof(struct trapframe, tf_rdi)
 define	TF_RSI			offsetof(struct trapframe, tf_rsi)
 define	TF_RDX			offsetof(struct trapframe, tf_rdx)
 define	TF_RCX			offsetof(struct trapframe, tf_rcx)
 define	TF_R8			offsetof(struct trapframe, tf_r8)
 define	TF_R9			offsetof(struct trapframe, tf_r9)
 define	TF_R10			offsetof(struct trapframe, tf_r10)
 define	TF_R11			offsetof(struct trapframe, tf_r11)
 define	TF_R12			offsetof(struct trapframe, tf_r12)
 define	TF_R13			offsetof(struct trapframe, tf_r13)
 define	TF_R14			offsetof(struct trapframe, tf_r14)
 define	TF_R15			offsetof(struct trapframe, tf_r15)
 @@ -234,27 +236,26 @@ define	CPU_INFO_CURLDT		offsetof(struct
 define	CPU_INFO_IDLELWP	offsetof(struct cpu_info, ci_data.cpu_idlelwp)
 define	CPU_INFO_PMAP		offsetof(struct cpu_info, ci_pmap)
 define	CPU_INFO_TSS		offsetof(struct cpu_info, ci_tss)
 ifdef SVS
 define	CPU_INFO_UPDIRPA	offsetof(struct cpu_info, ci_svs_updirpa)
 define	CPU_INFO_RSP0		offsetof(struct cpu_info, ci_svs_rsp0)
 define	CPU_INFO_URSP0		offsetof(struct cpu_info, ci_svs_ursp0)
 define	CPU_INFO_KRSP0		offsetof(struct cpu_info, ci_svs_krsp0)
 endif
 define	CPU_INFO_NSYSCALL	offsetof(struct cpu_info, ci_data.cpu_nsyscall)
 define	CPU_INFO_NTRAP		offsetof(struct cpu_info, ci_data.cpu_ntrap)
 define	CPU_INFO_NINTR		offsetof(struct cpu_info, ci_data.cpu_nintr)
 define	CPU_INFO_CURPRIORITY	offsetof(struct cpu_info, ci_schedstate.spc_curpriority)
 define	CPU_INFO_FPCURLWP	offsetof(struct cpu_info, ci_fpcurlwp)
 define	CPU_INFO_GDT		offsetof(struct cpu_info, ci_gdt)
 define	CPU_INFO_ILEVEL		offsetof(struct cpu_info, ci_ilevel)
 define	CPU_INFO_IDEPTH		offsetof(struct cpu_info, ci_idepth)
 if !defined(XENPV)
 define	CPU_INFO_IPENDING	offsetof(struct cpu_info, ci_ipending)
 define	CPU_INFO_IMASK		offsetof(struct cpu_info, ci_imask)
 define	CPU_INFO_IUNMASK	offsetof(struct cpu_info, ci_iunmask)
 define	CPU_INFO_ISOURCES	offsetof(struct cpu_info, ci_isources)
 endif
 define	CPU_INFO_MTX_COUNT	offsetof(struct cpu_info, ci_mtx_count)
 define	CPU_INFO_MTX_OLDSPL	offsetof(struct cpu_info, ci_mtx_oldspl)
 define  CPU_INFO_CPUID		offsetof(struct cpu_info, ci_cpuid)

--- src/sys/arch/amd64/amd64/locore.S 2019/10/04 11:47:07	1.188
+++ src/sys/arch/amd64/amd64/locore.S 2019/10/12 06:31:03	1.189

 @@ -1,14 +1,14 @@
-/*	$NetBSD: locore.S,v 1.188 2019/10/04 11:47:07 maxv Exp $	*/
+/*	$NetBSD: locore.S,v 1.189 2019/10/12 06:31:03 maxv Exp $	*/
 /*
  * Copyright-o-rama!
  */
 /*
  * Copyright (c) 1998, 2000, 2007, 2008, 2016 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Charles M. Hannum and by Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
 @@ -1160,49 +1160,30 @@ ENTRY(cpu_switchto)
 	movq	L_PROC(%r12),%rdi
 	cmpq	$0,P_RASLIST(%rdi)
 	je	.Lno_RAS
 	/* Handle restartable atomic sequences (RAS). */
 	movq	L_MD_REGS(%r12),%rbx
 	movq	TF_RIP(%rbx),%rsi
 	call	_C_LABEL(ras_lookup)
 	cmpq	$-1,%rax
 	je	.Lno_RAS
 	movq	%rax,TF_RIP(%rbx)
 .Lno_RAS:
 	/*
 	 * Restore cr0 including FPU state (may have CR0_TS set).  Note that
 	 * IPL_SCHED prevents from FPU interrupt altering the LWP's saved cr0.
 	 */
 #ifndef XENPV
 	/* Raise the IPL to IPL_HIGH. Dropping the priority is deferred until
 	 * mi_switch(), when cpu_switchto() returns. XXX Still needed? */
 	movl	$IPL_HIGH,CPUVAR(ILEVEL)
 	movl	PCB_CR0(%r14),%ecx	/* has CR0_TS clear */
 	movq	%cr0,%rdx
 	/*
 	 * If our floating point registers are on a different CPU,
 	 * set CR0_TS so we'll trap rather than reuse bogus state.
 	 */
 	cmpq	CPUVAR(FPCURLWP),%r12
 	je	.Lskip_TS
 	orq	$CR0_TS,%rcx
 .Lskip_TS:
 	/* Reloading CR0 is very expensive - avoid if possible. */
 	cmpq	%rdx,%rcx
 	je	.Lskip_CR0
 	movq	%rcx,%cr0
 .Lskip_CR0:
 	/* The 32bit LWPs are handled differently. */
 	testl	$PCB_COMPAT32,PCB_FLAGS(%r14)
 	jnz	.Llwp_32bit
 .Llwp_64bit:
 	/* Set default 64bit values in %ds, %es, %fs and %gs. */
 	movq	$GSEL(GUDATA_SEL, SEL_UPL),%rax
 	movw	%ax,%ds
 	movw	%ax,%es
 	xorq	%rax,%rax
 	movw	%ax,%fs
 	CLI(cx)
 @@ -1295,26 +1276,28 @@ ENTRY(handle_syscall)
 	 * registers loaded.
 	 */
 	CLI(si)
 	/* Check for ASTs on exit to user mode. */
 	movl	L_MD_ASTPENDING(%r14),%eax
 	orl	CPUVAR(WANT_PMAPLOAD),%eax
 	jnz	9f
 #ifdef DIAGNOSTIC
 	cmpl	$IPL_NONE,CPUVAR(ILEVEL)
 	jne	.Lspl_error
 #endif
 	HANDLE_DEFERRED_FPU
 	/*
 	 * Decide if we need to take a slow path. That's the case when we
 	 * want to reload %cs and %ss on a 64bit LWP (MDL_IRET set), or when
 	 * we're returning to a 32bit LWP (MDL_COMPAT32 set).
+	 *
 	 * In either case, we jump into intrfastexit and return to userland
 	 * with the iret instruction.
 	 */
 	testl	$(MDL_IRET|MDL_COMPAT32),L_MD_FLAGS(%r14)
 	jnz	intrfastexit
 	jmp	syscall_sysret

--- src/sys/arch/amd64/amd64/machdep.c 2019/08/21 20:30:36	1.336
+++ src/sys/arch/amd64/amd64/machdep.c 2019/10/12 06:31:03	1.337

 @@ -1,14 +1,14 @@
-/*	$NetBSD: machdep.c,v 1.336 2019/08/21 20:30:36 skrll Exp $	*/
+/*	$NetBSD: machdep.c,v 1.337 2019/10/12 06:31:03 maxv Exp $	*/
 /*
  * Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
  *     The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Charles M. Hannum and by Jason R. Thorpe of the Numerical Aerospace
  * Simulation Facility, NASA Ames Research Center.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Coyote Point Systems, Inc. which was written under contract to Coyote
  * Point by Jed Davis and Devon O'Dell.
 @@ -100,27 +100,27 @@
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  *	@(#)machdep.c	7.4 (Berkeley) 6/3/91
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.336 2019/08/21 20:30:36 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.337 2019/10/12 06:31:03 maxv Exp $");
 #include "opt_modular.h"
 #include "opt_user_ldt.h"
 #include "opt_ddb.h"
 #include "opt_kgdb.h"
 #include "opt_cpureset_delay.h"
 #include "opt_mtrr.h"
 #include "opt_realmem.h"
 #include "opt_xen.h"
 #include "opt_svs.h"
 #include "opt_kaslr.h"
 #include "opt_kasan.h"
 #ifndef XENPV
 @@ -432,38 +432,29 @@ x86_64_switch_context(struct pcb *new)
 	physop.u.set_iopl.iopl = new->pcb_iopl;
 	HYPERVISOR_physdev_op(&physop);
+}
 void
 x86_64_tls_switch(struct lwp *l)
+{
 	struct cpu_info *ci = curcpu();
 	struct pcb *pcb = lwp_getpcb(l);
 	struct trapframe *tf = l->l_md.md_regs;
 	uint64_t zero = 0;
 	/*
-	 * Raise the IPL to IPL_HIGH.
+	 * Raise the IPL to IPL_HIGH. XXX Still needed?
 	 * FPU IPIs can alter the LWP's saved cr0.  Dropping the priority
 	 * is deferred until mi_switch(), when cpu_switchto() returns.
 	 */
 	(void)splhigh();
 	/*
 	 * If our floating point registers are on a different CPU,
 	 * set CR0_TS so we'll trap rather than reuse bogus state.
 	 */
 	if (l != ci->ci_fpcurlwp) {
 		HYPERVISOR_fpu_taskswitch(1);
 	/* Update segment registers */
 	if (pcb->pcb_flags & PCB_COMPAT32) {
 		update_descriptor(&ci->ci_gdt[GUFS_SEL], &pcb->pcb_fs);
 		update_descriptor(&ci->ci_gdt[GUGS_SEL], &pcb->pcb_gs);
 		setds(GSEL(GUDATA32_SEL, SEL_UPL));
 		setes(GSEL(GUDATA32_SEL, SEL_UPL));
 		setfs(GSEL(GUDATA32_SEL, SEL_UPL));
 		HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, tf->tf_gs);
 	} else {
 		update_descriptor(&ci->ci_gdt[GUFS_SEL], &zero);
 		update_descriptor(&ci->ci_gdt[GUGS_SEL], &zero);
 		setds(GSEL(GUDATA_SEL, SEL_UPL));

--- src/sys/arch/amd64/amd64/spl.S 2019/02/14 08:18:25	1.40
+++ src/sys/arch/amd64/amd64/spl.S 2019/10/12 06:31:03	1.41

 @@ -1,14 +1,14 @@
-/*	$NetBSD: spl.S,v 1.40 2019/02/14 08:18:25 cherry Exp $	*/
+/*	$NetBSD: spl.S,v 1.41 2019/10/12 06:31:03 maxv Exp $	*/
 /*
  * Copyright (c) 2003 Wasabi Systems, Inc.
  * All rights reserved.
+ *
  * Written by Frank van der Linden for Wasabi Systems, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
 @@ -387,22 +387,23 @@ LABEL(doreti_checkast)
 	CHECK_ASTPENDING(%r14)
 	je	3f
 	CLEAR_ASTPENDING(%r14)
 	STI(si)
 	movl	$T_ASTFLT,TF_TRAPNO(%rsp)	/* XXX undo later.. */
 	/* Pushed T_ASTFLT into tf_trapno on entry. */
 	movq	%rsp,%rdi
 	call	_C_LABEL(trap)
 	CLI(si)
 	jmp	doreti_checkast
 :
 	CHECK_DEFERRED_SWITCH
 	jnz	9f
 	HANDLE_DEFERRED_FPU
 :
 	INTRFASTEXIT
 :
 	STI(si)
 	call	_C_LABEL(do_pmap_load)
 	CLI(si)
 	jmp	doreti_checkast		/* recheck ASTs */
 END(doreti_checkast)
 IDTVEC_END(doreti)

--- src/sys/arch/amd64/include/frameasm.h 2019/05/18 13:32:12	1.44
+++ src/sys/arch/amd64/include/frameasm.h 2019/10/12 06:31:03	1.45

 @@ -1,14 +1,14 @@
-/*	$NetBSD: frameasm.h,v 1.44 2019/05/18 13:32:12 maxv Exp $	*/
+/*	$NetBSD: frameasm.h,v 1.45 2019/10/12 06:31:03 maxv Exp $	*/
 #ifndef _AMD64_MACHINE_FRAMEASM_H
 #define _AMD64_MACHINE_FRAMEASM_H
 #ifdef _KERNEL_OPT
 #include "opt_xen.h"
 #include "opt_svs.h"
 #endif
 /*
  * Macros to define pushing/popping frames for interrupts, traps
  * and system calls. Currently all the same; will diverge later.
  */
 @@ -236,14 +236,28 @@
 	pushq	%r13			;
 #define INTR_RECURSE_ENTRY \
 	subq	$TF_REGSIZE,%rsp	; \
 	INTR_SAVE_GPRS			; \
 	cld
 #define	CHECK_DEFERRED_SWITCH \
 	cmpl	$0, CPUVAR(WANT_PMAPLOAD)
 #define CHECK_ASTPENDING(reg)	cmpl	$0, L_MD_ASTPENDING(reg)
 #define CLEAR_ASTPENDING(reg)	movl	$0, L_MD_ASTPENDING(reg)
 /*
  * If the FPU state is not in the CPU, restore it. Executed with interrupts
  * disabled.
+ *
  *     %r14 is curlwp, must not be modified
  *     %rbx must not be modified
  */
 #define HANDLE_DEFERRED_FPU	\
 	testl	$MDL_FPU_IN_CPU,L_MD_FLAGS(%r14)	; \
 	jnz	1f					; \
 	call	_C_LABEL(fpu_handle_deferred)		; \
 	orl	$MDL_FPU_IN_CPU,L_MD_FLAGS(%r14)	; \
 :
 #endif /* _AMD64_MACHINE_FRAMEASM_H */

--- src/sys/arch/amd64/include/pcb.h 2018/07/26 09:29:08	1.29
+++ src/sys/arch/amd64/include/pcb.h 2019/10/12 06:31:03	1.30

 @@ -1,14 +1,14 @@
-/*	$NetBSD: pcb.h,v 1.29 2018/07/26 09:29:08 maxv Exp $	*/
+/*	$NetBSD: pcb.h,v 1.30 2019/10/12 06:31:03 maxv Exp $	*/
 /*
  * Copyright (c) 1998 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Charles M. Hannum.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -86,26 +86,25 @@ struct pcb {
 	uint64_t pcb_cr2;		/* page fault address (CR2) */
 	uint64_t pcb_cr3;
 	uint64_t pcb_rsp;
 	uint64_t pcb_rbp;
 	void     *pcb_onfault;		/* copyin/out fault recovery */
 	uint64_t  pcb_fs;
 	uint64_t  pcb_gs;
 	struct dbreg *pcb_dbregs;
 	uint16_t pcb_fpu_dflt_cw;
 	int pcb_iopl;
 	uint32_t pcb_unused[8];		/* unused */
 	struct cpu_info *pcb_fpcpu;	/* cpu holding our fp state. */
 	union savefpu	pcb_savefpu __aligned(64); /* floating point state */
 	/* **** DO NOT ADD ANYTHING HERE **** */
 };
 __CTASSERT(sizeof(struct pcb) - sizeof (union savefpu) ==  128);
 #else	/*	__x86_64__	*/
 #include <i386/pcb.h>
 #endif	/*	__x86_64__	*/
 #endif /* _AMD64_PCB_H_ */

--- src/sys/arch/amd64/include/proc.h 2017/02/25 13:34:21	1.22
+++ src/sys/arch/amd64/include/proc.h 2019/10/12 06:31:03	1.23

 @@ -1,14 +1,14 @@
-/*	$NetBSD: proc.h,v 1.22 2017/02/25 13:34:21 kamil Exp $	*/
+/*	$NetBSD: proc.h,v 1.23 2019/10/12 06:31:03 maxv Exp $	*/
 /*
  * Copyright (c) 1991 Regents of the University of California.
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -45,26 +45,27 @@
 struct pmap;
 struct vm_page;
 struct mdlwp {
 	struct	trapframe *md_regs;	/* registers on current frame */
 	struct pmap *md_gc_pmap;	/* pmap being garbage collected */
 	struct vm_page *md_gc_ptp;	/* pages from pmap g/c */
 	int	md_flags;		/* machine-dependent flags */
 	volatile int md_astpending;
 };
 #define	MDL_COMPAT32		0x0008	/* i386, always return via iret */
 #define	MDL_IRET		0x0010	/* force return via iret, not sysret */
 #define	MDL_FPU_IN_CPU		0x0020	/* the FPU state is in the CPU */
 struct mdproc {
 	int	md_flags;
 					/* Syscall handling function */
 	void	(*md_syscall)(struct trapframe *);
 };
 /* md_flags */
 #define MDP_USEDMTRR	0x0008	/* has set volatile MTRRs */
 #else	/*	__x86_64__	*/
 #include <i386/proc.h>

--- src/sys/arch/i386/i386/genassym.cf 2019/03/09 08:42:25	1.113
+++ src/sys/arch/i386/i386/genassym.cf 2019/10/12 06:31:03	1.114

 @@ -1,14 +1,14 @@
-#	$NetBSD: genassym.cf,v 1.113 2019/03/09 08:42:25 maxv Exp $
+#	$NetBSD: genassym.cf,v 1.114 2019/10/12 06:31:03 maxv Exp $
+#
 # Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
 # All rights reserved.
+#
 # This code is derived from software contributed to The NetBSD Foundation
 # by Charles M. Hannum, and by Andrew Doran.
+#
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1. Redistributions of source code must retain the above copyright
 #    notice, this list of conditions and the following disclaimer.
 @@ -155,61 +155,65 @@ define	PDIR_SLOT_PTE		PDIR_SLOT_PTE
 define	PTE_BASE		PTE_BASE
 define	PDP_SIZE		PDP_SIZE
 define	PDE_SIZE		sizeof(pd_entry_t)
 define	IOMAPSIZE		IOMAPSIZE
 define	VM_MAXUSER_ADDRESS	(int)VM_MAXUSER_ADDRESS
 define	L_PCB			offsetof(struct lwp, l_addr)
 define	L_FLAG			offsetof(struct lwp, l_flag)
 define	L_PROC			offsetof(struct lwp, l_proc)
 define	L_MD_REGS		offsetof(struct lwp, l_md.md_regs)
 define	L_MD_FLAGS		offsetof(struct lwp, l_md.md_flags)
 define	L_CTXSWTCH		offsetof(struct lwp, l_ctxswtch)
 define	L_MD_ASTPENDING		offsetof(struct lwp, l_md.md_astpending)
 define	L_CPU			offsetof(struct lwp, l_cpu)
 define	L_NCSW			offsetof(struct lwp, l_ncsw)
 define	L_NOPREEMPT		offsetof(struct lwp, l_nopreempt)
 define	L_DOPREEMPT		offsetof(struct lwp, l_dopreempt)
 define	L_KPRIORITY		offsetof(struct lwp, l_kpriority)
 define	P_FLAG			offsetof(struct proc, p_flag)
 define	P_RASLIST		offsetof(struct proc, p_raslist)
 define	P_MD_SYSCALL		offsetof(struct proc, p_md.md_syscall)
 define	MDL_FPU_IN_CPU		MDL_FPU_IN_CPU
 define	LW_SYSTEM		LW_SYSTEM
 define	GUFS_SEL		GUFS_SEL
 define	GUGS_SEL		GUGS_SEL
 define	M_DATA			offsetof(struct mbuf, m_data)
 define	M_LEN			offsetof(struct mbuf, m_len)
 define	M_NEXT			offsetof(struct mbuf, m_next)
 define	IP_SRC			offsetof(struct ip, ip_src)
 define	IP_DST			offsetof(struct ip, ip_dst)
 define	IP6_SRC			offsetof(struct ip6_hdr, ip6_src)
 define	IP6_DST			offsetof(struct ip6_hdr, ip6_dst)
 define	PCB_CR3			offsetof(struct pcb, pcb_cr3)
 define	PCB_EBP			offsetof(struct pcb, pcb_ebp)
 define	PCB_ESP			offsetof(struct pcb, pcb_esp)
 define	PCB_ESP0		offsetof(struct pcb, pcb_esp0)
 define	PCB_CR0			offsetof(struct pcb, pcb_cr0)
 define	PCB_ONFAULT		offsetof(struct pcb, pcb_onfault)
 define	PCB_FSD			offsetof(struct pcb, pcb_fsd)
 define	PCB_GSD			offsetof(struct pcb, pcb_gsd)
 define	PCB_IOMAP		offsetof(struct pcb, pcb_iomap)
 define	PCB_SAVEFPU		offsetof(struct pcb, pcb_savefpu)
 define	TF_CS			offsetof(struct trapframe, tf_cs)
 define	TF_EIP			offsetof(struct trapframe, tf_eip)
 define	TF_ERR			offsetof(struct trapframe, tf_err)
 define	TF_TRAPNO		offsetof(struct trapframe, tf_trapno)
 define	TF_EFLAGS		offsetof(struct trapframe, tf_eflags)
 define	TF_GS			offsetof(struct trapframe, tf_gs)
 define	TF_FS			offsetof(struct trapframe, tf_fs)
 define	TF_ES			offsetof(struct trapframe, tf_es)
 define	TF_DS			offsetof(struct trapframe, tf_ds)
 define	TF_EDI			offsetof(struct trapframe, tf_edi)
 define	TF_ESI			offsetof(struct trapframe, tf_esi)
 @@ -241,27 +245,26 @@ define	IH_LEVEL		offsetof(struct intrhan
 define	IH_NEXT			offsetof(struct intrhand, ih_next)
 define	TSS_TSS			offsetof(struct cpu_tss, tss)
 define	TSS_ESP0		offsetof(struct cpu_tss, tss.tss_esp0)
 define	TSS_IOBASE		offsetof(struct cpu_tss, tss.tss_iobase)
 define	TSS_IOMAP		offsetof(struct cpu_tss, iomap)
 define	CPU_INFO_SELF		offsetof(struct cpu_info, ci_self)
 define	CPU_INFO_RESCHED	offsetof(struct cpu_info, ci_want_resched)
 define	CPU_INFO_WANT_PMAPLOAD	offsetof(struct cpu_info, ci_want_pmapload)
 define	CPU_INFO_TLBSTATE	offsetof(struct cpu_info, ci_tlbstate)
 define	TLBSTATE_VALID		TLBSTATE_VALID
 define	CPU_INFO_CURLWP		offsetof(struct cpu_info, ci_curlwp)
 define	CPU_INFO_FPCURLWP	offsetof(struct cpu_info, ci_fpcurlwp)
 define	CPU_INFO_CURLDT		offsetof(struct cpu_info, ci_curldt)
 define	CPU_INFO_IDLELWP	offsetof(struct cpu_info, ci_data.cpu_idlelwp)
 define	CPU_INFO_PMAP		offsetof(struct cpu_info, ci_pmap)
 define	CPU_INFO_TSS		offsetof(struct cpu_info, ci_tss)
 define	IOMAP_INVALOFF		IOMAP_INVALOFF
 define	IOMAP_VALIDOFF		IOMAP_VALIDOFF
 define	CPU_INFO_NSYSCALL	offsetof(struct cpu_info, ci_data.cpu_nsyscall)
 define	CPU_INFO_NTRAP		offsetof(struct cpu_info, ci_data.cpu_ntrap)
 define	CPU_INFO_NINTR		offsetof(struct cpu_info, ci_data.cpu_nintr)
 define	CPU_INFO_CURPRIORITY	offsetof(struct cpu_info, ci_schedstate.spc_curpriority)
 define	CPU_INFO_CC_SKEW	offsetof(struct cpu_info, ci_data.cpu_cc_skew)

--- src/sys/arch/i386/i386/i386_trap.S 2019/10/04 15:28:00	1.19
+++ src/sys/arch/i386/i386/i386_trap.S 2019/10/12 06:31:03	1.20

 @@ -1,14 +1,14 @@
-/*	$NetBSD: i386_trap.S,v 1.19 2019/10/04 15:28:00 maxv Exp $	*/
+/*	$NetBSD: i386_trap.S,v 1.20 2019/10/12 06:31:03 maxv Exp $	*/
 /*
  * Copyright 2002 (c) Wasabi Systems, Inc.
  * All rights reserved.
+ *
  * Written by Frank van der Linden for Wasabi Systems, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
 @@ -56,27 +56,27 @@
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #if 0
 #include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: i386_trap.S,v 1.19 2019/10/04 15:28:00 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: i386_trap.S,v 1.20 2019/10/12 06:31:03 maxv Exp $");
 #endif
 /*
  * Trap and fault vector routines
+ *
  * On exit from the kernel to user mode, we always need to check for ASTs.  In
  * addition, we need to do this atomically; otherwise an interrupt may occur
  * which causes an AST, but it won't get processed until the next kernel entry
  * (possibly the next clock tick).  Thus, we disable interrupt before checking,
  * and only enable them again on the final `iret' or before calling the AST
  * handler.
  */
 @@ -426,47 +426,50 @@ calltrap:
 	STI(%eax)
 	movl	$T_ASTFLT,TF_TRAPNO(%esp)
 	addl	$1,CPUVAR(NTRAP)	/* statistical info */
 	adcl	$0,CPUVAR(NTRAP)+4
 	pushl	%esp
 	call	_C_LABEL(trap)
 	addl	$4,%esp
 	jmp	.Lalltraps_checkast	/* re-check ASTs */
 :	CHECK_DEFERRED_SWITCH
 	jnz	9f
 #ifdef XEN
 	STIC(%eax)
-	jz	6f
+	jz	22f
 	call	_C_LABEL(stipending)
 	testl	%eax,%eax
-	jz	6f
+	jz	22f
 	/* process pending interrupts */
 	CLI(%eax)
 	movl	CPUVAR(ILEVEL),%ebx
 	movl	$.Lalltraps_resume,%esi /* address to resume loop at */
 .Lalltraps_resume:
 	movl	%ebx,%eax		/* get cpl */
 	movl	CPUVAR(XUNMASK)(,%eax,4),%eax
 	andl	CPUVAR(XPENDING),%eax	/* any non-masked bits left? */
-	jz	7f
+	jz	11f
 	bsrl	%eax,%eax
 	btrl	%eax,CPUVAR(XPENDING)
 	movl	CPUVAR(XSOURCES)(,%eax,4),%eax
 	jmp	*IS_RESUME(%eax)
 :	movl	%ebx,CPUVAR(ILEVEL)	/* restore cpl */
 	jmp	.Lalltraps_checkusr
 :
 #endif
 	HANDLE_DEFERRED_FPU
 :
 #ifdef DIAGNOSTIC
 	cmpl	CPUVAR(ILEVEL),%ebx
 	jne	.Lspl_error
 #endif
 	INTRFASTEXIT
 :	STI(%eax)
 	call	_C_LABEL(pmap_load)
 	jmp	.Lalltraps_checkast	/* re-check ASTs */
 #ifdef DIAGNOSTIC
 .Lspl_error:

--- src/sys/arch/i386/i386/locore.S 2019/10/04 15:28:00	1.171
+++ src/sys/arch/i386/i386/locore.S 2019/10/12 06:31:03	1.172

 @@ -1,14 +1,14 @@
-/*	$NetBSD: locore.S,v 1.171 2019/10/04 15:28:00 maxv Exp $	*/
+/*	$NetBSD: locore.S,v 1.172 2019/10/12 06:31:03 maxv Exp $	*/
 /*
  * Copyright-o-rama!
  */
 /*
  * Copyright (c) 1998, 2000, 2004, 2006, 2007, 2009, 2016
  * The NetBSD Foundation, Inc., All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Charles M. Hannum, by Andrew Doran and by Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
 @@ -118,27 +118,27 @@
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  *	@(#)locore.s	7.3 (Berkeley) 5/13/91
  */
 #include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: locore.S,v 1.171 2019/10/04 15:28:00 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: locore.S,v 1.172 2019/10/12 06:31:03 maxv Exp $");
 #include "opt_copy_symtab.h"
 #include "opt_ddb.h"
 #include "opt_modular.h"
 #include "opt_multiboot.h"
 #include "opt_realmem.h"
 #include "opt_xen.h"
 #include "assym.h"
 #include "lapic.h"
 #include "ioapic.h"
 #include "ksyms.h"
 @@ -1177,53 +1177,34 @@ skip_save:
 	/* Handle restartable atomic sequences (RAS). */
 	movl	L_MD_REGS(%edi),%ecx
 	pushl	TF_EIP(%ecx)
 	pushl	%eax
 	call	_C_LABEL(ras_lookup)
 	addl	$8,%esp
 	cmpl	$-1,%eax
 	je	no_RAS
 	movl	L_MD_REGS(%edi),%ecx
 	movl	%eax,TF_EIP(%ecx)
 no_RAS:
 	/*
 	 * Restore cr0 (including FPU state).  Raise the IPL to IPL_HIGH.
 	 * FPU IPIs can alter the LWP's saved cr0.  Dropping the priority
 	 * is deferred until mi_switch(), when cpu_switchto() returns.
 	 */
 #ifdef XENPV
 	pushl	%edi
 	call	_C_LABEL(i386_tls_switch)
 	addl	$4,%esp
 #else
 	/* Raise the IPL to IPL_HIGH. Dropping the priority is deferred until
 	 * mi_switch(), when cpu_switchto() returns. XXX Still needed? */
 	movl	$IPL_HIGH,CPUVAR(ILEVEL)
 	movl	PCB_CR0(%ebx),%ecx		/* has CR0_TS clear */
 	movl	%cr0,%edx
 	/*
 	 * If our floating point registers are on a different CPU,
 	 * set CR0_TS so we'll trap rather than reuse bogus state.
 	 */
 	cmpl	CPUVAR(FPCURLWP),%edi
 	je	skip_TS
 	orl	$CR0_TS,%ecx
 skip_TS:
 	/* Reloading CR0 is very expensive - avoid if possible. */
 	cmpl	%edx,%ecx
 	je	switch_return
 	movl	%ecx,%cr0
 #endif
 switch_return:
 	/* Return to the new LWP, returning 'oldlwp' in %eax. */
 	movl	%esi,%eax
 	popl	%edi
 	popl	%esi
 	popl	%ebx
 	ret
 .Lcopy_iobitmap:
 	/* Copy I/O bitmap. */
 	incl	_C_LABEL(pmap_iobmp_evcnt)+EV_COUNT
 @@ -1312,26 +1293,28 @@ IDTVEC(syscall)
 	btrl	%eax,CPUVAR(XPENDING)
 	movl	CPUVAR(XSOURCES)(,%eax,4),%eax
 	jmp	*IS_RESUME(%eax)
 :	movl	%ebx, CPUVAR(ILEVEL)	/* restore cpl  */
 	jmp	.Lsyscall_checkast
 :
 #endif /* XEN */
 #ifdef DIAGNOSTIC
 	cmpl	$IPL_NONE,CPUVAR(ILEVEL)
 	jne	3f
 #endif
 	HANDLE_DEFERRED_FPU
 	INTRFASTEXIT
 #ifdef DIAGNOSTIC
 :	STI(%eax)
 	pushl	$4f
 	call	_C_LABEL(panic)
 	addl	$4,%esp
 	pushl	$IPL_NONE
 	call	_C_LABEL(spllower)
 	addl	$4,%esp
 	jmp	.Lsyscall_checkast
 :	.asciz	"SPL NOT LOWERED ON SYSCALL EXIT\n"
 :	.asciz	"SPL NOT ZERO ON SYSCALL ENTRY\n"

--- src/sys/arch/i386/i386/machdep.c 2019/05/19 08:46:15	1.820
+++ src/sys/arch/i386/i386/machdep.c 2019/10/12 06:31:03	1.821

 @@ -1,14 +1,14 @@
-/*	$NetBSD: machdep.c,v 1.820 2019/05/19 08:46:15 maxv Exp $	*/
+/*	$NetBSD: machdep.c,v 1.821 2019/10/12 06:31:03 maxv Exp $	*/
 /*
  * Copyright (c) 1996, 1997, 1998, 2000, 2004, 2006, 2008, 2009, 2017
  *     The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Charles M. Hannum, by Jason R. Thorpe of the Numerical Aerospace
  * Simulation Facility NASA Ames Research Center, by Julio M. Merino Vidal,
  * by Andrew Doran, and by Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
 @@ -57,27 +57,27 @@
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  *	@(#)machdep.c	7.4 (Berkeley) 6/3/91
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.820 2019/05/19 08:46:15 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.821 2019/10/12 06:31:03 maxv Exp $");
 #include "opt_beep.h"
 #include "opt_compat_freebsd.h"
 #include "opt_compat_netbsd.h"
 #include "opt_cpureset_delay.h"
 #include "opt_ddb.h"
 #include "opt_kgdb.h"
 #include "opt_mtrr.h"
 #include "opt_modular.h"
 #include "opt_multiboot.h"
 #include "opt_multiprocessor.h"
 #include "opt_physmem.h"
 #include "opt_realmem.h"
 @@ -499,40 +499,30 @@ i386_switch_context(lwp_t *l)
 	physop.cmd = PHYSDEVOP_SET_IOPL;
 	physop.u.set_iopl.iopl = pcb->pcb_iopl;
 	HYPERVISOR_physdev_op(&physop);
+}
 void
 i386_tls_switch(lwp_t *l)
+{
 	struct cpu_info *ci = curcpu();
 	struct pcb *pcb = lwp_getpcb(l);
 	/*
-	 * Raise the IPL to IPL_HIGH.
+	 * Raise the IPL to IPL_HIGH. XXX Still needed?
 	 * FPU IPIs can alter the LWP's saved cr0.  Dropping the priority
 	 * is deferred until mi_switch(), when cpu_switchto() returns.
 	 */
 	(void)splhigh();
 	/*
 	 * If our floating point registers are on a different CPU,
 	 * set CR0_TS so we'll trap rather than reuse bogus state.
 	 */
 	if (l != ci->ci_fpcurlwp) {
 		HYPERVISOR_fpu_taskswitch(1);
 	/* Update TLS segment pointers */
 	update_descriptor(&ci->ci_gdt[GUFS_SEL],
 	    (union descriptor *)&pcb->pcb_fsd);
 	update_descriptor(&ci->ci_gdt[GUGS_SEL],
 	    (union descriptor *)&pcb->pcb_gsd);
+}
 #endif /* XENPV */
 /* XXX */
 #define IDTVEC(name)	__CONCAT(X, name)
 typedef void (vector)(void);
 #ifndef XENPV

--- src/sys/arch/i386/i386/spl.S 2019/10/04 15:28:00	1.48
+++ src/sys/arch/i386/i386/spl.S 2019/10/12 06:31:03	1.49

 @@ -1,14 +1,14 @@
-/*	$NetBSD: spl.S,v 1.48 2019/10/04 15:28:00 maxv Exp $	*/
+/*	$NetBSD: spl.S,v 1.49 2019/10/12 06:31:03 maxv Exp $	*/
 /*
  * Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Charles M. Hannum and Andrew Doran.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -20,27 +20,27 @@
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: spl.S,v 1.48 2019/10/04 15:28:00 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: spl.S,v 1.49 2019/10/12 06:31:03 maxv Exp $");
 #include "opt_ddb.h"
 #include "opt_spldebug.h"
 #include "opt_xen.h"
 #include <machine/trap.h>
 #include <machine/segments.h>
 #include <machine/frameasm.h>
 #include "assym.h"
 	.text
 @@ -316,26 +316,27 @@ LABEL(doreti_checkast)
 	STI(%eax)
 	movl	$T_ASTFLT,TF_TRAPNO(%esp)	/* XXX undo later.. */
 	/* Pushed T_ASTFLT into tf_trapno on entry. */
 	pushl	%esp
 	call	_C_LABEL(trap)
 	addl	$4,%esp
 	CLI(%eax)
 	jmp	5b
 END(doreti_checkast)
 :
 	CHECK_DEFERRED_SWITCH
 	jnz	9f
 	HANDLE_DEFERRED_FPU
 :
 #ifdef XEN
 	STIC(%eax)
 	jz	4f
 	call	_C_LABEL(stipending)
 	testl	%eax,%eax
 	jz	4f
 	CLI(%eax)
 	jmp	.Ldoreti_resume_stic
 :
 #endif
 	INTRFASTEXIT
 :

--- src/sys/arch/i386/include/frameasm.h 2019/02/14 08:18:25	1.28
+++ src/sys/arch/i386/include/frameasm.h 2019/10/12 06:31:03	1.29

 @@ -1,14 +1,14 @@
-/*	$NetBSD: frameasm.h,v 1.28 2019/02/14 08:18:25 cherry Exp $	*/
+/*	$NetBSD: frameasm.h,v 1.29 2019/10/12 06:31:03 maxv Exp $	*/
 #ifndef _I386_FRAMEASM_H_
 #define _I386_FRAMEASM_H_
 #ifdef _KERNEL_OPT
 #include "opt_multiprocessor.h"
 #include "opt_xen.h"
 #endif
 #ifdef XEN
 /* XXX assym.h */
 #define TRAP_INSTR	int	$0x82
 @@ -89,26 +89,42 @@
 #define INTR_RECURSE_HWFRAME \
 	pushfl				; \
 	pushl	%cs			; \
 	pushl	%esi			;
 #define	CHECK_DEFERRED_SWITCH \
 	cmpl	$0, CPUVAR(WANT_PMAPLOAD)
 #define	CHECK_ASTPENDING(reg)	movl	CPUVAR(CURLWP),reg	; \
 				cmpl	$0, L_MD_ASTPENDING(reg)
 #define	CLEAR_ASTPENDING(reg)	movl	$0, L_MD_ASTPENDING(reg)
 /*
  * If the FPU state is not in the CPU, restore it. Executed with interrupts
  * disabled.
+ *
  *     %ebx must not be modified
  */
 #define HANDLE_DEFERRED_FPU	\
 	movl	CPUVAR(CURLWP),%eax			; \
 	testl	$MDL_FPU_IN_CPU,L_MD_FLAGS(%eax)	; \
 	jnz	1f					; \
 	pushl	%eax					; \
 	call	_C_LABEL(fpu_handle_deferred)		; \
 	popl	%eax					; \
 	orl	$MDL_FPU_IN_CPU,L_MD_FLAGS(%eax)	; \
 :
 /*
  * IDEPTH_INCR:
  * increase ci_idepth and switch to the interrupt stack if necessary.
  * note that the initial value of ci_idepth is -1.
+ *
  * => should be called with interrupt disabled.
  * => save the old value of %esp in %eax.
  */
 #define	IDEPTH_INCR \
 	incl	CPUVAR(IDEPTH); \
 	movl	%esp, %eax; \
 	jne	999f; \
 	movl	CPUVAR(INTRSTACK), %esp; \

--- src/sys/arch/i386/include/pcb.h 2018/07/26 09:29:08	1.58
+++ src/sys/arch/i386/include/pcb.h 2019/10/12 06:31:03	1.59

 @@ -1,14 +1,14 @@
-/*	$NetBSD: pcb.h,v 1.58 2018/07/26 09:29:08 maxv Exp $	*/
+/*	$NetBSD: pcb.h,v 1.59 2019/10/12 06:31:03 maxv Exp $	*/
 /*
  * Copyright (c) 1998, 2009 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Charles M. Hannum, and by Andrew Doran.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -89,24 +89,23 @@ struct pcb {
 	struct segment_descriptor pcb_fsd;	/* %fs descriptor */
 	struct segment_descriptor pcb_gsd; 	/* %gs descriptor */
 	void 	*pcb_onfault;		/* copyin/out fault recovery */
 	char	*pcb_iomap;		/* I/O permission bitmap */
 	struct dbreg	*pcb_dbregs;	/* CPU Debug Registers */
 	uint16_t pcb_fpu_dflt_cw;
 #define	PCB_DBREGS	0x01
 	int	pcb_flags;
 	int	not_used[15];
 	/* floating point state */
 	struct cpu_info	*pcb_fpcpu;	/* cpu holding our fp state. */
 	union savefpu	pcb_savefpu __aligned(64);
 	/* **** DO NOT ADD ANYTHING HERE **** */
 };
 #ifndef __lint__
 /* This doesn't really matter, but there is a lot of implied padding */
 __CTASSERT(sizeof(struct pcb) - sizeof (union savefpu) == 128);
 #endif
 #endif /* _I386_PCB_H_ */

--- src/sys/arch/i386/include/proc.h 2017/02/25 13:34:21	1.45
+++ src/sys/arch/i386/include/proc.h 2019/10/12 06:31:03	1.46

 @@ -1,14 +1,14 @@
-/*	$NetBSD: proc.h,v 1.45 2017/02/25 13:34:21 kamil Exp $	*/
+/*	$NetBSD: proc.h,v 1.46 2019/10/12 06:31:03 maxv Exp $	*/
 /*
  * Copyright (c) 1991 Regents of the University of California.
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -33,26 +33,28 @@
 #ifndef _I386_PROC_H_
 #define _I386_PROC_H_
 #include <machine/frame.h>
 #include <machine/pcb.h>
 /*
  * Machine-dependent part of the lwp structure for i386.
  */
 struct pmap;
 struct vm_page;
 #define	MDL_FPU_IN_CPU		0x0020	/* the FPU state is in the CPU */
 struct mdlwp {
 	struct	trapframe *md_regs;	/* registers on current frame */
 	int	md_flags;		/* machine-dependent flags */
 	volatile int md_astpending;	/* AST pending for this process */
 	struct pmap *md_gc_pmap;	/* pmap being garbage collected */
 	struct vm_page *md_gc_ptp;	/* pages from pmap g/c */
 };
 /* md_flags */
 #define	MDL_IOPL		0x0002	/* XEN: i/o privilege */
 struct mdproc {
 	int	md_flags;

--- src/sys/arch/x86/acpi/acpi_wakeup.c 2019/06/17 16:34:02	1.50
+++ src/sys/arch/x86/acpi/acpi_wakeup.c 2019/10/12 06:31:03	1.51

 @@ -1,14 +1,14 @@
-/*	$NetBSD: acpi_wakeup.c,v 1.50 2019/06/17 16:34:02 jmcneill Exp $	*/
+/*	$NetBSD: acpi_wakeup.c,v 1.51 2019/10/12 06:31:03 maxv Exp $	*/
 /*-
  * Copyright (c) 2002, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Takuya SHIOZAKI.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -49,27 +49,27 @@
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  *      FreeBSD: src/sys/i386/acpica/acpi_wakeup.c,v 1.9 2002/01/10 03:26:46 wes Exp
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: acpi_wakeup.c,v 1.50 2019/06/17 16:34:02 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: acpi_wakeup.c,v 1.51 2019/10/12 06:31:03 maxv Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/bus.h>
 #include <sys/cpu.h>
 #include <sys/kcpuset.h>
 #include <sys/sysctl.h>
 #include <uvm/uvm_extern.h>
 #include <uvm/uvm_page.h>
 #ifdef __i386__
 @@ -239,27 +239,27 @@ acpi_md_sleep_enter(int state)
+}
 #ifdef MULTIPROCESSOR
 void
 acpi_cpu_sleep(struct cpu_info *ci)
+{
 	uint64_t xcr0 = 0;
 	int s;
 	KASSERT(!CPU_IS_PRIMARY(ci));
 	KASSERT(ci == curcpu());
 	s = splhigh();
-	fpusave_cpu(true);
+	fpu_save();
 	x86_disable_intr();
 	/*
 	 * XXX also need to save the PMCs, the dbregs, and probably a few
 	 * MSRs too.
 	 */
 	if (rcr4() & CR4_OSXSAVE)
 		xcr0 = rdxcr(0);
 	/* Go get some sleep */
 	if (acpi_md_sleep_prepare(-1))
 		goto out;
 @@ -303,27 +303,27 @@ acpi_md_sleep(int state)
 #endif
 	KASSERT(acpi_wakeup_paddr != 0);
 	KASSERT(sizeof(wakecode) <= PAGE_SIZE);
 	if (!CPU_IS_PRIMARY(curcpu())) {
 		printf("acpi0: WARNING: ignoring sleep from secondary CPU\n");
 		return -1;
+	}
 	AcpiSetFirmwareWakingVector(acpi_wakeup_paddr, 0);
 	s = splhigh();
-	fpusave_cpu(true);
+	fpu_save();
 	x86_disable_intr();
 #ifdef MULTIPROCESSOR
 	/* Save and suspend Application Processors. */
 	x86_broadcast_ipi(X86_IPI_ACPI_CPU_SLEEP);
 	cid = cpu_index(curcpu());
 	while (kcpuset_isotherset(kcpuset_running, cid)) {
 		delay(1);
+	}
 #endif
 	/*
 	 * XXX also need to save the PMCs, the dbregs, and probably a few

--- src/sys/arch/x86/include/cpu.h 2019/10/03 05:06:29	1.109
+++ src/sys/arch/x86/include/cpu.h 2019/10/12 06:31:03	1.110

 @@ -1,14 +1,14 @@
-/*	$NetBSD: cpu.h,v 1.109 2019/10/03 05:06:29 maxv Exp $	*/
+/*	$NetBSD: cpu.h,v 1.110 2019/10/12 06:31:03 maxv Exp $	*/
 /*
  * Copyright (c) 1990 The Regents of the University of California.
  * All rights reserved.
+ *
  * This code is derived from software contributed to Berkeley by
  * William Jolitz.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -109,27 +109,26 @@ struct clockframe {
 struct cpu_info {
 	struct cpu_data ci_data;	/* MI per-cpu data */
 	device_t ci_dev;		/* pointer to our device */
 	struct cpu_info *ci_self;	/* self-pointer */
 #ifdef XEN
 	volatile struct vcpu_info *ci_vcpu; /* for XEN */
 #endif
 	/*
 	 * Will be accessed by other CPUs.
 	 */
 	struct cpu_info *ci_next;	/* next cpu */
 	struct lwp *ci_curlwp;		/* current owner of the processor */
 	struct lwp *ci_fpcurlwp;	/* current owner of the FPU */
 	cpuid_t ci_cpuid;		/* our CPU ID */
 	uint32_t ci_acpiid;		/* our ACPI/MADT ID */
 	uint32_t ci_initapicid;		/* our initial APIC ID */
 	/*
 	 * Private members.
 	 */
 	struct pmap *ci_pmap;		/* current pmap */
 	int ci_want_pmapload;		/* pmap_load() is needed */
 	volatile int ci_tlbstate;	/* one of TLBSTATE_ states. see below */
 #define	TLBSTATE_VALID	0	/* all user tlbs are valid */
 #define	TLBSTATE_LAZY	1	/* tlbs are valid but won't be kept uptodate */
 #define	TLBSTATE_STALE	2	/* we might have stale user tlbs */

--- src/sys/arch/x86/include/fpu.h 2019/10/04 11:47:08	1.18
+++ src/sys/arch/x86/include/fpu.h 2019/10/12 06:31:03	1.19

 @@ -1,44 +1,44 @@
-/*	$NetBSD: fpu.h,v 1.18 2019/10/04 11:47:08 maxv Exp $	*/
+/*	$NetBSD: fpu.h,v 1.19 2019/10/12 06:31:03 maxv Exp $	*/
 #ifndef	_X86_FPU_H_
 #define	_X86_FPU_H_
 #include <x86/cpu_extended_state.h>
 #ifdef _KERNEL
 struct cpu_info;
 struct lwp;
 struct trapframe;
 void fpuinit(struct cpu_info *);
 void fpuinit_mxcsr_mask(void);
 void fpu_area_save(void *, uint64_t);
 void fpu_area_restore(void *, uint64_t);
-void fpusave_lwp(struct lwp *, bool);
+void fpu_save(void);
 void fpusave_cpu(bool);
 void fpu_set_default_cw(struct lwp *, unsigned int);
 void fputrap(struct trapframe *);
 void fpudna(struct trapframe *);
 void fpu_clear(struct lwp *, unsigned int);
 void fpu_sigreset(struct lwp *);
-void fpu_save_area_fork(struct pcb *, const struct pcb *);
+void fpu_lwp_fork(struct lwp *, struct lwp *);
 void fpu_lwp_abandon(struct lwp *l);
 void process_write_fpregs_xmm(struct lwp *, const struct fxsave *);
 void process_write_fpregs_s87(struct lwp *, const struct save87 *);
 void process_read_fpregs_xmm(struct lwp *, struct fxsave *);
 void process_read_fpregs_s87(struct lwp *, struct save87 *);
 int process_read_xstate(struct lwp *, struct xstate *);
 int process_verify_xstate(const struct xstate *);
 int process_write_xstate(struct lwp *, const struct xstate *);
 #endif

--- src/sys/arch/x86/x86/cpu.c 2019/08/30 07:53:47	1.172
+++ src/sys/arch/x86/x86/cpu.c 2019/10/12 06:31:04	1.173

 @@ -1,14 +1,14 @@
-/*	$NetBSD: cpu.c,v 1.172 2019/08/30 07:53:47 mrg Exp $	*/
+/*	$NetBSD: cpu.c,v 1.173 2019/10/12 06:31:04 maxv Exp $	*/
 /*
  * Copyright (c) 2000-2012 NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -52,27 +52,27 @@
  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.172 2019/08/30 07:53:47 mrg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.173 2019/10/12 06:31:04 maxv Exp $");
 #include "opt_ddb.h"
 #include "opt_mpbios.h"		/* for MPDEBUG */
 #include "opt_mtrr.h"
 #include "opt_multiprocessor.h"
 #include "opt_svs.h"
 #include "lapic.h"
 #include "ioapic.h"
 #include <sys/param.h>
 #include <sys/proc.h>
 #include <sys/systm.h>
 @@ -976,35 +976,34 @@ cpu_hatch(void *v)
  */
 void
 cpu_debug_dump(void)
+{
 	struct cpu_info *ci;
 	CPU_INFO_ITERATOR cii;
 	const char sixtyfour64space[] =
 #ifdef _LP64
 			   "        "
 #endif
 			   "";
 	db_printf("addr		%sdev	id	flags	ipis	curlwp 		"
-		  "fpcurlwp\n", sixtyfour64space);
+		  "\n", sixtyfour64space);
 	for (CPU_INFO_FOREACH(cii, ci)) {
-		db_printf("%p	%s	%ld	%x	%x	%10p	%10p\n",
+		db_printf("%p	%s	%ld	%x	%x	%10p\n",
 		    ci,
 		    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
 		    (long)ci->ci_cpuid,
 		    ci->ci_flags, ci->ci_ipis,
-		    ci->ci_curlwp,
+		    ci->ci_curlwp);
 		    ci->ci_fpcurlwp);
+	}
+}
 #endif
 #ifdef MULTIPROCESSOR
 #if NLAPIC > 0
 static void
 cpu_copy_trampoline(paddr_t pdir_pa)
+{
 	extern uint32_t nox_flag;
 	extern u_char cpu_spinup_trampoline[];
 	extern u_char cpu_spinup_trampoline_end[];
 	vaddr_t mp_trampoline_vaddr;
 @@ -1149,31 +1148,27 @@ cpu_init_msrs(struct cpu_info *ci, bool
 		wrmsr(MSR_FSBASE, 0);
 		wrmsr(MSR_GSBASE, (uint64_t)ci);
 		wrmsr(MSR_KERNELGSBASE, 0);
+	}
 #endif	/* __x86_64__ */
 	if (cpu_feature[2] & CPUID_NOX)
 		wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
+}
 void
 cpu_offline_md(void)
+{
-	int s;
+	return;
 	s = splhigh();
 	fpusave_cpu(true);
 	splx(s);
+}
 /* XXX joerg restructure and restart CPUs individually */
 static bool
 cpu_stop(device_t dv)
+{
 	struct cpu_softc *sc = device_private(dv);
 	struct cpu_info *ci = sc->sc_info;
 	int err;
 	KASSERT((ci->ci_flags & CPUF_PRESENT) != 0);
 	if ((ci->ci_flags & CPUF_PRIMARY) != 0)

--- src/sys/arch/x86/x86/fpu.c 2019/10/04 11:47:08	1.57
+++ src/sys/arch/x86/x86/fpu.c 2019/10/12 06:31:04	1.58

 @@ -1,21 +1,21 @@
-/*	$NetBSD: fpu.c,v 1.57 2019/10/04 11:47:08 maxv Exp $	*/
+/*	$NetBSD: fpu.c,v 1.58 2019/10/12 06:31:04 maxv Exp $	*/
 /*
- * Copyright (c) 2008 The NetBSD Foundation, Inc.  All
+ * Copyright (c) 2008, 2019 The NetBSD Foundation, Inc.  All
  * rights reserved.
+ *
  * This code is derived from software developed for The NetBSD Foundation
- * by Andrew Doran.
+ * by Andrew Doran and Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 @@ -86,27 +86,27 @@
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  *	@(#)npx.c	7.2 (Berkeley) 5/12/91
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.57 2019/10/04 11:47:08 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.58 2019/10/12 06:31:04 maxv Exp $");
 #include "opt_multiprocessor.h"
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/conf.h>
 #include <sys/cpu.h>
 #include <sys/file.h>
 #include <sys/proc.h>
 #include <sys/kernel.h>
 #include <sys/sysctl.h>
 #include <sys/xcall.h>
 @@ -116,34 +116,64 @@ __KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.57
 #include <machine/intr.h>
 #include <machine/cpufunc.h>
 #include <machine/pcb.h>
 #include <machine/trap.h>
 #include <machine/specialreg.h>
 #include <x86/cpu.h>
 #include <x86/fpu.h>
 #ifdef XENPV
 #define clts() HYPERVISOR_fpu_taskswitch(0)
 #define stts() HYPERVISOR_fpu_taskswitch(1)
 #endif
 void fpu_handle_deferred(void);
 void fpu_switch(struct lwp *, struct lwp *);
 uint32_t x86_fpu_mxcsr_mask __read_mostly = 0;
 static inline union savefpu *
-lwp_fpuarea(struct lwp *l)
+fpu_lwp_area(struct lwp *l)
+{
 	struct pcb *pcb = lwp_getpcb(l);
 	union savefpu *area = &pcb->pcb_savefpu;
 	KASSERT((l->l_flag & LW_SYSTEM) == 0);
 	if (l == curlwp) {
 		fpu_save();
+	}
 	KASSERT(!(l->l_md.md_flags & MDL_FPU_IN_CPU));
 	return area;
+}
 /*
  * Bring curlwp's FPU state in memory. It will get installed back in the CPU
  * when returning to userland.
  */
 void
 fpu_save(void)
+{
 	struct lwp *l = curlwp;
 	struct pcb *pcb = lwp_getpcb(l);
 	union savefpu *area = &pcb->pcb_savefpu;
-	return &pcb->pcb_savefpu;
+	kpreempt_disable();
 	if (l->l_md.md_flags & MDL_FPU_IN_CPU) {
 		KASSERT((l->l_flag & LW_SYSTEM) == 0);
 		fpu_area_save(area, x86_xsave_features);
 		l->l_md.md_flags &= ~MDL_FPU_IN_CPU;
+	}
 	kpreempt_enable();
+}
 void
 fpuinit(struct cpu_info *ci)
+{
 	/*
 	 * This might not be strictly necessary since it will be initialized
 	 * for each process. However it does no harm.
 	 */
 	clts();
 	fninit();
 	stts();
+}
 @@ -203,106 +233,117 @@ fpu_errata_amd(void)
 	 * Newer generations of AMD CPUs have CPUID_Fn80000008_EBX[2],
 	 * which indicates that FIP/FDP/FOP are restored (same behavior
 	 * as Intel). We're not using it though.
 	 */
 	fnstsw(&sw);
 	if (sw & 0x80)
 		fnclex();
 	fldummy();
+}
 void
 fpu_area_save(void *area, uint64_t xsave_features)
+{
 	clts();
 	switch (x86_fpu_save) {
 	case FPU_SAVE_FSAVE:
 		fnsave(area);
 		break;
 	case FPU_SAVE_FXSAVE:
 		fxsave(area);
 		break;
 	case FPU_SAVE_XSAVE:
 		xsave(area, xsave_features);
 		break;
 	case FPU_SAVE_XSAVEOPT:
 		xsaveopt(area, xsave_features);
 		break;
+	}
 	stts();
+}
 void
 fpu_area_restore(void *area, uint64_t xsave_features)
+{
 	clts();
 	switch (x86_fpu_save) {
 	case FPU_SAVE_FSAVE:
 		frstor(area);
 		break;
 	case FPU_SAVE_FXSAVE:
 		if (cpu_vendor == CPUVENDOR_AMD)
 			fpu_errata_amd();
 		fxrstor(area);
 		break;
 	case FPU_SAVE_XSAVE:
 	case FPU_SAVE_XSAVEOPT:
 		if (cpu_vendor == CPUVENDOR_AMD)
 			fpu_errata_amd();
 		xrstor(area, xsave_features);
 		break;
+	}
+}
-static void
+void
-fpu_lwp_install(struct lwp *l)
+fpu_handle_deferred(void)
+{
-	struct pcb *pcb = lwp_getpcb(l);
+	struct pcb *pcb = lwp_getpcb(curlwp);
 	struct cpu_info *ci = curcpu();
 	KASSERT(ci->ci_fpcurlwp == NULL);
 	KASSERT(pcb->pcb_fpcpu == NULL);
 	ci->ci_fpcurlwp = l;
 	pcb->pcb_fpcpu = ci;
 	fpu_area_restore(&pcb->pcb_savefpu, x86_xsave_features);
+}
 void fpu_switch(struct lwp *, struct lwp *);
 void
 fpu_switch(struct lwp *oldlwp, struct lwp *newlwp)
+{
-	int s;
+	struct pcb *pcb;
-	s = splhigh();
+	if ((oldlwp != NULL) && (oldlwp->l_md.md_flags & MDL_FPU_IN_CPU)) {
-#ifdef DIAGNOSTIC
+		KASSERT(!(oldlwp->l_flag & LW_SYSTEM));
-	if (oldlwp != NULL) {
+		pcb = lwp_getpcb(oldlwp);
-		struct pcb *pcb = lwp_getpcb(oldlwp);
+		fpu_area_save(&pcb->pcb_savefpu, x86_xsave_features);
-		struct cpu_info *ci = curcpu();
+		oldlwp->l_md.md_flags &= ~MDL_FPU_IN_CPU;
 		if (pcb->pcb_fpcpu == NULL) {
 			KASSERT(ci->ci_fpcurlwp != oldlwp);
 		} else if (pcb->pcb_fpcpu == ci) {
 			KASSERT(ci->ci_fpcurlwp == oldlwp);
 		} else {
 			panic("%s: oldlwp's state installed elsewhere",
 			    __func__);
+	}
-#endif
+	KASSERT(!(newlwp->l_md.md_flags & MDL_FPU_IN_CPU));
 	fpusave_cpu(true);
 	if (!(newlwp->l_flag & LW_SYSTEM))
-		fpu_lwp_install(newlwp);
+void
-	splx(s);
+fpu_lwp_fork(struct lwp *l1, struct lwp *l2)
+{
 	struct pcb *pcb2 = lwp_getpcb(l2);
 	union savefpu *fpu_save;
 	/* Kernel threads have no FPU. */
 	if (__predict_false(l2->l_flag & LW_SYSTEM)) {
 		return;
+	}
 	/* For init(8). */
 	if (__predict_false(l1->l_flag & LW_SYSTEM)) {
 		memset(&pcb2->pcb_savefpu, 0, x86_fpu_save_size);
 		return;
+	}
 	fpu_save = fpu_lwp_area(l1);
 	memcpy(&pcb2->pcb_savefpu, fpu_save, x86_fpu_save_size);
 	l2->l_md.md_flags &= ~MDL_FPU_IN_CPU;
+}
 void
 fpu_lwp_abandon(struct lwp *l)
+{
 	KASSERT(l == curlwp);
 	kpreempt_disable();
 	l->l_md.md_flags &= ~MDL_FPU_IN_CPU;
 	stts();
 	kpreempt_enable();
+}
 /* -------------------------------------------------------------------------- */
 /*
  * The following table is used to ensure that the FPE_... value
  * that is passed as a trapcode to the signal handler of the user
  * process does not have more than one bit set.
+ *
  * Multiple bits may be set if SSE simd instructions generate errors
  * on more than one value or if the user process modifies the control
  * word while a status word bit is already set (which this is a sign
  * of bad coding).
 @@ -389,31 +430,27 @@ static const uint8_t fpetable[128] = {
+ *
  * We come here with interrupts disabled.
  */
 void
 fputrap(struct trapframe *frame)
+{
 	uint32_t statbits;
 	ksiginfo_t ksi;
 	if (__predict_false(!USERMODE(frame->tf_cs))) {
 		panic("fpu trap from kernel, trapframe %p\n", frame);
+	}
-	/*
+	KASSERT(curlwp->l_md.md_flags & MDL_FPU_IN_CPU);
 	 * At this point, fpcurlwp should be curlwp.  If it wasn't, the TS bit
 	 * should be set, and we should have gotten a DNA exception.
 	 */
 	KASSERT(curcpu()->ci_fpcurlwp == curlwp);
 	if (frame->tf_trapno == T_XMM) {
 		uint32_t mxcsr;
 		x86_stmxcsr(&mxcsr);
 		statbits = mxcsr;
 		/* Clear the sticky status bits */
 		mxcsr &= ~0x3f;
 		x86_ldmxcsr(&mxcsr);
 		/* Remove masked interrupts and non-status bits */
 		statbits &= ~(statbits >> 7) & 0x3f;
 		/* Mark this is an XMM status */
 		statbits |= 0x10000;
 @@ -430,232 +467,122 @@ fputrap(struct trapframe *frame)
+	}
 	/* Doesn't matter now if we get pre-empted */
 	x86_enable_intr();
 	KSI_INIT_TRAP(&ksi);
 	ksi.ksi_signo = SIGFPE;
 	ksi.ksi_addr = (void *)X86_TF_RIP(frame);
 	ksi.ksi_code = fpetable[statbits & 0x7f];
 	ksi.ksi_trap = statbits;
 	(*curlwp->l_proc->p_emul->e_trapsignal)(curlwp, &ksi);
+}
 /*
  * Implement device not available (DNA) exception. Called with interrupts still
  * disabled.
  */
 void
 fpudna(struct trapframe *frame)
+{
-	struct cpu_info *ci = curcpu();
+	panic("fpudna from %s, ip %p, trapframe %p",
-	int s;
+	    USERMODE(frame->tf_cs) ? "userland" : "kernel",
 	    (void *)X86_TF_RIP(frame), frame);
 	if (!USERMODE(frame->tf_cs)) {
 		panic("fpudna from kernel, ip %p, trapframe %p\n",
 		    (void *)X86_TF_RIP(frame), frame);
 	/* Install the LWP's FPU state. */
 	s = splhigh();
 	fpu_lwp_install(ci->ci_curlwp);
 	splx(s);
+}
 /* -------------------------------------------------------------------------- */
 /*
  * Save current CPU's FPU state.  Must be called at IPL_HIGH.
  */
 void
 fpusave_cpu(bool save)
 	struct cpu_info *ci;
 	struct pcb *pcb;
 	struct lwp *l;
 	KASSERT(curcpu()->ci_ilevel == IPL_HIGH);
 	ci = curcpu();
 	l = ci->ci_fpcurlwp;
 	if (l == NULL) {
 		return;
 	pcb = lwp_getpcb(l);
 	if (save) {
 		fpu_area_save(&pcb->pcb_savefpu, x86_xsave_features);
 	stts();
 	pcb->pcb_fpcpu = NULL;
 	ci->ci_fpcurlwp = NULL;
 /*
  * Save l's FPU state, which may be on this processor or another processor.
  * It may take some time, so we avoid disabling preemption where possible.
  * Caller must know that the target LWP is stopped, otherwise this routine
  * may race against it.
  */
 void
 fpusave_lwp(struct lwp *l, bool save)
 	struct pcb *pcb = lwp_getpcb(l);
 	struct cpu_info *oci;
 	int s, spins, ticks;
 	spins = 0;
 	ticks = hardclock_ticks;
 	for (;;) {
 		s = splhigh();
 		oci = pcb->pcb_fpcpu;
 		if (oci == NULL) {
 			splx(s);
 			break;
 		if (oci == curcpu()) {
 			KASSERT(oci->ci_fpcurlwp == l);
 			fpusave_cpu(save);
 			splx(s);
 			break;
 		splx(s);
 #ifdef XENPV
 		if (xen_send_ipi(oci, XEN_IPI_SYNCH_FPU) != 0) {
 			panic("xen_send_ipi(%s, XEN_IPI_SYNCH_FPU) failed.",
 			    cpu_name(oci));
 #else
 		x86_send_ipi(oci, X86_IPI_SYNCH_FPU);
 #endif
 		while (pcb->pcb_fpcpu == oci && ticks == hardclock_ticks) {
 			x86_pause();
 			spins++;
 		if (spins > 100000000) {
 			panic("fpusave_lwp: did not");
 static inline void
 fpu_xstate_reload(union savefpu *fpu_save, uint64_t xstate)
+{
 	/*
 	 * Force a reload of the given xstate during the next XRSTOR.
 	 */
 	if (x86_fpu_save >= FPU_SAVE_XSAVE) {
 		fpu_save->sv_xsave_hdr.xsh_xstate_bv |= xstate;
+	}
+}
 void
 fpu_set_default_cw(struct lwp *l, unsigned int x87_cw)
+{
-	union savefpu *fpu_save = lwp_fpuarea(l);
+	union savefpu *fpu_save = fpu_lwp_area(l);
 	struct pcb *pcb = lwp_getpcb(l);
 	if (i386_use_fxsave) {
 		fpu_save->sv_xmm.fx_cw = x87_cw;
 		if (x87_cw != __INITIAL_NPXCW__) {
 			fpu_xstate_reload(fpu_save, XCR0_X87);
+		}
 	} else {
 		fpu_save->sv_87.s87_cw = x87_cw;
+	}
 	pcb->pcb_fpu_dflt_cw = x87_cw;
+}
 void
 fpu_clear(struct lwp *l, unsigned int x87_cw)
+{
 	union savefpu *fpu_save;
 	struct pcb *pcb;
 	int s;
 	KASSERT(l == curlwp);
-	KASSERT((l->l_flag & LW_SYSTEM) == 0);
+	fpu_save = fpu_lwp_area(l);
 	fpu_save = lwp_fpuarea(l);
 	pcb = lwp_getpcb(l);
 	s = splhigh();
 	KASSERT(pcb->pcb_fpcpu == NULL || pcb->pcb_fpcpu == curcpu());
 	fpusave_cpu(false);
 	KASSERT(pcb->pcb_fpcpu == NULL);
 	switch (x86_fpu_save) {
 	case FPU_SAVE_FSAVE:
 		memset(&fpu_save->sv_87, 0, x86_fpu_save_size);
 		fpu_save->sv_87.s87_tw = 0xffff;
 		fpu_save->sv_87.s87_cw = x87_cw;
 		break;
 	case FPU_SAVE_FXSAVE:
 		memset(&fpu_save->sv_xmm, 0, x86_fpu_save_size);
 		fpu_save->sv_xmm.fx_mxcsr = __INITIAL_MXCSR__;
 		fpu_save->sv_xmm.fx_mxcsr_mask = x86_fpu_mxcsr_mask;
 		fpu_save->sv_xmm.fx_cw = x87_cw;
 		break;
 	case FPU_SAVE_XSAVE:
 	case FPU_SAVE_XSAVEOPT:
 		memset(&fpu_save->sv_xmm, 0, x86_fpu_save_size);
 		fpu_save->sv_xmm.fx_mxcsr = __INITIAL_MXCSR__;
 		fpu_save->sv_xmm.fx_mxcsr_mask = x86_fpu_mxcsr_mask;
 		fpu_save->sv_xmm.fx_cw = x87_cw;
 		if (__predict_false(x87_cw != __INITIAL_NPXCW__)) {
 			fpu_xstate_reload(fpu_save, XCR0_X87);
+		}
 		break;
+	}
 	pcb = lwp_getpcb(l);
 	pcb->pcb_fpu_dflt_cw = x87_cw;
 	fpu_lwp_install(l);
 	splx(s);
+}
 void
 fpu_sigreset(struct lwp *l)
+{
-	union savefpu *fpu_save = lwp_fpuarea(l);
+	union savefpu *fpu_save = fpu_lwp_area(l);
 	struct pcb *pcb = lwp_getpcb(l);
 	/*
 	 * For signal handlers the register values don't matter. Just reset
 	 * a few fields.
 	 */
 	if (i386_use_fxsave) {
 		fpu_save->sv_xmm.fx_mxcsr = __INITIAL_MXCSR__;
 		fpu_save->sv_xmm.fx_mxcsr_mask = x86_fpu_mxcsr_mask;
 		fpu_save->sv_xmm.fx_tw = 0;
 		fpu_save->sv_xmm.fx_cw = pcb->pcb_fpu_dflt_cw;
 	} else {
 		fpu_save->sv_87.s87_tw = 0xffff;
 		fpu_save->sv_87.s87_cw = pcb->pcb_fpu_dflt_cw;
+	}
+}
 void
 fpu_save_area_fork(struct pcb *pcb2, const struct pcb *pcb1)
 	const uint8_t *src = (const uint8_t *)&pcb1->pcb_savefpu;
 	uint8_t *dst = (uint8_t *)&pcb2->pcb_savefpu;
 	memcpy(dst, src, x86_fpu_save_size);
 	KASSERT(pcb2->pcb_fpcpu == NULL);
 /* -------------------------------------------------------------------------- */
 static void
 process_xmm_to_s87(const struct fxsave *sxmm, struct save87 *s87)
+{
 	unsigned int tag, ab_tag;
 	const struct fpaccfx *fx_reg;
 	struct fpacc87 *s87_reg;
 	int i;
 	/*
 	 * For historic reasons core dumps and ptrace all use the old save87
 	 * layout.  Convert the important parts.
 @@ -759,103 +686,86 @@ process_s87_to_xmm(const struct save87 *
 	while ((tag >>= 1) >= (i <<= 1));
 	sxmm->fx_tw = ab_tag;
 	/* FP registers (in stack order) */
 	fx_reg = sxmm->fx_87_ac;
 	s87_reg = s87->s87_ac;
 	for (i = 0; i < 8; fx_reg++, s87_reg++, i++)
 		fx_reg->r = *s87_reg;
+}
 void
 process_write_fpregs_xmm(struct lwp *l, const struct fxsave *fpregs)
+{
-	union savefpu *fpu_save;
+	union savefpu *fpu_save = fpu_lwp_area(l);
 	fpusave_lwp(l, true);
 	fpu_save = lwp_fpuarea(l);
 	if (i386_use_fxsave) {
 		memcpy(&fpu_save->sv_xmm, fpregs, sizeof(fpu_save->sv_xmm));
 		/*
 		 * Invalid bits in mxcsr or mxcsr_mask will cause faults.
 		 */
 		fpu_save->sv_xmm.fx_mxcsr_mask &= x86_fpu_mxcsr_mask;
 		fpu_save->sv_xmm.fx_mxcsr &= fpu_save->sv_xmm.fx_mxcsr_mask;
 		fpu_xstate_reload(fpu_save, XCR0_X87 | XCR0_SSE);
 	} else {
 		process_xmm_to_s87(fpregs, &fpu_save->sv_87);
+	}
+}
 void
 process_write_fpregs_s87(struct lwp *l, const struct save87 *fpregs)
+{
-	union savefpu *fpu_save;
+	union savefpu *fpu_save = fpu_lwp_area(l);
 	if (i386_use_fxsave) {
 		/* Save so we don't lose the xmm registers */
 		fpusave_lwp(l, true);
 		fpu_save = lwp_fpuarea(l);
 		process_s87_to_xmm(fpregs, &fpu_save->sv_xmm);
 		fpu_xstate_reload(fpu_save, XCR0_X87 | XCR0_SSE);
 	} else {
 		fpusave_lwp(l, false);
 		fpu_save = lwp_fpuarea(l);
 		memcpy(&fpu_save->sv_87, fpregs, sizeof(fpu_save->sv_87));
+	}
+}
 void
 process_read_fpregs_xmm(struct lwp *l, struct fxsave *fpregs)
+{
-	union savefpu *fpu_save;
+	union savefpu *fpu_save = fpu_lwp_area(l);
 	fpusave_lwp(l, true);
 	fpu_save = lwp_fpuarea(l);
 	if (i386_use_fxsave) {
 		memcpy(fpregs, &fpu_save->sv_xmm, sizeof(fpu_save->sv_xmm));
 	} else {
 		memset(fpregs, 0, sizeof(*fpregs));
 		process_s87_to_xmm(&fpu_save->sv_87, fpregs);
+	}
+}
 void
 process_read_fpregs_s87(struct lwp *l, struct save87 *fpregs)
+{
-	union savefpu *fpu_save;
+	union savefpu *fpu_save = fpu_lwp_area(l);
 	fpusave_lwp(l, true);
 	fpu_save = lwp_fpuarea(l);
 	if (i386_use_fxsave) {
 		memset(fpregs, 0, sizeof(*fpregs));
 		process_xmm_to_s87(&fpu_save->sv_xmm, fpregs);
 	} else {
 		memcpy(fpregs, &fpu_save->sv_87, sizeof(fpu_save->sv_87));
+	}
+}
 int
 process_read_xstate(struct lwp *l, struct xstate *xstate)
+{
-	union savefpu *fpu_save;
+	union savefpu *fpu_save = fpu_lwp_area(l);
 	fpusave_lwp(l, true);
 	fpu_save = lwp_fpuarea(l);
 	if (x86_fpu_save == FPU_SAVE_FSAVE) {
 		/* Convert from legacy FSAVE format. */
 		memset(&xstate->xs_fxsave, 0, sizeof(xstate->xs_fxsave));
 		process_s87_to_xmm(&fpu_save->sv_87, &xstate->xs_fxsave);
 		/* We only got x87 data. */
 		xstate->xs_rfbm = XCR0_X87;
 		xstate->xs_xstate_bv = XCR0_X87;
 		return 0;
+	}
 	/* Copy the legacy area. */
 @@ -914,30 +824,27 @@ process_verify_xstate(const struct xstat
 		break;
 	default:
 		/* Verify whether no unsupported features are enabled */
 		if ((xstate->xs_rfbm & ~(x86_xsave_features & XCR0_FPU)) != 0)
 			return EINVAL;
+	}
 	return 0;
+}
 int
 process_write_xstate(struct lwp *l, const struct xstate *xstate)
+{
-	union savefpu *fpu_save;
+	union savefpu *fpu_save = fpu_lwp_area(l);
 	fpusave_lwp(l, true);
 	fpu_save = lwp_fpuarea(l);
 	/* Convert data into legacy FSAVE format. */
 	if (x86_fpu_save == FPU_SAVE_FSAVE) {
 		if (xstate->xs_xstate_bv & XCR0_X87)
 			process_xmm_to_s87(&xstate->xs_fxsave, &fpu_save->sv_87);
 		return 0;
+	}
 	/* If XSAVE is supported, make sure that xstate_bv is set correctly. */
 	if (x86_fpu_save >= FPU_SAVE_XSAVE) {
 		/*
 		 * Bit-wise xstate->xs_rfbm ? xstate->xs_xstate_bv
 		 *                          : fpu_save->sv_xsave_hdr.xsh_xstate_bv

--- src/sys/arch/x86/x86/ipi.c 2017/02/08 10:08:26	1.27
+++ src/sys/arch/x86/x86/ipi.c 2019/10/12 06:31:04	1.28

 @@ -1,14 +1,14 @@
-/*	$NetBSD: ipi.c,v 1.27 2017/02/08 10:08:26 maxv Exp $	*/
+/*	$NetBSD: ipi.c,v 1.28 2019/10/12 06:31:04 maxv Exp $	*/
 /*-
  * Copyright (c) 2000, 2008, 2009 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by RedBack Networks Inc.
+ *
  * Author: Bill Sommerfeld
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
 @@ -22,27 +22,27 @@
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.27 2017/02/08 10:08:26 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipi.c,v 1.28 2019/10/12 06:31:04 maxv Exp $");
 #include "opt_mtrr.h"
 #include <sys/param.h>
 #include <sys/device.h>
 #include <sys/systm.h>
 #include <sys/atomic.h>
 #include <sys/intr.h>
 #include <sys/ipi.h>
 #include <sys/cpu.h>
 #include <sys/xcall.h>
 #ifdef MULTIPROCESSOR
 @@ -166,27 +166,27 @@ x86_ipi_halt(struct cpu_info *ci)
 	x86_disable_intr();
 	atomic_and_32(&ci->ci_flags, ~CPUF_RUNNING);
 	for (;;) {
 		x86_hlt();
+	}
+}
 static void
 x86_ipi_synch_fpu(struct cpu_info *ci)
+{
-	fpusave_cpu(true);
+	panic("%s: impossible", __func__);
+}
 #ifdef MTRR
 static void
 x86_ipi_reload_mtrr(struct cpu_info *ci)
+{
 	if (mtrr_funcs != NULL) {
 		/*
 		 * mtrr_reload_cpu() is a macro in mtrr.h which picks
 		 * the appropriate function to use.
 		 */
 		mtrr_reload_cpu(ci);

--- src/sys/arch/x86/x86/vm_machdep.c 2019/02/11 14:59:33	1.37
+++ src/sys/arch/x86/x86/vm_machdep.c 2019/10/12 06:31:04	1.38

 @@ -1,14 +1,14 @@
-/*	$NetBSD: vm_machdep.c,v 1.37 2019/02/11 14:59:33 cherry Exp $	*/
+/*	$NetBSD: vm_machdep.c,v 1.38 2019/10/12 06:31:04 maxv Exp $	*/
 /*-
  * Copyright (c) 1982, 1986 The Regents of the University of California.
  * All rights reserved.
+ *
  * This code is derived from software contributed to Berkeley by
  * the Systems Programming Group of the University of Utah Computer
  * Science Department, and William Jolitz.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
 @@ -70,27 +70,27 @@
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  *	@(#)vm_machdep.c	7.3 (Berkeley) 5/13/91
  */
 /*
  *	Utah $Hdr: vm_machdep.c 1.16.1.1 89/06/23$
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: vm_machdep.c,v 1.37 2019/02/11 14:59:33 cherry Exp $");
+__KERNEL_RCSID(0, "$NetBSD: vm_machdep.c,v 1.38 2019/10/12 06:31:04 maxv Exp $");
 #include "opt_mtrr.h"
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/proc.h>
 #include <sys/vnode.h>
 #include <sys/buf.h>
 #include <sys/core.h>
 #include <sys/exec.h>
 #include <sys/ptrace.h>
 #include <uvm/uvm.h>
 @@ -130,49 +130,40 @@ cpu_proc_fork(struct proc *p1, struct pr
 void
 cpu_lwp_fork(struct lwp *l1, struct lwp *l2, void *stack, size_t stacksize,
     void (*func)(void *), void *arg)
+{
 	struct pcb *pcb1, *pcb2;
 	struct trapframe *tf;
 	struct switchframe *sf;
 	vaddr_t uv;
 	pcb1 = lwp_getpcb(l1);
 	pcb2 = lwp_getpcb(l2);
 	/*
 	 * If parent LWP was using FPU, then we have to save the FPU h/w
 	 * state to PCB so that we can copy it.
 	 */
 	fpusave_lwp(l1, true);
 	/*
 	 * Sync the PCB before we copy it.
 	 */
 	if (l1 == curlwp) {
 		KASSERT(pcb1 == curpcb);
 		savectx(pcb1);
 	} else {
 		KASSERT(l1 == &lwp0);
+	}
 	/* Copy the PCB from parent, except the FPU state. */
 	memcpy(pcb2, pcb1, offsetof(struct pcb, pcb_savefpu));
-	/* FPU state not installed. */
+	/* Fork the FPU state. */
-	pcb2->pcb_fpcpu = NULL;
+	fpu_lwp_fork(l1, l2);
 	/* Copy FPU state. */
 	fpu_save_area_fork(pcb2, pcb1);
 	/* Never inherit CPU Debug Registers */
 	pcb2->pcb_dbregs = NULL;
 	pcb2->pcb_flags &= ~PCB_DBREGS;
 #if defined(XENPV)
 	pcb2->pcb_iopl = IOPL_KPL;
 #endif
 	/*
 	 * Set the kernel stack address (from the address to uarea) and
 	 * trapframe address for child.
+	 *
 @@ -250,28 +241,28 @@ cpu_lwp_fork(struct lwp *l1, struct lwp
 	pcb2->pcb_ebp = (int)l2;
 #endif
+}
 /*
  * cpu_lwp_free is called from exit() to let machine-dependent
  * code free machine-dependent resources.  Note that this routine
  * must not block.
  */
 void
 cpu_lwp_free(struct lwp *l, int proc)
+{
-	/* If we were using the FPU, forget about it. */
+	/* Abandon the FPU state. */
-	fpusave_lwp(l, false);
+	fpu_lwp_abandon(l);
 	/* Abandon the dbregs state. */
 	x86_dbregs_abandon(l);
 #ifdef MTRR
 	if (proc && l->l_proc->p_md.md_flags & MDP_USEDMTRR)
 		mtrr_clean(l->l_proc);
 #endif
 	/*
 	 * Free deferred mappings if any.
 	 */
 	struct vm_page *empty_ptps = l->l_md.md_gc_ptp;
 	l->l_md.md_gc_ptp = NULL;

--- src/sys/arch/xen/x86/cpu.c 2019/03/09 08:42:25	1.129
+++ src/sys/arch/xen/x86/cpu.c 2019/10/12 06:31:04	1.130

 @@ -1,14 +1,14 @@
-/*	$NetBSD: cpu.c,v 1.129 2019/03/09 08:42:25 maxv Exp $	*/
+/*	$NetBSD: cpu.c,v 1.130 2019/10/12 06:31:04 maxv Exp $	*/
 /*-
  * Copyright (c) 2000 The NetBSD Foundation, Inc.
  * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by RedBack Networks Inc.
+ *
  * Author: Bill Sommerfeld
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
 @@ -55,27 +55,27 @@
  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.129 2019/03/09 08:42:25 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.130 2019/10/12 06:31:04 maxv Exp $");
 #include "opt_ddb.h"
 #include "opt_multiprocessor.h"
 #include "opt_mpbios.h"		/* for MPDEBUG */
 #include "opt_mtrr.h"
 #include "opt_xen.h"
 #include "lapic.h"
 #include "ioapic.h"
 #include <sys/param.h>
 #include <sys/proc.h>
 #include <sys/systm.h>
 @@ -749,35 +749,34 @@ cpu_hatch(void *v)
 #include <ddb/db_output.h>
 #include <machine/db_machdep.h>
 /*
  * Dump CPU information from ddb.
  */
 void
 cpu_debug_dump(void)
+{
 	struct cpu_info *ci;
 	CPU_INFO_ITERATOR cii;
-	db_printf("addr		dev	id	flags	ipis	curlwp 		fpcurlwp\n");
+	db_printf("addr		dev	id	flags	ipis	curlwp\n");
 	for (CPU_INFO_FOREACH(cii, ci)) {
-		db_printf("%p	%s	%ld	%x	%x	%10p	%10p\n",
+		db_printf("%p	%s	%ld	%x	%x	%10p\n",
 		    ci,
 		    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
 		    (long)ci->ci_cpuid,
 		    ci->ci_flags, ci->ci_ipis,
-		    ci->ci_curlwp,
+		    ci->ci_curlwp);
 		    ci->ci_fpcurlwp);
+	}
+}
 #endif /* DDB */
 #endif /* MULTIPROCESSOR */
 extern void hypervisor_callback(void);
 extern void failsafe_callback(void);
 #ifdef __x86_64__
 typedef void (vector)(void);
 extern vector Xsyscall, Xsyscall32;
 #endif
 @@ -1056,31 +1055,27 @@ cpu_init_msrs(struct cpu_info *ci, bool
 		HYPERVISOR_set_segment_base(SEGBASE_FS, 0);
 		HYPERVISOR_set_segment_base(SEGBASE_GS_KERNEL, (uint64_t)ci);
 		HYPERVISOR_set_segment_base(SEGBASE_GS_USER, 0);
+	}
 #endif
 	if (cpu_feature[2] & CPUID_NOX)
 		wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
+}
 void
 cpu_offline_md(void)
+{
-	int s;
+	return;
 	s = splhigh();
 	fpusave_cpu(true);
 	splx(s);
+}
 void
 cpu_get_tsc_freq(struct cpu_info *ci)
+{
 	uint32_t vcpu_tversion;
 	const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time;
 	vcpu_tversion = tinfo->version;
 	while (tinfo->version == vcpu_tversion); /* Wait for a time update. XXX: timeout ? */
 	uint64_t freq = 1000000000ULL << 32;
 	freq = freq / (uint64_t)tinfo->tsc_to_system_mul;

--- src/sys/arch/xen/x86/xen_ipi.c 2019/02/02 12:32:55	1.32
+++ src/sys/arch/xen/x86/xen_ipi.c 2019/10/12 06:31:04	1.33

 @@ -1,14 +1,14 @@
-/* $NetBSD: xen_ipi.c,v 1.32 2019/02/02 12:32:55 cherry Exp $ */
+/* $NetBSD: xen_ipi.c,v 1.33 2019/10/12 06:31:04 maxv Exp $ */
 /*-
  * Copyright (c) 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Cherry G. Mathew <cherry@zyx.in>
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -23,30 +23,30 @@
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>			/* RCS ID macro */
 /*
  * Based on: x86/ipi.c
- * __KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.32 2019/02/02 12:32:55 cherry Exp $");
+ * __KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.33 2019/10/12 06:31:04 maxv Exp $");
  */
-__KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.32 2019/02/02 12:32:55 cherry Exp $");
+__KERNEL_RCSID(0, "$NetBSD: xen_ipi.c,v 1.33 2019/10/12 06:31:04 maxv Exp $");
 #include "opt_ddb.h"
 #include <sys/types.h>
 #include <sys/atomic.h>
 #include <sys/cpu.h>
 #include <sys/mutex.h>
 #include <sys/device.h>
 #include <sys/xcall.h>
 #include <sys/ipi.h>
 #include <sys/errno.h>
 #include <sys/systm.h>
 @@ -227,27 +227,27 @@ xen_ipi_halt(struct cpu_info *ci, struct
 	KASSERT(ci != NULL);
 	if (HYPERVISOR_vcpu_op(VCPUOP_down, ci->ci_cpuid, NULL)) {
 		panic("%s shutdown failed.\n", device_xname(ci->ci_dev));
+	}
+}
 static void
 xen_ipi_synch_fpu(struct cpu_info *ci, struct intrframe *intrf)
+{
 	KASSERT(ci != NULL);
 	KASSERT(intrf != NULL);
-	fpusave_cpu(true);
+	panic("%s: impossible", __func__);
+}
 #ifdef DDB
 static void
 xen_ipi_ddb(struct cpu_info *ci, struct intrframe *intrf)
+{
 	KASSERT(ci != NULL);
 	KASSERT(intrf != NULL);
 #ifdef __x86_64__
 	ddb_ipi(intrf->if_tf);
 #else
 	struct trapframe tf;

--- src/sys/dev/nvmm/x86/nvmm_x86_svm.c 2019/10/04 12:17:05	1.49
+++ src/sys/dev/nvmm/x86/nvmm_x86_svm.c 2019/10/12 06:31:04	1.50

 @@ -1,14 +1,14 @@
-/*	$NetBSD: nvmm_x86_svm.c,v 1.49 2019/10/04 12:17:05 maxv Exp $	*/
+/*	$NetBSD: nvmm_x86_svm.c,v 1.50 2019/10/12 06:31:04 maxv Exp $	*/
 /*
  * Copyright (c) 2018 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -20,27 +20,27 @@
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.49 2019/10/04 12:17:05 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.50 2019/10/12 06:31:04 maxv Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/cpu.h>
 #include <sys/xcall.h>
 #include <sys/mman.h>
 #include <uvm/uvm.h>
 #include <uvm/uvm_page.h>
 #include <x86/cputypes.h>
 @@ -531,28 +531,26 @@ struct svm_cpudata {
 	/* MSR bitmap */
 	uint8_t *msrbm;
 	paddr_t msrbm_pa;
 	/* Host state */
 	uint64_t hxcr0;
 	uint64_t star;
 	uint64_t lstar;
 	uint64_t cstar;
 	uint64_t sfmask;
 	uint64_t fsbase;
 	uint64_t kernelgsbase;
 	bool ts_set;
 	struct xsave_header hfpu __aligned(64);
 	/* Intr state */
 	bool int_window_exit;
 	bool nmi_window_exit;
 	bool evt_pending;
 	/* Guest state */
 	uint64_t gxcr0;
 	uint64_t gprs[NVMM_X64_NGPR];
 	uint64_t drs[NVMM_X64_NDR];
 	uint64_t gtsc;
 	struct xsave_header gfpu __aligned(64);
 };
 @@ -1127,75 +1125,71 @@ svm_exit_xsetbv(struct nvmm_machine *mac
 	    (vmcb->state.rax & 0xFFFFFFFF);
 	if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) {
 		goto error;
 	} else if (__predict_false(vmcb->state.cpl != 0)) {
 		goto error;
 	} else if (__predict_false((val & ~svm_xcr0_mask) != 0)) {
 		goto error;
 	} else if (__predict_false((val & XCR0_X87) == 0)) {
 		goto error;
+	}
 	cpudata->gxcr0 = val;
 	if (svm_xcr0_mask != 0) {
 		wrxcr(0, cpudata->gxcr0);
+	}
 	svm_inkernel_advance(cpudata->vmcb);
 	return;
 error:
 	svm_inject_gp(vcpu);
+}
 static void
 svm_exit_invalid(struct nvmm_exit *exit, uint64_t code)
+{
 	exit->u.inv.hwcode = code;
 	exit->reason = NVMM_EXIT_INVALID;
+}
 /* -------------------------------------------------------------------------- */
 static void
 svm_vcpu_guest_fpu_enter(struct nvmm_cpu *vcpu)
+{
 	struct svm_cpudata *cpudata = vcpu->cpudata;
-	cpudata->ts_set = (rcr0() & CR0_TS) != 0;
+	fpu_save();
 	fpu_area_save(&cpudata->hfpu, svm_xcr0_mask);
 	fpu_area_restore(&cpudata->gfpu, svm_xcr0_mask);
 	if (svm_xcr0_mask != 0) {
 		cpudata->hxcr0 = rdxcr(0);
 		wrxcr(0, cpudata->gxcr0);
+	}
+}
 static void
 svm_vcpu_guest_fpu_leave(struct nvmm_cpu *vcpu)
+{
 	struct svm_cpudata *cpudata = vcpu->cpudata;
 	if (svm_xcr0_mask != 0) {
 		cpudata->gxcr0 = rdxcr(0);
 		wrxcr(0, cpudata->hxcr0);
+	}
 	fpu_area_save(&cpudata->gfpu, svm_xcr0_mask);
 	fpu_area_restore(&cpudata->hfpu, svm_xcr0_mask);
 	if (cpudata->ts_set) {
 		stts();
+}
 static void
 svm_vcpu_guest_dbregs_enter(struct nvmm_cpu *vcpu)
+{
 	struct svm_cpudata *cpudata = vcpu->cpudata;
 	x86_dbregs_save(curlwp);
 	ldr7(0);
 	ldr0(cpudata->drs[NVMM_X64_DR_DR0]);
 	ldr1(cpudata->drs[NVMM_X64_DR_DR1]);
 @@ -1317,44 +1311,43 @@ svm_vcpu_run(struct nvmm_machine *mach,
 	kpreempt_disable();
 	hcpu = cpu_number();
 	svm_gtlb_catchup(vcpu, hcpu);
 	svm_htlb_catchup(vcpu, hcpu);
 	if (vcpu->hcpu_last != hcpu) {
 		svm_vmcb_cache_flush_all(vmcb);
 		cpudata->gtsc_want_update = true;
+	}
 	svm_vcpu_guest_dbregs_enter(vcpu);
 	svm_vcpu_guest_misc_enter(vcpu);
 	svm_vcpu_guest_fpu_enter(vcpu);
 	while (1) {
 		if (cpudata->gtlb_want_flush) {
 			vmcb->ctrl.tlb_ctrl = svm_ctrl_tlb_flush;
 		} else {
 			vmcb->ctrl.tlb_ctrl = 0;
+		}
 		if (__predict_false(cpudata->gtsc_want_update)) {
 			vmcb->ctrl.tsc_offset = cpudata->gtsc - rdtsc();
 			svm_vmcb_cache_flush(vmcb, VMCB_CTRL_VMCB_CLEAN_I);
+		}
 		s = splhigh();
 		machgen = svm_htlb_flush(machdata, cpudata);
 		svm_vcpu_guest_fpu_enter(vcpu);
 		svm_vmrun(cpudata->vmcb_pa, cpudata->gprs);
 		svm_vcpu_guest_fpu_leave(vcpu);
 		svm_htlb_flush_ack(cpudata, machgen);
 		splx(s);
 		svm_vmcb_cache_default(vmcb);
 		if (vmcb->ctrl.exitcode != VMCB_EXITCODE_INVALID) {
 			cpudata->gtlb_want_flush = false;
 			cpudata->gtsc_want_update = false;
 			vcpu->hcpu_last = hcpu;
+		}
 		svm_exit_evt(cpudata, vmcb);
 		switch (vmcb->ctrl.exitcode) {
 @@ -1427,26 +1420,27 @@ svm_vcpu_run(struct nvmm_machine *mach,
 		if (curcpu()->ci_data.cpu_softints != 0) {
 			break;
+		}
 		if (curlwp->l_flag & LW_USERRET) {
 			break;
+		}
 		if (exit->reason != NVMM_EXIT_NONE) {
 			break;
+		}
+	}
 	cpudata->gtsc = rdtsc() + vmcb->ctrl.tsc_offset;
 	svm_vcpu_guest_fpu_leave(vcpu);
 	svm_vcpu_guest_misc_leave(vcpu);
 	svm_vcpu_guest_dbregs_leave(vcpu);
 	kpreempt_enable();
 	exit->exitstate[NVMM_X64_EXITSTATE_CR8] = __SHIFTOUT(vmcb->ctrl.v,
 	    VMCB_CTRL_V_TPR);
 	exit->exitstate[NVMM_X64_EXITSTATE_RFLAGS] = vmcb->state.rflags;
 	exit->exitstate[NVMM_X64_EXITSTATE_INT_SHADOW] =
 	    ((vmcb->ctrl.intr & VMCB_CTRL_INTR_SHADOW) != 0);
 	exit->exitstate[NVMM_X64_EXITSTATE_INT_WINDOW_EXIT] =
 	    cpudata->int_window_exit;

--- src/sys/dev/nvmm/x86/nvmm_x86_vmx.c 2019/10/04 12:17:05	1.38
+++ src/sys/dev/nvmm/x86/nvmm_x86_vmx.c 2019/10/12 06:31:04	1.39

 @@ -1,14 +1,14 @@
-/*	$NetBSD: nvmm_x86_vmx.c,v 1.38 2019/10/04 12:17:05 maxv Exp $	*/
+/*	$NetBSD: nvmm_x86_vmx.c,v 1.39 2019/10/12 06:31:04 maxv Exp $	*/
 /*
  * Copyright (c) 2018 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -20,27 +20,27 @@
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.38 2019/10/04 12:17:05 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.39 2019/10/12 06:31:04 maxv Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/cpu.h>
 #include <sys/xcall.h>
 #include <sys/mman.h>
 #include <uvm/uvm.h>
 #include <uvm/uvm_page.h>
 #include <x86/cputypes.h>
 @@ -723,28 +723,26 @@ struct vmx_cpudata {
 	bool vmcs_launched;
 	/* MSR bitmap */
 	uint8_t *msrbm;
 	paddr_t msrbm_pa;
 	/* Host state */
 	uint64_t hxcr0;
 	uint64_t star;
 	uint64_t lstar;
 	uint64_t cstar;
 	uint64_t sfmask;
 	uint64_t kernelgsbase;
 	bool ts_set;
 	struct xsave_header hfpu __aligned(64);
 	/* Intr state */
 	bool int_window_exit;
 	bool nmi_window_exit;
 	bool evt_pending;
 	/* Guest state */
 	struct msr_entry *gmsr;
 	paddr_t gmsr_pa;
 	uint64_t gmsr_misc_enable;
 	uint64_t gcr2;
 	uint64_t gcr8;
 	uint64_t gxcr0;
 @@ -1649,26 +1647,29 @@ vmx_exit_xsetbv(struct nvmm_machine *mac
 	val = (cpudata->gprs[NVMM_X64_GPR_RDX] << 32) |
 	    (cpudata->gprs[NVMM_X64_GPR_RAX] & 0xFFFFFFFF);
 	if (__predict_false(cpudata->gprs[NVMM_X64_GPR_RCX] != 0)) {
 		goto error;
 	} else if (__predict_false((val & ~vmx_xcr0_mask) != 0)) {
 		goto error;
 	} else if (__predict_false((val & XCR0_X87) == 0)) {
 		goto error;
+	}
 	cpudata->gxcr0 = val;
 	if (vmx_xcr0_mask != 0) {
 		wrxcr(0, cpudata->gxcr0);
+	}
 	vmx_inkernel_advance();
 	return;
 error:
 	vmx_inject_gp(vcpu);
+}
 #define VMX_EPT_VIOLATION_READ		__BIT(0)
 #define VMX_EPT_VIOLATION_WRITE		__BIT(1)
 #define VMX_EPT_VIOLATION_EXECUTE	__BIT(2)
 static void
 @@ -1693,53 +1694,46 @@ vmx_exit_epf(struct nvmm_machine *mach,
 	vmx_vcpu_state_provide(vcpu,
 	    NVMM_X64_STATE_GPRS | NVMM_X64_STATE_SEGS |
 	    NVMM_X64_STATE_CRS | NVMM_X64_STATE_MSRS);
+}
 /* -------------------------------------------------------------------------- */
 static void
 vmx_vcpu_guest_fpu_enter(struct nvmm_cpu *vcpu)
+{
 	struct vmx_cpudata *cpudata = vcpu->cpudata;
-	cpudata->ts_set = (rcr0() & CR0_TS) != 0;
+	fpu_save();
 	fpu_area_save(&cpudata->hfpu, vmx_xcr0_mask);
 	fpu_area_restore(&cpudata->gfpu, vmx_xcr0_mask);
 	if (vmx_xcr0_mask != 0) {
 		cpudata->hxcr0 = rdxcr(0);
 		wrxcr(0, cpudata->gxcr0);
+	}
+}
 static void
 vmx_vcpu_guest_fpu_leave(struct nvmm_cpu *vcpu)
+{
 	struct vmx_cpudata *cpudata = vcpu->cpudata;
 	if (vmx_xcr0_mask != 0) {
 		cpudata->gxcr0 = rdxcr(0);
 		wrxcr(0, cpudata->hxcr0);
+	}
 	fpu_area_save(&cpudata->gfpu, vmx_xcr0_mask);
 	fpu_area_restore(&cpudata->hfpu, vmx_xcr0_mask);
 	if (cpudata->ts_set) {
 		stts();
+}
 static void
 vmx_vcpu_guest_dbregs_enter(struct nvmm_cpu *vcpu)
+{
 	struct vmx_cpudata *cpudata = vcpu->cpudata;
 	x86_dbregs_save(curlwp);
 	ldr7(0);
 	ldr0(cpudata->drs[NVMM_X64_DR_DR0]);
 	ldr1(cpudata->drs[NVMM_X64_DR_DR1]);
 @@ -1901,51 +1895,50 @@ vmx_vcpu_run(struct nvmm_machine *mach,
 	vmx_htlb_catchup(vcpu, hcpu);
 	if (vcpu->hcpu_last != hcpu) {
 		vmx_vmwrite(VMCS_HOST_TR_SELECTOR, ci->ci_tss_sel);
 		vmx_vmwrite(VMCS_HOST_TR_BASE, (uint64_t)ci->ci_tss);
 		vmx_vmwrite(VMCS_HOST_GDTR_BASE, (uint64_t)ci->ci_gdt);
 		vmx_vmwrite(VMCS_HOST_GS_BASE, rdmsr(MSR_GSBASE));
 		cpudata->gtsc_want_update = true;
 		vcpu->hcpu_last = hcpu;
+	}
 	vmx_vcpu_guest_dbregs_enter(vcpu);
 	vmx_vcpu_guest_misc_enter(vcpu);
 	vmx_vcpu_guest_fpu_enter(vcpu);
 	while (1) {
 		if (cpudata->gtlb_want_flush) {
 			vpid_desc.vpid = cpudata->asid;
 			vpid_desc.addr = 0;
 			vmx_invvpid(vmx_tlb_flush_op, &vpid_desc);
 			cpudata->gtlb_want_flush = false;
+		}
 		if (__predict_false(cpudata->gtsc_want_update)) {
 			vmx_vmwrite(VMCS_TSC_OFFSET, cpudata->gtsc - rdtsc());
 			cpudata->gtsc_want_update = false;
+		}
 		s = splhigh();
 		machgen = vmx_htlb_flush(machdata, cpudata);
 		vmx_vcpu_guest_fpu_enter(vcpu);
 		lcr2(cpudata->gcr2);
 		if (launched) {
 			ret = vmx_vmresume(cpudata->gprs);
 		} else {
 			ret = vmx_vmlaunch(cpudata->gprs);
+		}
 		cpudata->gcr2 = rcr2();
 		vmx_vcpu_guest_fpu_leave(vcpu);
 		vmx_htlb_flush_ack(cpudata, machgen);
 		splx(s);
 		if (__predict_false(ret != 0)) {
 			vmx_exit_invalid(exit, -1);
 			break;
+		}
 		vmx_exit_evt(cpudata);
 		launched = true;
 		exitcode = vmx_vmread(VMCS_EXIT_REASON);
 		exitcode &= __BITS(15,0);
 @@ -2029,26 +2022,27 @@ vmx_vcpu_run(struct nvmm_machine *mach,
+		}
 		if (curlwp->l_flag & LW_USERRET) {
 			break;
+		}
 		if (exit->reason != NVMM_EXIT_NONE) {
 			break;
+		}
+	}
 	cpudata->vmcs_launched = launched;
 	cpudata->gtsc = vmx_vmread(VMCS_TSC_OFFSET) + rdtsc();
 	vmx_vcpu_guest_fpu_leave(vcpu);
 	vmx_vcpu_guest_misc_leave(vcpu);
 	vmx_vcpu_guest_dbregs_leave(vcpu);
 	exit->exitstate[NVMM_X64_EXITSTATE_CR8] = cpudata->gcr8;
 	exit->exitstate[NVMM_X64_EXITSTATE_RFLAGS] =
 	    vmx_vmread(VMCS_GUEST_RFLAGS);
 	intstate = vmx_vmread(VMCS_GUEST_INTERRUPTIBILITY);
 	exit->exitstate[NVMM_X64_EXITSTATE_INT_SHADOW] =
 	    (intstate & (INT_STATE_STI|INT_STATE_MOVSS)) != 0;
 	exit->exitstate[NVMM_X64_EXITSTATE_INT_WINDOW_EXIT] =
 	    cpudata->int_window_exit;
 	exit->exitstate[NVMM_X64_EXITSTATE_NMI_WINDOW_EXIT] =
 	    cpudata->nmi_window_exit;

--- src/sys/sys/param.h 2019/09/30 21:18:00	1.616
+++ src/sys/sys/param.h 2019/10/12 06:31:04	1.617

 @@ -1,14 +1,14 @@
-/*	$NetBSD: param.h,v 1.616 2019/09/30 21:18:00 kamil Exp $	*/
+/*	$NetBSD: param.h,v 1.617 2019/10/12 06:31:04 maxv Exp $	*/
 /*-
  * Copyright (c) 1982, 1986, 1989, 1993
  *	The Regents of the University of California.  All rights reserved.
  * (c) UNIX System Laboratories, Inc.
  * All or some portions of this file are derived from material licensed
  * to the University of California by American Telephone and Telegraph
  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
  * the permission of UNIX System Laboratories, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
 @@ -57,27 +57,27 @@
  *	m = minor version; a minor number of 99 indicates current.
  *	r = 0 (*)
  *	p = patchlevel
+ *
  * When new releases are made, src/gnu/usr.bin/groff/tmac/mdoc.local
  * needs to be updated and the changes sent back to the groff maintainers.
+ *
  * (*)	Up to 2.0I "release" used to be "",A-Z,Z[A-Z] but numeric
  *	    	e.g. NetBSD-1.2D  = 102040000 ('D' == 4)
  *	NetBSD-2.0H 	(200080000) was changed on 20041001 to:
  *	2.99.9		(299000900)
  */
-#define	__NetBSD_Version__	999001500	/* NetBSD 9.99.15 */
+#define	__NetBSD_Version__	999001600	/* NetBSD 9.99.16 */
 #define __NetBSD_Prereq__(M,m,p) (((((M) * 100000000) + \
     (m) * 1000000) + (p) * 100) <= __NetBSD_Version__)
 /*
  * Historical NetBSD #define
+ *
  * NetBSD 1.4 was the last release for which this value was incremented.
  * The value is now permanently fixed at 199905. It will never be
  * changed again.
+ *
  * New code must use __NetBSD_Version__ instead, and should not even
  * count on NetBSD being defined.