 @@ -1,14 +1,14 @@
-/*	$NetBSD: fpu.c,v 1.83 2023/02/25 18:28:57 riastradh Exp $	*/
+/*	$NetBSD: fpu.c,v 1.84 2023/03/03 14:32:38 riastradh Exp $	*/
 /*
  * Copyright (c) 2008, 2019 The NetBSD Foundation, Inc.  All
  * rights reserved.
+ *
  * This code is derived from software developed for The NetBSD Foundation
  * by Andrew Doran and Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -86,58 +86,55 @@
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
  *	@(#)npx.c	7.2 (Berkeley) 5/12/91
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.83 2023/02/25 18:28:57 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: fpu.c,v 1.84 2023/03/03 14:32:38 riastradh Exp $");
 #include "opt_multiprocessor.h"
 #include <sys/param.h>
-#include <sys/types.h>
+#include <sys/systm.h>
 #include <sys/conf.h>
 #include <sys/cpu.h>
 #include <sys/file.h>
 #include <sys/proc.h>
 #include <sys/kernel.h>
 #include <sys/kthread.h>
 #include <sys/proc.h>
 #include <sys/sysctl.h>
 #include <sys/systm.h>
 #include <sys/xcall.h>
 #include <machine/cpu.h>
 #include <machine/cpufunc.h>
 #include <machine/cputypes.h>
 #include <machine/cpuvar.h>
 #include <machine/cputypes.h>
 #include <machine/intr.h>
 #include <machine/cpufunc.h>
 #include <machine/pcb.h>
 #include <machine/specialreg.h>
 #include <machine/trap.h>
 #include <machine/specialreg.h>
 #include <x86/cpu.h>
 #include <x86/fpu.h>
 #ifdef XENPV
-#define	clts()	HYPERVISOR_fpu_taskswitch(0)
+#define clts() HYPERVISOR_fpu_taskswitch(0)
-#define	stts()	HYPERVISOR_fpu_taskswitch(1)
+#define stts() HYPERVISOR_fpu_taskswitch(1)
 #endif
 void fpu_handle_deferred(void);
 void fpu_switch(struct lwp *, struct lwp *);
 uint32_t x86_fpu_mxcsr_mask __read_mostly = 0;
 /*
  * True if this a thread that is allowed to use the FPU -- either a
  * user thread, or a system thread with LW_SYSTEM_FPU enabled.
  */
 static inline bool
 lwp_can_haz_fpu(struct lwp *l)
 @@ -172,48 +169,45 @@ fpu_lwp_area(struct lwp *l)
 	return area;
+}
 static inline void
 fpu_save_lwp(struct lwp *l)
+{
 	struct pcb *pcb = lwp_getpcb(l);
 	union savefpu *area = &pcb->pcb_savefpu;
 	int s;
 	s = splvm();
 	if (l->l_md.md_flags & MDL_FPU_IN_CPU) {
 		KASSERT(lwp_can_haz_fpu(l));
-		fpu_area_save(area, x86_xsave_features,
+		fpu_area_save(area, x86_xsave_features, !(l->l_proc->p_flag & PK_32));
 		    !(l->l_proc->p_flag & PK_32));
 		l->l_md.md_flags &= ~MDL_FPU_IN_CPU;
+	}
 	splx(s);
+}
 /*
  * Bring curlwp's FPU state in memory. It will get installed back in the CPU
  * when returning to userland.
  */
 void
 fpu_save(void)
+{
 	fpu_save_lwp(curlwp);
+}
 void
 fpuinit(struct cpu_info *ci)
+{
 	/*
 	 * This might not be strictly necessary since it will be initialized
 	 * for each process. However it does no harm.
 	 */
 	clts();
 	fninit();
 	stts();
+}
 void
 fpuinit_mxcsr_mask(void)
+{
 #ifndef XENPV
 @@ -274,74 +268,71 @@ fpu_errata_amd(void)
 		fnclex();
 	fldummy();
+}
 #ifdef __x86_64__
 #define XS64(x) (is_64bit ? x##64 : x)
 #else
 #define XS64(x) x
 #endif
 void
 fpu_area_save(void *area, uint64_t xsave_features, bool is_64bit)
+{
 	switch (x86_fpu_save) {
 	case FPU_SAVE_FSAVE:
 		fnsave(area);
 		break;
 	case FPU_SAVE_FXSAVE:
 		XS64(fxsave)(area);
 		break;
 	case FPU_SAVE_XSAVE:
 		XS64(xsave)(area, xsave_features);
 		break;
 	case FPU_SAVE_XSAVEOPT:
 		XS64(xsaveopt)(area, xsave_features);
 		break;
+	}
 	stts();
+}
 void
 fpu_area_restore(const void *area, uint64_t xsave_features, bool is_64bit)
+{
 	clts();
 	switch (x86_fpu_save) {
 	case FPU_SAVE_FSAVE:
 		frstor(area);
 		break;
 	case FPU_SAVE_FXSAVE:
 		if (cpu_vendor == CPUVENDOR_AMD)
 			fpu_errata_amd();
 		XS64(fxrstor)(area);
 		break;
 	case FPU_SAVE_XSAVE:
 	case FPU_SAVE_XSAVEOPT:
 		if (cpu_vendor == CPUVENDOR_AMD)
 			fpu_errata_amd();
 		XS64(xrstor)(area, xsave_features);
 		break;
+	}
+}
 void
 fpu_handle_deferred(void)
+{
 	struct pcb *pcb = lwp_getpcb(curlwp);
 	fpu_area_restore(&pcb->pcb_savefpu, x86_xsave_features,
 	    !(curlwp->l_proc->p_flag & PK_32));
+}
 void
 fpu_switch(struct lwp *oldlwp, struct lwp *newlwp)
+{
 	struct cpu_info *ci __diagused = curcpu();
 	struct pcb *pcb;
 	KASSERTMSG(ci->ci_ilevel >= IPL_SCHED, "cpu%d ilevel=%d",
 	    cpu_index(ci), ci->ci_ilevel);
 @@ -378,27 +369,27 @@ fpu_lwp_fork(struct lwp *l1, struct lwp
 void
 fpu_lwp_abandon(struct lwp *l)
+{
 	int s;
 	KASSERT(l == curlwp);
 	s = splvm();
 	l->l_md.md_flags &= ~MDL_FPU_IN_CPU;
 	stts();
 	splx(s);
+}
-/* ------------------------------------------------------------------------- */
+/* -------------------------------------------------------------------------- */
 static const union savefpu safe_fpu __aligned(64) = {
 	.sv_xmm = {
 		.fx_mxcsr = __SAFE_MXCSR__,
 	},
 };
 static const union savefpu zero_fpu __aligned(64);
 /*
  * fpu_kern_enter()
+ *
  *	Begin using the FPU.  Raises to splvm, disabling most
  *	interrupts and rendering the thread non-preemptible; caller
 @@ -427,28 +418,28 @@ fpu_kern_enter(void)
 #if 0
 	/*
 	 * Can't assert this because if the caller holds a spin lock at
 	 * IPL_VM, and previously held and released a spin lock at
 	 * higher IPL, the IPL remains raised above IPL_VM.
 	 */
 	KASSERTMSG(ci->ci_ilevel <= IPL_VM || cold, "ilevel=%d",
 	    ci->ci_ilevel);
 #endif
 	KASSERT(ci->ci_kfpu_spl == -1);
 	ci->ci_kfpu_spl = s;
 	/*
-	 * If we are in a softint and have a pinned lwp, the fpu state
+	 * If we are in a softint and have a pinned lwp, the fpu state is that
-	 * is that of the pinned lwp, so save it there.
+	 * of the pinned lwp, so save it there.
 	 */
 	while ((l->l_pflag & LP_INTR) && (l->l_switchto != NULL))
 		l = l->l_switchto;
 	fpu_save_lwp(l);
 	/*
 	 * Clear CR0_TS, which fpu_save_lwp set if it saved anything --
 	 * otherwise the CPU will trap if we try to use the FPU under
 	 * the false impression that there has been a task switch since
 	 * the last FPU usage requiring that we save the FPU state.
 	 */
 	clts();
 @@ -512,27 +503,27 @@ kthread_fpu_enter_md(void)
 	clts();
 	fpu_area_restore(&safe_fpu, x86_xsave_features, /*is_64bit*/false);
+}
 void
 kthread_fpu_exit_md(void)
+{
 	/* Zero the FPU state and disable the FPU by setting CR0_TS.  */
 	fpu_area_restore(&zero_fpu, x86_xsave_features, /*is_64bit*/false);
 	stts();
+}
-/* ------------------------------------------------------------------------- */
+/* -------------------------------------------------------------------------- */
 /*
  * The following table is used to ensure that the FPE_... value
  * that is passed as a trapcode to the signal handler of the user
  * process does not have more than one bit set.
+ *
  * Multiple bits may be set if SSE simd instructions generate errors
  * on more than one value or if the user process modifies the control
  * word while a status word bit is already set (which this is a sign
  * of bad coding).
  * We have no choice than to narrow them down to one bit, since we must
  * not send a trapcode that is not exactly one of the FPE_ macros.
+ *
 @@ -666,27 +657,26 @@ fputrap(struct trapframe *frame)
 void
 fpudna(struct trapframe *frame)
+{
 	panic("fpudna from %s, ip %p, trapframe %p",
 	    USERMODE(frame->tf_cs) ? "userland" : "kernel",
 	    (void *)X86_TF_RIP(frame), frame);
+}
 /* -------------------------------------------------------------------------- */
 static inline void
 fpu_xstate_reload(union savefpu *fpu_save, uint64_t xstate)
+{
 	/*
 	 * Force a reload of the given xstate during the next XRSTOR.
 	 */
 	if (x86_fpu_save >= FPU_SAVE_XSAVE) {
 		fpu_save->sv_xsave_hdr.xsh_xstate_bv |= xstate;
+	}
+}
 void
 fpu_set_default_cw(struct lwp *l, unsigned int x87_cw)
+{
 	union savefpu *fpu_save = fpu_lwp_area(l);
 	struct pcb *pcb = lwp_getpcb(l);
 @@ -865,27 +855,26 @@ process_read_xstate(struct lwp *l, struc
 	COPY_COMPONENT(XCR0_YMM_Hi128, XSAVE_YMM_Hi128, xs_ymm_hi128);
 	COPY_COMPONENT(XCR0_Opmask, XSAVE_Opmask, xs_opmask);
 	COPY_COMPONENT(XCR0_ZMM_Hi256, XSAVE_ZMM_Hi256, xs_zmm_hi256);
 	COPY_COMPONENT(XCR0_Hi16_ZMM, XSAVE_Hi16_ZMM, xs_hi16_zmm);
 #undef COPY_COMPONENT
 	return 0;
+}
 int
 process_verify_xstate(const struct xstate *xstate)
+{
 	/* xstate_bv must be a subset of RFBM */
 	if (xstate->xs_xstate_bv & ~xstate->xs_rfbm)
 		return EINVAL;
 	switch (x86_fpu_save) {
 	case FPU_SAVE_FSAVE:
 		if ((xstate->xs_rfbm & ~XCR0_X87))
 			return EINVAL;
 		break;
 	case FPU_SAVE_FXSAVE:
 		if ((xstate->xs_rfbm & ~(XCR0_X87 | XCR0_SSE)))
 			return EINVAL;
 		break;
 @@ -895,30 +884,28 @@ process_verify_xstate(const struct xstat
 			return EINVAL;
+	}
 	return 0;
+}
 int
 process_write_xstate(struct lwp *l, const struct xstate *xstate)
+{
 	union savefpu *fpu_save = fpu_lwp_area(l);
 	/* Convert data into legacy FSAVE format. */
 	if (x86_fpu_save == FPU_SAVE_FSAVE) {
-		if (xstate->xs_xstate_bv & XCR0_X87) {
+		if (xstate->xs_xstate_bv & XCR0_X87)
-			process_xmm_to_s87(&xstate->xs_fxsave,
+			process_xmm_to_s87(&xstate->xs_fxsave, &fpu_save->sv_87);
 			    &fpu_save->sv_87);
 		return 0;
+	}
 	/* If XSAVE is supported, make sure that xstate_bv is set correctly. */
 	if (x86_fpu_save >= FPU_SAVE_XSAVE) {
 		/*
 		 * Bit-wise "xstate->xs_rfbm ? xstate->xs_xstate_bv :
 		 *           fpu_save->sv_xsave_hdr.xsh_xstate_bv"
 		 */
 		fpu_save->sv_xsave_hdr.xsh_xstate_bv =
 		    (fpu_save->sv_xsave_hdr.xsh_xstate_bv & ~xstate->xs_rfbm) |
 		    xstate->xs_xstate_bv;
+	}
 @@ -931,36 +918,35 @@ process_write_xstate(struct lwp *l, cons
 		memcpy(&fpu_save->sv_xmm, &xstate->xs_fxsave, 24);
 		memcpy(fpu_save->sv_xmm.fx_87_ac, xstate->xs_fxsave.fx_87_ac,
 		    sizeof(xstate->xs_fxsave.fx_87_ac));
+	}
 	/*
 	 * Copy MXCSR if either SSE or AVX state is requested, to match the
 	 * XSAVE behavior for those flags.
 	 */
 	if (xstate->xs_xstate_bv & (XCR0_SSE|XCR0_YMM_Hi128)) {
 		/*
 		 * Invalid bits in mxcsr or mxcsr_mask will cause faults.
 		 */
-		fpu_save->sv_xmm.fx_mxcsr_mask =
+		fpu_save->sv_xmm.fx_mxcsr_mask = xstate->xs_fxsave.fx_mxcsr_mask
-		    xstate->xs_fxsave.fx_mxcsr_mask & x86_fpu_mxcsr_mask;
+		    & x86_fpu_mxcsr_mask;
 		fpu_save->sv_xmm.fx_mxcsr = xstate->xs_fxsave.fx_mxcsr &
 		    fpu_save->sv_xmm.fx_mxcsr_mask;
+	}
 	if (xstate->xs_xstate_bv & XCR0_SSE) {
 		memcpy(&fpu_save->sv_xsave_hdr.xsh_fxsave[160],
-		    xstate->xs_fxsave.fx_xmm,
+		    xstate->xs_fxsave.fx_xmm, sizeof(xstate->xs_fxsave.fx_xmm));
 		    sizeof(xstate->xs_fxsave.fx_xmm));
+	}
 #define COPY_COMPONENT(xcr0_val, xsave_val, field)			\
 	if (xstate->xs_xstate_bv & xcr0_val) {				\
 		KASSERT(x86_xsave_offsets[xsave_val]			\
 		    >= sizeof(struct xsave_header));			\
 		KASSERT(x86_xsave_sizes[xsave_val]			\
 		    >= sizeof(xstate->field));				\
 		memcpy((char *)fpu_save + x86_xsave_offsets[xsave_val],	\
 		    &xstate->field, sizeof(xstate->field));		\
+	}
 	COPY_COMPONENT(XCR0_YMM_Hi128, XSAVE_YMM_Hi128, xs_ymm_hi128);