 @@ -1,14 +1,14 @@
-/*	$NetBSD: nvmm_x86.c,v 1.7.4.3 2020/08/18 09:29:52 martin Exp $	*/
+/*	$NetBSD: nvmm_x86.c,v 1.7.4.4 2020/08/26 17:55:49 martin Exp $	*/
 /*
  * Copyright (c) 2018-2020 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -20,27 +20,27 @@
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86.c,v 1.7.4.3 2020/08/18 09:29:52 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86.c,v 1.7.4.4 2020/08/26 17:55:49 martin Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/cpu.h>
 #include <uvm/uvm.h>
 #include <uvm/uvm_page.h>
 #include <x86/cputypes.h>
 #include <x86/specialreg.h>
 #include <x86/pmap.h>
 @@ -294,86 +294,86 @@ const struct nvmm_x86_cpuid_mask nvmm_cp
 	    CPUID_SSE2 |
 	    CPUID_SS |
 	    CPUID_HTT |
 	    /* CPUID_TM excluded */
 	    CPUID_IA64 |
 	    CPUID_SBF
 };
 const struct nvmm_x86_cpuid_mask nvmm_cpuid_00000007 = {
 	.eax = ~0,
 	.ebx =
 	    CPUID_SEF_FSGSBASE |
 	    /* CPUID_SEF_TSC_ADJUST excluded */
-	    CPUID_SEF_SGX |
+	    /* CPUID_SEF_SGX excluded */
 	    CPUID_SEF_BMI1 |
-	    CPUID_SEF_HLE |
+	    /* CPUID_SEF_HLE excluded */
 	    /* CPUID_SEF_AVX2 excluded */
 	    CPUID_SEF_FDPEXONLY |
 	    CPUID_SEF_SMEP |
 	    CPUID_SEF_BMI2 |
 	    CPUID_SEF_ERMS |
 	    /* CPUID_SEF_INVPCID excluded, but re-included in VMX */
-	    CPUID_SEF_RTM |
+	    /* CPUID_SEF_RTM excluded */
 	    /* CPUID_SEF_QM excluded */
 	    CPUID_SEF_FPUCSDS |
 	    /* CPUID_SEF_MPX excluded */
 	    CPUID_SEF_PQE |
 	    /* CPUID_SEF_AVX512F excluded */
 	    /* CPUID_SEF_AVX512DQ excluded */
 	    CPUID_SEF_RDSEED |
 	    CPUID_SEF_ADX |
 	    CPUID_SEF_SMAP |
 	    /* CPUID_SEF_AVX512_IFMA excluded */
 	    CPUID_SEF_CLFLUSHOPT |
 	    CPUID_SEF_CLWB,
 	    /* CPUID_SEF_PT excluded */
 	    /* CPUID_SEF_AVX512PF excluded */
 	    /* CPUID_SEF_AVX512ER excluded */
 	    /* CPUID_SEF_AVX512CD excluded */
 	    /* CPUID_SEF_SHA excluded */
 	    /* CPUID_SEF_AVX512BW excluded */
 	    /* CPUID_SEF_AVX512VL excluded */
 	.ecx =
 	    CPUID_SEF_PREFETCHWT1 |
 	    /* CPUID_SEF_AVX512_VBMI excluded */
 	    CPUID_SEF_UMIP |
-	    CPUID_SEF_PKU |
+	    /* CPUID_SEF_PKU excluded */
-	    CPUID_SEF_OSPKE |
+	    /* CPUID_SEF_OSPKE excluded */
-	    CPUID_SEF_WAITPKG |
+	    /* CPUID_SEF_WAITPKG excluded */
 	    /* CPUID_SEF_AVX512_VBMI2 excluded */
 	    /* CPUID_SEF_CET_SS excluded */
 	    CPUID_SEF_GFNI |
 	    CPUID_SEF_VAES |
 	    CPUID_SEF_VPCLMULQDQ |
 	    /* CPUID_SEF_AVX512_VNNI excluded */
 	    /* CPUID_SEF_AVX512_BITALG excluded */
 	    /* CPUID_SEF_AVX512_VPOPCNTDQ excluded */
 	    /* CPUID_SEF_MAWAU excluded */
 	    /* CPUID_SEF_RDPID excluded */
 	    CPUID_SEF_CLDEMOTE |
 	    CPUID_SEF_MOVDIRI |
-	    CPUID_SEF_MOVDIR64B |
+	    CPUID_SEF_MOVDIR64B,
-	    CPUID_SEF_SGXLC,
+	    /* CPUID_SEF_SGXLC excluded */
 	    /* CPUID_SEF_PKS excluded */
 	.edx =
 	    /* CPUID_SEF_AVX512_4VNNIW excluded */
 	    /* CPUID_SEF_AVX512_4FMAPS excluded */
-	    /* CPUID_SEF_FSREP_MOV excluded */
+	    CPUID_SEF_FSREP_MOV |
 	    /* CPUID_SEF_AVX512_VP2INTERSECT excluded */
 	    /* CPUID_SEF_SRBDS_CTRL excluded */
 	    CPUID_SEF_MD_CLEAR |
 	    /* CPUID_SEF_TSX_FORCE_ABORT excluded */
-	    /* CPUID_SEF_SERIALIZE excluded */
+	    CPUID_SEF_SERIALIZE |
 	    /* CPUID_SEF_HYBRID excluded */
 	    /* CPUID_SEF_TSXLDTRK excluded */
 	    /* CPUID_SEF_CET_IBT excluded */
 	    /* CPUID_SEF_IBRS excluded */
 	    /* CPUID_SEF_STIBP excluded */
 	    /* CPUID_SEF_L1D_FLUSH excluded */
 	    CPUID_SEF_ARCH_CAP
 	    /* CPUID_SEF_CORE_CAP excluded */
 	    /* CPUID_SEF_SSBD excluded */
 };
 const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000001 = {
 	.eax = ~0,
 @@ -381,27 +381,27 @@ const struct nvmm_x86_cpuid_mask nvmm_cp
 	.ecx =
 	    CPUID_LAHF |
 	    CPUID_CMPLEGACY |
 	    /* CPUID_SVM excluded */
 	    /* CPUID_EAPIC excluded */
 	    CPUID_ALTMOVCR0 |
 	    CPUID_LZCNT |
 	    CPUID_SSE4A |
 	    CPUID_MISALIGNSSE |
 	    CPUID_3DNOWPF |
 	    /* CPUID_OSVW excluded */
 	    CPUID_IBS |
 	    CPUID_XOP |
-	    CPUID_SKINIT |
+	    /* CPUID_SKINIT excluded */
 	    CPUID_WDT |
 	    CPUID_LWP |
 	    CPUID_FMA4 |
 	    CPUID_TCE |
 	    CPUID_NODEID |
 	    CPUID_TBM |
 	    CPUID_TOPOEXT |
 	    CPUID_PCEC |
 	    CPUID_PCENB |
 	    CPUID_SPM |
 	    CPUID_DBE |
 	    CPUID_PTSC |
 	    CPUID_L2IPERFC,
 @@ -411,26 +411,46 @@ const struct nvmm_x86_cpuid_mask nvmm_cp
 	    CPUID_MPC |
 	    CPUID_XD |
 	    CPUID_MMXX |
 	    CPUID_MMX |
 	    CPUID_FXSR |
 	    CPUID_FFXSR |
 	    CPUID_P1GB |
 	    /* CPUID_RDTSCP excluded */
 	    CPUID_EM64T |
 	    CPUID_3DNOW2 |
 	    CPUID_3DNOW
 };
 const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000007 = {
 	.eax = 0,
 	.ebx = 0,
 	.ecx = 0,
 	.edx = CPUID_APM_ITSC
 };
 const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000008 = {
 	.eax = ~0,
 	.ebx =
 	    CPUID_CAPEX_CLZERO |
 	    /* CPUID_CAPEX_IRPERF excluded */
 	    CPUID_CAPEX_XSAVEERPTR |
 	    /* CPUID_CAPEX_RDPRU excluded */
 	    /* CPUID_CAPEX_MCOMMIT excluded */
 	    CPUID_CAPEX_WBNOINVD,
 	.ecx = ~0, /* TODO? */
 	.edx = 0
 };
 bool
 nvmm_x86_pat_validate(uint64_t val)
+{
 	uint8_t *pat = (uint8_t *)&val;
 	size_t i;
 	for (i = 0; i < 8; i++) {
 		if (__predict_false(pat[i] & ~__BITS(2,0)))
 			return false;
 		if (__predict_false(pat[i] == 2 || pat[i] == 3))
 			return false;
+	}

 @@ -1,14 +1,14 @@
-/*	$NetBSD: nvmm_x86_svm.c,v 1.46.4.8 2020/08/18 09:29:52 martin Exp $	*/
+/*	$NetBSD: nvmm_x86_svm.c,v 1.46.4.9 2020/08/26 17:55:48 martin Exp $	*/
 /*
  * Copyright (c) 2018-2019 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -20,27 +20,27 @@
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.46.4.8 2020/08/18 09:29:52 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.46.4.9 2020/08/26 17:55:48 martin Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/cpu.h>
 #include <sys/xcall.h>
 #include <sys/mman.h>
 #include <uvm/uvm.h>
 #include <uvm/uvm_page.h>
 #include <x86/cputypes.h>
 @@ -773,35 +773,71 @@ svm_vcpu_event_commit(struct nvmm_cpu *v
 static inline void
 svm_inkernel_advance(struct vmcb *vmcb)
+{
 	/*
 	 * Maybe we should also apply single-stepping and debug exceptions.
 	 * Matters for guest-ring3, because it can execute 'cpuid' under a
 	 * debugger.
 	 */
 	vmcb->state.rip = vmcb->ctrl.nrip;
 	vmcb->ctrl.intr &= ~VMCB_CTRL_INTR_SHADOW;
+}
 #define SVM_CPUID_MAX_BASIC		0xD
 #define SVM_CPUID_MAX_HYPERVISOR	0x40000000
 #define SVM_CPUID_MAX_EXTENDED		0x8000001F
 static uint32_t svm_cpuid_max_basic __read_mostly;
 static uint32_t svm_cpuid_max_extended __read_mostly;
 static void
 svm_inkernel_exec_cpuid(struct svm_cpudata *cpudata, uint64_t eax, uint64_t ecx)
+{
 	u_int descs[4];
 	x86_cpuid2(eax, ecx, descs);
 	cpudata->vmcb->state.rax = descs[0];
 	cpudata->gprs[NVMM_X64_GPR_RBX] = descs[1];
 	cpudata->gprs[NVMM_X64_GPR_RCX] = descs[2];
 	cpudata->gprs[NVMM_X64_GPR_RDX] = descs[3];
+}
 static void
 svm_inkernel_handle_cpuid(struct nvmm_cpu *vcpu, uint64_t eax, uint64_t ecx)
+{
 	struct svm_cpudata *cpudata = vcpu->cpudata;
 	uint64_t cr4;
 	if (eax < 0x40000000) {
 		if (__predict_false(eax > svm_cpuid_max_basic)) {
 			eax = svm_cpuid_max_basic;
 			svm_inkernel_exec_cpuid(cpudata, eax, ecx);
+		}
 	} else if (eax < 0x80000000) {
 		if (__predict_false(eax > SVM_CPUID_MAX_HYPERVISOR)) {
 			eax = svm_cpuid_max_basic;
 			svm_inkernel_exec_cpuid(cpudata, eax, ecx);
+		}
 	} else {
 		if (__predict_false(eax > svm_cpuid_max_extended)) {
 			eax = svm_cpuid_max_basic;
 			svm_inkernel_exec_cpuid(cpudata, eax, ecx);
+		}
+	}
 	switch (eax) {
 	case 0x00000000:
 		cpudata->vmcb->state.rax = svm_cpuid_max_basic;
 		break;
 	case 0x00000001:
 		cpudata->vmcb->state.rax &= nvmm_cpuid_00000001.eax;
 		cpudata->gprs[NVMM_X64_GPR_RBX] &= ~CPUID_LOCAL_APIC_ID;
 		cpudata->gprs[NVMM_X64_GPR_RBX] |= __SHIFTIN(vcpu->cpuid,
 		    CPUID_LOCAL_APIC_ID);
 		cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000001.ecx;
 		cpudata->gprs[NVMM_X64_GPR_RCX] |= CPUID2_RAZ;
 		cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000001.edx;
 		/* CPUID2_OSXSAVE depends on CR4. */
 @@ -811,30 +847,40 @@ svm_inkernel_handle_cpuid(struct nvmm_cp
+		}
 		break;
 	case 0x00000002: /* Empty */
 	case 0x00000003: /* Empty */
 	case 0x00000004: /* Empty */
 	case 0x00000005: /* Monitor/MWait */
 	case 0x00000006: /* Power Management Related Features */
 		cpudata->vmcb->state.rax = 0;
 		cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
 		break;
 	case 0x00000007: /* Structured Extended Features */
-		cpudata->vmcb->state.rax &= nvmm_cpuid_00000007.eax;
+		switch (ecx) {
-		cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_00000007.ebx;
+		case 0:
-		cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000007.ecx;
+			cpudata->vmcb->state.rax = 0;
-		cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000007.edx;
+			cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_00000007.ebx;
 			cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000007.ecx;
 			cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000007.edx;
 			break;
 		default:
 			cpudata->vmcb->state.rax = 0;
 			cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
 			cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
 			cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
 			break;
+		}
 		break;
 	case 0x00000008: /* Empty */
 	case 0x00000009: /* Empty */
 	case 0x0000000A: /* Empty */
 	case 0x0000000B: /* Empty */
 	case 0x0000000C: /* Empty */
 		cpudata->vmcb->state.rax = 0;
 		cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
 		break;
 	case 0x0000000D: /* Processor Extended State Enumeration */
 		if (svm_xcr0_mask == 0) {
 @@ -869,32 +915,94 @@ svm_inkernel_handle_cpuid(struct nvmm_cp
+		}
 		break;
 	case 0x40000000: /* Hypervisor Information */
 		cpudata->vmcb->state.rax = SVM_CPUID_MAX_HYPERVISOR;
 		cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
 		memcpy(&cpudata->gprs[NVMM_X64_GPR_RBX], "___ ", 4);
 		memcpy(&cpudata->gprs[NVMM_X64_GPR_RCX], "NVMM", 4);
 		memcpy(&cpudata->gprs[NVMM_X64_GPR_RDX], " ___", 4);
 		break;
 	case 0x80000000:
 		cpudata->vmcb->state.rax = svm_cpuid_max_extended;
 		break;
 	case 0x80000001:
 		cpudata->vmcb->state.rax &= nvmm_cpuid_80000001.eax;
 		cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000001.ebx;
 		cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000001.ecx;
 		cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000001.edx;
 		break;
 	case 0x80000002: /* Extended Processor Name String */
 	case 0x80000003: /* Extended Processor Name String */
 	case 0x80000004: /* Extended Processor Name String */
 	case 0x80000005: /* L1 Cache and TLB Information */
 	case 0x80000006: /* L2 Cache and TLB and L3 Cache Information */
 		break;
 	case 0x80000007: /* Processor Power Management and RAS Capabilities */
 		cpudata->vmcb->state.rax &= nvmm_cpuid_80000007.eax;
 		cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000007.ebx;
 		cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000007.ecx;
 		cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000007.edx;
 		break;
 	case 0x80000008: /* Processor Capacity Parameters and Ext Feat Ident */
 		cpudata->vmcb->state.rax &= nvmm_cpuid_80000008.eax;
 		cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000008.ebx;
 		cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000008.ecx;
 		cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000008.edx;
 		break;
 	case 0x80000009: /* Empty */
 	case 0x8000000A: /* SVM Features */
 	case 0x8000000B: /* Empty */
 	case 0x8000000C: /* Empty */
 	case 0x8000000D: /* Empty */
 	case 0x8000000E: /* Empty */
 	case 0x8000000F: /* Empty */
 	case 0x80000010: /* Empty */
 	case 0x80000011: /* Empty */
 	case 0x80000012: /* Empty */
 	case 0x80000013: /* Empty */
 	case 0x80000014: /* Empty */
 	case 0x80000015: /* Empty */
 	case 0x80000016: /* Empty */
 	case 0x80000017: /* Empty */
 	case 0x80000018: /* Empty */
 		cpudata->vmcb->state.rax = 0;
 		cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
 		break;
 	case 0x80000019: /* TLB Characteristics for 1GB pages */
 	case 0x8000001A: /* Instruction Optimizations */
 		break;
 	case 0x8000001B: /* Instruction-Based Sampling Capabilities */
 	case 0x8000001C: /* Lightweight Profiling Capabilities */
 		cpudata->vmcb->state.rax = 0;
 		cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
 		break;
 	case 0x8000001D: /* Cache Topology Information */
 	case 0x8000001E: /* Processor Topology Information */
 		break; /* TODO? */
 	case 0x8000001F: /* Encrypted Memory Capabilities */
 		cpudata->vmcb->state.rax = 0;
 		cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
 		break;
 	default:
 		break;
+	}
+}
 static void
 svm_exit_insn(struct vmcb *vmcb, struct nvmm_vcpu_exit *exit, uint64_t reason)
+{
 	exit->u.insn.npc = vmcb->ctrl.nrip;
 	exit->reason = reason;
+}
 static void
 @@ -2401,26 +2509,33 @@ svm_init(void)
 	/* The guest TLB flush command. */
 	if (descs[3] & CPUID_AMD_SVM_FlushByASID) {
 		svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_GUEST;
 	} else {
 		svm_ctrl_tlb_flush = VMCB_CTRL_TLB_CTRL_FLUSH_ALL;
+	}
 	/* Init the ASID. */
 	svm_init_asid(descs[1]);
 	/* Init the XCR0 mask. */
 	svm_xcr0_mask = SVM_XCR0_MASK_DEFAULT & x86_xsave_features;
 	/* Init the max basic CPUID leaf. */
 	svm_cpuid_max_basic = uimin(cpuid_level, SVM_CPUID_MAX_BASIC);
 	/* Init the max extended CPUID leaf. */
 	x86_cpuid(0x80000000, descs);
 	svm_cpuid_max_extended = uimin(descs[0], SVM_CPUID_MAX_EXTENDED);
 	memset(hsave, 0, sizeof(hsave));
 	for (CPU_INFO_FOREACH(cii, ci)) {
 		pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
 		hsave[cpu_index(ci)].pa = VM_PAGE_TO_PHYS(pg);
+	}
 	xc = xc_broadcast(0, svm_change_cpu, (void *)true, NULL);
 	xc_wait(xc);
+}
 static void
 svm_fini_asid(void)
+{

 @@ -1,14 +1,14 @@
-/*	$NetBSD: nvmm_x86_vmx.c,v 1.36.2.10 2020/08/18 09:29:52 martin Exp $	*/
+/*	$NetBSD: nvmm_x86_vmx.c,v 1.36.2.11 2020/08/26 17:55:49 martin Exp $	*/
 /*
  * Copyright (c) 2018-2019 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -20,27 +20,27 @@
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.36.2.10 2020/08/18 09:29:52 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.36.2.11 2020/08/26 17:55:49 martin Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/cpu.h>
 #include <sys/xcall.h>
 #include <sys/mman.h>
 #include <sys/bitops.h>
 #include <uvm/uvm.h>
 #include <uvm/uvm_page.h>
 @@ -1152,26 +1152,27 @@ vmx_exit_exc_nmi(struct nvmm_machine *ma
+	}
 	exit->reason = NVMM_VCPU_EXIT_NONE;
 	return;
 error:
 	vmx_exit_invalid(exit, VMCS_EXITCODE_EXC_NMI);
+}
 #define VMX_CPUID_MAX_BASIC		0x16
 #define VMX_CPUID_MAX_HYPERVISOR	0x40000000
 #define VMX_CPUID_MAX_EXTENDED		0x80000008
 static uint32_t vmx_cpuid_max_basic __read_mostly;
 static uint32_t vmx_cpuid_max_extended __read_mostly;
 static void
 vmx_inkernel_exec_cpuid(struct vmx_cpudata *cpudata, uint64_t eax, uint64_t ecx)
+{
 	u_int descs[4];
 	x86_cpuid2(eax, ecx, descs);
 	cpudata->gprs[NVMM_X64_GPR_RAX] = descs[0];
 	cpudata->gprs[NVMM_X64_GPR_RBX] = descs[1];
 	cpudata->gprs[NVMM_X64_GPR_RCX] = descs[2];
 	cpudata->gprs[NVMM_X64_GPR_RDX] = descs[3];
+}
 @@ -1183,26 +1184,31 @@ vmx_inkernel_handle_cpuid(struct nvmm_ma
 	unsigned int ncpus;
 	uint64_t cr4;
 	if (eax < 0x40000000) {
 		if (__predict_false(eax > vmx_cpuid_max_basic)) {
 			eax = vmx_cpuid_max_basic;
 			vmx_inkernel_exec_cpuid(cpudata, eax, ecx);
+		}
 	} else if (eax < 0x80000000) {
 		if (__predict_false(eax > VMX_CPUID_MAX_HYPERVISOR)) {
 			eax = vmx_cpuid_max_basic;
 			vmx_inkernel_exec_cpuid(cpudata, eax, ecx);
+		}
 	} else {
 		if (__predict_false(eax > vmx_cpuid_max_extended)) {
 			eax = vmx_cpuid_max_basic;
 			vmx_inkernel_exec_cpuid(cpudata, eax, ecx);
+		}
+	}
 	switch (eax) {
 	case 0x00000000:
 		cpudata->gprs[NVMM_X64_GPR_RAX] = vmx_cpuid_max_basic;
 		break;
 	case 0x00000001:
 		cpudata->gprs[NVMM_X64_GPR_RAX] &= nvmm_cpuid_00000001.eax;
 		cpudata->gprs[NVMM_X64_GPR_RBX] &= ~CPUID_LOCAL_APIC_ID;
 		cpudata->gprs[NVMM_X64_GPR_RBX] |= __SHIFTIN(vcpu->cpuid,
 		    CPUID_LOCAL_APIC_ID);
 @@ -1228,32 +1234,42 @@ vmx_inkernel_handle_cpuid(struct nvmm_ma
 		cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
 		break;
 	case 0x00000004: /* Deterministic Cache Parameters */
 		break; /* TODO? */
 	case 0x00000005: /* MONITOR/MWAIT */
 	case 0x00000006: /* Thermal and Power Management */
 		cpudata->gprs[NVMM_X64_GPR_RAX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
 		break;
 	case 0x00000007: /* Structured Extended Feature Flags Enumeration */
-		cpudata->gprs[NVMM_X64_GPR_RAX] &= nvmm_cpuid_00000007.eax;
+		switch (ecx) {
-		cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_00000007.ebx;
+		case 0:
-		cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000007.ecx;
+			cpudata->gprs[NVMM_X64_GPR_RAX] = 0;
-		cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000007.edx;
+			cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_00000007.ebx;
-		if (vmx_procbased_ctls2 & PROC_CTLS2_INVPCID_ENABLE) {
+			cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_00000007.ecx;
-			cpudata->gprs[NVMM_X64_GPR_RBX] |= CPUID_SEF_INVPCID;
+			cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_00000007.edx;
 			if (vmx_procbased_ctls2 & PROC_CTLS2_INVPCID_ENABLE) {
 				cpudata->gprs[NVMM_X64_GPR_RBX] |= CPUID_SEF_INVPCID;
+			}
 			break;
 		default:
 			cpudata->gprs[NVMM_X64_GPR_RAX] = 0;
 			cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
 			cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
 			cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
 			break;
+		}
 		break;
 	case 0x00000008: /* Empty */
 	case 0x00000009: /* Direct Cache Access Information */
 		cpudata->gprs[NVMM_X64_GPR_RAX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
 		break;
 	case 0x0000000A: /* Architectural Performance Monitoring */
 		cpudata->gprs[NVMM_X64_GPR_RAX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
 @@ -1345,39 +1361,52 @@ vmx_inkernel_handle_cpuid(struct nvmm_ma
 	case 0x00000016: /* Processor Frequency Information */
 		break;
 	case 0x40000000: /* Hypervisor Information */
 		cpudata->gprs[NVMM_X64_GPR_RAX] = VMX_CPUID_MAX_HYPERVISOR;
 		cpudata->gprs[NVMM_X64_GPR_RBX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RCX] = 0;
 		cpudata->gprs[NVMM_X64_GPR_RDX] = 0;
 		memcpy(&cpudata->gprs[NVMM_X64_GPR_RBX], "___ ", 4);
 		memcpy(&cpudata->gprs[NVMM_X64_GPR_RCX], "NVMM", 4);
 		memcpy(&cpudata->gprs[NVMM_X64_GPR_RDX], " ___", 4);
 		break;
 	case 0x80000000:
 		cpudata->gprs[NVMM_X64_GPR_RAX] = vmx_cpuid_max_extended;
 		break;
 	case 0x80000001:
 		cpudata->gprs[NVMM_X64_GPR_RAX] &= nvmm_cpuid_80000001.eax;
 		cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000001.ebx;
 		cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000001.ecx;
 		cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000001.edx;
 		break;
 	case 0x80000002: /* Processor Brand String */
 	case 0x80000003: /* Processor Brand String */
 	case 0x80000004: /* Processor Brand String */
 	case 0x80000005: /* Reserved Zero */
 	case 0x80000006: /* Cache Information */
 		break;
 	case 0x80000007: /* TSC Information */
 		cpudata->gprs[NVMM_X64_GPR_RAX] &= nvmm_cpuid_80000007.eax;
 		cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000007.ebx;
 		cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000007.ecx;
 		cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000007.edx;
 		break;
 	case 0x80000008: /* Address Sizes */
 		cpudata->gprs[NVMM_X64_GPR_RAX] &= nvmm_cpuid_80000008.eax;
 		cpudata->gprs[NVMM_X64_GPR_RBX] &= nvmm_cpuid_80000008.ebx;
 		cpudata->gprs[NVMM_X64_GPR_RCX] &= nvmm_cpuid_80000008.ecx;
 		cpudata->gprs[NVMM_X64_GPR_RDX] &= nvmm_cpuid_80000008.edx;
 		break;
 	default:
 		break;
+	}
+}
 static void
 vmx_exit_insn(struct nvmm_vcpu_exit *exit, uint64_t reason)
+{
 	uint64_t inslen, rip;
 	inslen = vmx_vmread(VMCS_EXIT_INSTRUCTION_LENGTH);
 @@ -3313,39 +3342,44 @@ vmx_init_l1tf(void)
 		/* Enable hardware mitigation. */
 		vmx_msrlist_entry_nmsr += 1;
+	}
+}
 static void
 vmx_init(void)
+{
 	CPU_INFO_ITERATOR cii;
 	struct cpu_info *ci;
 	uint64_t xc, msr;
 	struct vmxon *vmxon;
 	uint32_t revision;
 	u_int descs[4];
 	paddr_t pa;
 	vaddr_t va;
 	int error;
 	/* Init the ASID bitmap (VPID). */
 	vmx_init_asid(VPID_MAX);
 	/* Init the XCR0 mask. */
 	vmx_xcr0_mask = VMX_XCR0_MASK_DEFAULT & x86_xsave_features;
-	/* Init the max CPUID leaves. */
+	/* Init the max basic CPUID leaf. */
 	vmx_cpuid_max_basic = uimin(cpuid_level, VMX_CPUID_MAX_BASIC);
 	/* Init the max extended CPUID leaf. */
 	x86_cpuid(0x80000000, descs);
 	vmx_cpuid_max_extended = uimin(descs[0], VMX_CPUID_MAX_EXTENDED);
 	/* Init the TLB flush op, the EPT flush op and the EPTP type. */
 	msr = rdmsr(MSR_IA32_VMX_EPT_VPID_CAP);
 	if ((msr & IA32_VMX_EPT_VPID_INVVPID_CONTEXT) != 0) {
 		vmx_tlb_flush_op = VMX_INVVPID_CONTEXT;
 	} else {
 		vmx_tlb_flush_op = VMX_INVVPID_ALL;
+	}
 	if ((msr & IA32_VMX_EPT_VPID_INVEPT_CONTEXT) != 0) {
 		vmx_ept_flush_op = VMX_INVEPT_CONTEXT;
 	} else {
 		vmx_ept_flush_op = VMX_INVEPT_ALL;
+	}
 	if ((msr & IA32_VMX_EPT_VPID_WB) != 0) {

 @@ -1,17 +1,17 @@
-/*	$NetBSD: nvmm_x86.h,v 1.15.4.1 2019/11/10 12:58:30 martin Exp $	*/
+/*	$NetBSD: nvmm_x86.h,v 1.15.4.2 2020/08/26 17:55:49 martin Exp $	*/
 /*
- * Copyright (c) 2018-2019 The NetBSD Foundation, Inc.
+ * Copyright (c) 2018-2020 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -310,19 +310,21 @@ struct nvmm_vcpu_conf_tpr {
 #ifdef _KERNEL
 #define NVMM_X86_MACH_NCONF	0
 #define NVMM_X86_VCPU_NCONF	2
 struct nvmm_x86_cpuid_mask {
 	uint32_t eax;
 	uint32_t ebx;
 	uint32_t ecx;
 	uint32_t edx;
 };
 extern const struct nvmm_x64_state nvmm_x86_reset_state;
 extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_00000001;
 extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_00000007;
 extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000001;
 extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000007;
 extern const struct nvmm_x86_cpuid_mask nvmm_cpuid_80000008;
 bool nvmm_x86_pat_validate(uint64_t);
 #endif
 #endif /* ASM_NVMM */
 #endif /* _NVMM_X86_H_ */