 @@ -1,14 +1,14 @@
-/*	$NetBSD: nvmm_x86_svm.c,v 1.66 2020/08/05 10:31:37 maxv Exp $	*/
+/*	$NetBSD: nvmm_x86_svm.c,v 1.67 2020/08/05 15:22:25 maxv Exp $	*/
 /*
  * Copyright (c) 2018-2020 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -20,27 +20,27 @@
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.66 2020/08/05 10:31:37 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.67 2020/08/05 15:22:25 maxv Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/cpu.h>
 #include <sys/xcall.h>
 #include <sys/mman.h>
 #include <uvm/uvm.h>
 #include <uvm/uvm_page.h>
 #include <x86/cputypes.h>
 @@ -222,31 +222,36 @@ svm_stgi(void)
 #define VMCB_EXITCODE_CR3_WRITE_TRAP	0x0093
 #define VMCB_EXITCODE_CR4_WRITE_TRAP	0x0094
 #define VMCB_EXITCODE_CR5_WRITE_TRAP	0x0095
 #define VMCB_EXITCODE_CR6_WRITE_TRAP	0x0096
 #define VMCB_EXITCODE_CR7_WRITE_TRAP	0x0097
 #define VMCB_EXITCODE_CR8_WRITE_TRAP	0x0098
 #define VMCB_EXITCODE_CR9_WRITE_TRAP	0x0099
 #define VMCB_EXITCODE_CR10_WRITE_TRAP	0x009A
 #define VMCB_EXITCODE_CR11_WRITE_TRAP	0x009B
 #define VMCB_EXITCODE_CR12_WRITE_TRAP	0x009C
 #define VMCB_EXITCODE_CR13_WRITE_TRAP	0x009D
 #define VMCB_EXITCODE_CR14_WRITE_TRAP	0x009E
 #define VMCB_EXITCODE_CR15_WRITE_TRAP	0x009F
 #define VMCB_EXITCODE_INVLPGB		0x00A0
 #define VMCB_EXITCODE_INVLPGB_ILLEGAL	0x00A1
 #define VMCB_EXITCODE_INVPCID		0x00A2
 #define VMCB_EXITCODE_MCOMMIT		0x00A3
 #define VMCB_EXITCODE_TLBSYNC		0x00A4
 #define VMCB_EXITCODE_NPF		0x0400
 #define VMCB_EXITCODE_AVIC_INCOMP_IPI	0x0401
 #define VMCB_EXITCODE_AVIC_NOACCEL	0x0402
 #define VMCB_EXITCODE_VMGEXIT		0x0403
 #define VMCB_EXITCODE_BUSY		-2ULL
 #define VMCB_EXITCODE_INVALID		-1ULL
 /* -------------------------------------------------------------------------- */
 struct vmcb_ctrl {
 	uint32_t intercept_cr;
 #define VMCB_CTRL_INTERCEPT_RCR(x)	__BIT( 0 + x)
 #define VMCB_CTRL_INTERCEPT_WCR(x)	__BIT(16 + x)
 	uint32_t intercept_dr;
 #define VMCB_CTRL_INTERCEPT_RDR(x)	__BIT( 0 + x)
 #define VMCB_CTRL_INTERCEPT_WDR(x)	__BIT(16 + x)
 @@ -297,27 +302,31 @@ struct vmcb_ctrl {
 #define VMCB_CTRL_INTERCEPT_SKINIT	__BIT(6)
 #define VMCB_CTRL_INTERCEPT_RDTSCP	__BIT(7)
 #define VMCB_CTRL_INTERCEPT_ICEBP	__BIT(8)
 #define VMCB_CTRL_INTERCEPT_WBINVD	__BIT(9)
 #define VMCB_CTRL_INTERCEPT_MONITOR	__BIT(10)
 #define VMCB_CTRL_INTERCEPT_MWAIT	__BIT(11)
 #define VMCB_CTRL_INTERCEPT_MWAIT_ARMED	__BIT(12)
 #define VMCB_CTRL_INTERCEPT_XSETBV	__BIT(13)
 #define VMCB_CTRL_INTERCEPT_RDPRU	__BIT(14)
 #define VMCB_CTRL_INTERCEPT_EFER_SPEC	__BIT(15)
 #define VMCB_CTRL_INTERCEPT_WCR_SPEC(x)	__BIT(16 + x)
 	uint32_t intercept_misc3;
 #define VMCB_CTRL_INTERCEPT_INVLPGB_ALL	__BIT(0)
 #define VMCB_CTRL_INTERCEPT_INVLPGB_ILL	__BIT(1)
 #define VMCB_CTRL_INTERCEPT_PCID	__BIT(2)
 #define VMCB_CTRL_INTERCEPT_MCOMMIT	__BIT(3)
 #define VMCB_CTRL_INTERCEPT_TLBSYNC	__BIT(4)
 	uint8_t  rsvd1[36];
 	uint16_t pause_filt_thresh;
 	uint16_t pause_filt_cnt;
 	uint64_t iopm_base_pa;
 	uint64_t msrpm_base_pa;
 	uint64_t tsc_offset;
 	uint32_t guest_asid;
 	uint32_t tlb_ctrl;
 #define VMCB_CTRL_TLB_CTRL_FLUSH_ALL			0x01
 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST			0x03
 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST_NONGLOBAL	0x07
 @@ -325,26 +334,27 @@ struct vmcb_ctrl {
 	uint64_t v;
 #define VMCB_CTRL_V_TPR			__BITS(3,0)
 #define VMCB_CTRL_V_IRQ			__BIT(8)
 #define VMCB_CTRL_V_VGIF		__BIT(9)
 #define VMCB_CTRL_V_INTR_PRIO		__BITS(19,16)
 #define VMCB_CTRL_V_IGN_TPR		__BIT(20)
 #define VMCB_CTRL_V_INTR_MASKING	__BIT(24)
 #define VMCB_CTRL_V_GUEST_VGIF		__BIT(25)
 #define VMCB_CTRL_V_AVIC_EN		__BIT(31)
 #define VMCB_CTRL_V_INTR_VECTOR		__BITS(39,32)
 	uint64_t intr;
 #define VMCB_CTRL_INTR_SHADOW		__BIT(0)
 #define VMCB_CTRL_INTR_MASK		__BIT(1)
 	uint64_t exitcode;
 	uint64_t exitinfo1;
 	uint64_t exitinfo2;
 	uint64_t exitintinfo;
 #define VMCB_CTRL_EXITINTINFO_VECTOR	__BITS(7,0)
 #define VMCB_CTRL_EXITINTINFO_TYPE	__BITS(10,8)
 #define VMCB_CTRL_EXITINTINFO_EV	__BIT(11)
 #define VMCB_CTRL_EXITINTINFO_V		__BIT(31)
 #define VMCB_CTRL_EXITINTINFO_ERRORCODE	__BITS(63,32)
 	uint64_t enable1;
 @@ -389,27 +399,27 @@ struct vmcb_ctrl {
 	uint32_t rsvd2;
 	uint64_t nrip;
 	uint8_t	inst_len;
 	uint8_t	inst_bytes[15];
 	uint64_t avic_abpp;
 	uint64_t rsvd3;
 	uint64_t avic_ltp;
 	uint64_t avic_phys;
 #define VMCB_CTRL_AVIC_PHYS_TABLE_PTR	__BITS(51,12)
 #define VMCB_CTRL_AVIC_PHYS_MAX_INDEX	__BITS(7,0)
 	uint64_t rsvd4;
-	uint64_t vmcb_ptr;
+	uint64_t vmsa_ptr;
 	uint8_t	pad[752];
 } __packed;
 CTASSERT(sizeof(struct vmcb_ctrl) == 1024);
 struct vmcb_segment {
 	uint16_t selector;
 	uint16_t attrib;	/* hidden */
 	uint32_t limit;		/* hidden */
 	uint64_t base;		/* hidden */
 } __packed;
 @@ -1439,26 +1449,31 @@ svm_vcpu_run(struct nvmm_machine *mach,
 			exit->reason = NVMM_VCPU_EXIT_SHUTDOWN;
 			break;
 		case VMCB_EXITCODE_RDPMC:
 		case VMCB_EXITCODE_RSM:
 		case VMCB_EXITCODE_INVLPGA:
 		case VMCB_EXITCODE_VMRUN:
 		case VMCB_EXITCODE_VMMCALL:
 		case VMCB_EXITCODE_VMLOAD:
 		case VMCB_EXITCODE_VMSAVE:
 		case VMCB_EXITCODE_STGI:
 		case VMCB_EXITCODE_CLGI:
 		case VMCB_EXITCODE_SKINIT:
 		case VMCB_EXITCODE_RDTSCP:
 		case VMCB_EXITCODE_RDPRU:
 		case VMCB_EXITCODE_INVLPGB:
 		case VMCB_EXITCODE_INVPCID:
 		case VMCB_EXITCODE_MCOMMIT:
 		case VMCB_EXITCODE_TLBSYNC:
 			svm_inject_ud(vcpu);
 			exit->reason = NVMM_VCPU_EXIT_NONE;
 			break;
 		case VMCB_EXITCODE_MONITOR:
 			svm_exit_insn(vmcb, exit, NVMM_VCPU_EXIT_MONITOR);
 			break;
 		case VMCB_EXITCODE_MWAIT:
 		case VMCB_EXITCODE_MWAIT_CONDITIONAL:
 			svm_exit_insn(vmcb, exit, NVMM_VCPU_EXIT_MWAIT);
 			break;
 		case VMCB_EXITCODE_XSETBV:
 			svm_exit_xsetbv(mach, vcpu, exit);
 			break;
 @@ -2032,27 +2047,37 @@ svm_vcpu_init(struct nvmm_machine *mach,
 	 * Intercept the rest below.
 	 */
 	vmcb->ctrl.intercept_misc2 =
 	    VMCB_CTRL_INTERCEPT_VMRUN |
 	    VMCB_CTRL_INTERCEPT_VMMCALL |
 	    VMCB_CTRL_INTERCEPT_VMLOAD |
 	    VMCB_CTRL_INTERCEPT_VMSAVE |
 	    VMCB_CTRL_INTERCEPT_STGI |
 	    VMCB_CTRL_INTERCEPT_CLGI |
 	    VMCB_CTRL_INTERCEPT_SKINIT |
 	    VMCB_CTRL_INTERCEPT_RDTSCP |
 	    VMCB_CTRL_INTERCEPT_MONITOR |
 	    VMCB_CTRL_INTERCEPT_MWAIT |
-	    VMCB_CTRL_INTERCEPT_XSETBV;
+	    VMCB_CTRL_INTERCEPT_XSETBV |
 	    VMCB_CTRL_INTERCEPT_RDPRU;
 	/*
 	 * Intercept everything.
 	 */
 	vmcb->ctrl.intercept_misc3 =
 	    VMCB_CTRL_INTERCEPT_INVLPGB_ALL |
 	    VMCB_CTRL_INTERCEPT_PCID |
 	    VMCB_CTRL_INTERCEPT_MCOMMIT |
 	    VMCB_CTRL_INTERCEPT_TLBSYNC;
 	/* Intercept all I/O accesses. */
 	memset(cpudata->iobm, 0xFF, IOBM_SIZE);
 	vmcb->ctrl.iopm_base_pa = cpudata->iobm_pa;
 	/* Allow direct access to certain MSRs. */
 	memset(cpudata->msrbm, 0xFF, MSRBM_SIZE);
 	svm_vcpu_msr_allow(cpudata->msrbm, MSR_EFER, true, false);
 	svm_vcpu_msr_allow(cpudata->msrbm, MSR_STAR, true, true);
 	svm_vcpu_msr_allow(cpudata->msrbm, MSR_LSTAR, true, true);
 	svm_vcpu_msr_allow(cpudata->msrbm, MSR_CSTAR, true, true);
 	svm_vcpu_msr_allow(cpudata->msrbm, MSR_SFMASK, true, true);
 	svm_vcpu_msr_allow(cpudata->msrbm, MSR_KERNELGSBASE, true, true);