 @@ -1,14 +1,14 @@
-/*	$NetBSD: nvmm_x86_svm.c,v 1.46 2019/05/11 07:31:56 maxv Exp $	*/
+/*	$NetBSD: nvmm_x86_svm.c,v 1.46.4.1 2019/10/06 11:04:55 martin Exp $	*/
 /*
  * Copyright (c) 2018 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -20,27 +20,27 @@
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.46 2019/05/11 07:31:56 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_svm.c,v 1.46.4.1 2019/10/06 11:04:55 martin Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/cpu.h>
 #include <sys/xcall.h>
 #include <sys/mman.h>
 #include <uvm/uvm.h>
 #include <uvm/uvm_page.h>
 #include <x86/cputypes.h>
 @@ -192,43 +192,45 @@ int svm_vmrun(paddr_t, uint64_t *);
 #define VMCB_EXITCODE_VMMCALL		0x0081
 #define VMCB_EXITCODE_VMLOAD		0x0082
 #define VMCB_EXITCODE_VMSAVE		0x0083
 #define VMCB_EXITCODE_STGI		0x0084
 #define VMCB_EXITCODE_CLGI		0x0085
 #define VMCB_EXITCODE_SKINIT		0x0086
 #define VMCB_EXITCODE_RDTSCP		0x0087
 #define VMCB_EXITCODE_ICEBP		0x0088
 #define VMCB_EXITCODE_WBINVD		0x0089
 #define VMCB_EXITCODE_MONITOR		0x008A
 #define VMCB_EXITCODE_MWAIT		0x008B
 #define VMCB_EXITCODE_MWAIT_CONDITIONAL	0x008C
 #define VMCB_EXITCODE_XSETBV		0x008D
 #define VMCB_EXITCODE_RDPRU		0x008E
 #define VMCB_EXITCODE_EFER_WRITE_TRAP	0x008F
 #define VMCB_EXITCODE_CR0_WRITE_TRAP	0x0090
 #define VMCB_EXITCODE_CR1_WRITE_TRAP	0x0091
 #define VMCB_EXITCODE_CR2_WRITE_TRAP	0x0092
 #define VMCB_EXITCODE_CR3_WRITE_TRAP	0x0093
 #define VMCB_EXITCODE_CR4_WRITE_TRAP	0x0094
 #define VMCB_EXITCODE_CR5_WRITE_TRAP	0x0095
 #define VMCB_EXITCODE_CR6_WRITE_TRAP	0x0096
 #define VMCB_EXITCODE_CR7_WRITE_TRAP	0x0097
 #define VMCB_EXITCODE_CR8_WRITE_TRAP	0x0098
 #define VMCB_EXITCODE_CR9_WRITE_TRAP	0x0099
 #define VMCB_EXITCODE_CR10_WRITE_TRAP	0x009A
 #define VMCB_EXITCODE_CR11_WRITE_TRAP	0x009B
 #define VMCB_EXITCODE_CR12_WRITE_TRAP	0x009C
 #define VMCB_EXITCODE_CR13_WRITE_TRAP	0x009D
 #define VMCB_EXITCODE_CR14_WRITE_TRAP	0x009E
 #define VMCB_EXITCODE_CR15_WRITE_TRAP	0x009F
 #define VMCB_EXITCODE_MCOMMIT		0x00A3
 #define VMCB_EXITCODE_NPF		0x0400
 #define VMCB_EXITCODE_AVIC_INCOMP_IPI	0x0401
 #define VMCB_EXITCODE_AVIC_NOACCEL	0x0402
 #define VMCB_EXITCODE_VMGEXIT		0x0403
 #define VMCB_EXITCODE_INVALID		-1
 /* -------------------------------------------------------------------------- */
 struct vmcb_ctrl {
 	uint32_t intercept_cr;
 #define VMCB_CTRL_INTERCEPT_RCR(x)	__BIT( 0 + x)
 #define VMCB_CTRL_INTERCEPT_WCR(x)	__BIT(16 + x)
 @@ -275,32 +277,37 @@ struct vmcb_ctrl {
 	uint32_t intercept_misc2;
 #define VMCB_CTRL_INTERCEPT_VMRUN	__BIT(0)
 #define VMCB_CTRL_INTERCEPT_VMMCALL	__BIT(1)
 #define VMCB_CTRL_INTERCEPT_VMLOAD	__BIT(2)
 #define VMCB_CTRL_INTERCEPT_VMSAVE	__BIT(3)
 #define VMCB_CTRL_INTERCEPT_STGI	__BIT(4)
 #define VMCB_CTRL_INTERCEPT_CLGI	__BIT(5)
 #define VMCB_CTRL_INTERCEPT_SKINIT	__BIT(6)
 #define VMCB_CTRL_INTERCEPT_RDTSCP	__BIT(7)
 #define VMCB_CTRL_INTERCEPT_ICEBP	__BIT(8)
 #define VMCB_CTRL_INTERCEPT_WBINVD	__BIT(9)
 #define VMCB_CTRL_INTERCEPT_MONITOR	__BIT(10)
-#define VMCB_CTRL_INTERCEPT_MWAIT	__BIT(12)
+#define VMCB_CTRL_INTERCEPT_MWAIT	__BIT(11)
 #define VMCB_CTRL_INTERCEPT_MWAIT_ARMED	__BIT(12)
 #define VMCB_CTRL_INTERCEPT_XSETBV	__BIT(13)
 #define VMCB_CTRL_INTERCEPT_RDPRU	__BIT(14)
 #define VMCB_CTRL_INTERCEPT_EFER_SPEC	__BIT(15)
 #define VMCB_CTRL_INTERCEPT_WCR_SPEC(x)	__BIT(16 + x)
-	uint8_t  rsvd1[40];
+	uint32_t intercept_misc3;
 #define VMCB_CTRL_INTERCEPT_MCOMMIT	__BIT(3)
 	uint8_t  rsvd1[36];
 	uint16_t pause_filt_thresh;
 	uint16_t pause_filt_cnt;
 	uint64_t iopm_base_pa;
 	uint64_t msrpm_base_pa;
 	uint64_t tsc_offset;
 	uint32_t guest_asid;
 	uint32_t tlb_ctrl;
 #define VMCB_CTRL_TLB_CTRL_FLUSH_ALL			0x01
 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST			0x03
 #define VMCB_CTRL_TLB_CTRL_FLUSH_GUEST_NONGLOBAL	0x07
 	uint64_t v;
 @@ -322,26 +329,28 @@ struct vmcb_ctrl {
 	uint64_t exitinfo2;
 	uint64_t exitintinfo;
 #define VMCB_CTRL_EXITINTINFO_VECTOR	__BITS(7,0)
 #define VMCB_CTRL_EXITINTINFO_TYPE	__BITS(10,8)
 #define VMCB_CTRL_EXITINTINFO_EV	__BIT(11)
 #define VMCB_CTRL_EXITINTINFO_V		__BIT(31)
 #define VMCB_CTRL_EXITINTINFO_ERRORCODE	__BITS(63,32)
 	uint64_t enable1;
 #define VMCB_CTRL_ENABLE_NP		__BIT(0)
 #define VMCB_CTRL_ENABLE_SEV		__BIT(1)
 #define VMCB_CTRL_ENABLE_ES_SEV		__BIT(2)
 #define VMCB_CTRL_ENABLE_GMET		__BIT(3)
 #define VMCB_CTRL_ENABLE_VTE		__BIT(5)
 	uint64_t avic;
 #define VMCB_CTRL_AVIC_APIC_BAR		__BITS(51,0)
 	uint64_t ghcb;
 	uint64_t eventinj;
 #define VMCB_CTRL_EVENTINJ_VECTOR	__BITS(7,0)
 #define VMCB_CTRL_EVENTINJ_TYPE		__BITS(10,8)
 #define VMCB_CTRL_EVENTINJ_EV		__BIT(11)
 #define VMCB_CTRL_EVENTINJ_V		__BIT(31)
 #define VMCB_CTRL_EVENTINJ_ERRORCODE	__BITS(63,32)
 @@ -2112,27 +2121,27 @@ svm_vcpu_destroy(struct nvmm_machine *ma
+}
 /* -------------------------------------------------------------------------- */
 static void
 svm_tlb_flush(struct pmap *pm)
+{
 	struct nvmm_machine *mach = pm->pm_data;
 	struct svm_machdata *machdata = mach->machdata;
 	atomic_inc_64(&machdata->mach_htlb_gen);
 	/* Generates IPIs, which cause #VMEXITs. */
-	pmap_tlb_shootdown(pmap_kernel(), -1, PG_G, TLBSHOOT_UPDATE);
+	pmap_tlb_shootdown(pmap_kernel(), -1, PTE_G, TLBSHOOT_UPDATE);
+}
 static void
 svm_machine_create(struct nvmm_machine *mach)
+{
 	struct svm_machdata *machdata;
 	/* Fill in pmap info. */
 	mach->vm->vm_map.pmap->pm_data = (void *)mach;
 	mach->vm->vm_map.pmap->pm_tlb_flush = svm_tlb_flush;
 	machdata = kmem_zalloc(sizeof(struct svm_machdata), KM_SLEEP);
 	mach->machdata = machdata;

 @@ -1,14 +1,14 @@
-/*	$NetBSD: nvmm_x86_vmx.c,v 1.36.2.1 2019/09/24 18:14:59 martin Exp $	*/
+/*	$NetBSD: nvmm_x86_vmx.c,v 1.36.2.2 2019/10/06 11:04:55 martin Exp $	*/
 /*
  * Copyright (c) 2018 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Maxime Villard.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -20,27 +20,27 @@
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.36.2.1 2019/09/24 18:14:59 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: nvmm_x86_vmx.c,v 1.36.2.2 2019/10/06 11:04:55 martin Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/cpu.h>
 #include <sys/xcall.h>
 #include <sys/mman.h>
 #include <uvm/uvm.h>
 #include <uvm/uvm_page.h>
 #include <x86/cputypes.h>
 @@ -2739,27 +2739,27 @@ vmx_vcpu_destroy(struct nvmm_machine *ma
+}
 /* -------------------------------------------------------------------------- */
 static void
 vmx_tlb_flush(struct pmap *pm)
+{
 	struct nvmm_machine *mach = pm->pm_data;
 	struct vmx_machdata *machdata = mach->machdata;
 	atomic_inc_64(&machdata->mach_htlb_gen);
 	/* Generates IPIs, which cause #VMEXITs. */
-	pmap_tlb_shootdown(pmap_kernel(), -1, PG_G, TLBSHOOT_UPDATE);
+	pmap_tlb_shootdown(pmap_kernel(), -1, PTE_G, TLBSHOOT_UPDATE);
+}
 static void
 vmx_machine_create(struct nvmm_machine *mach)
+{
 	struct pmap *pmap = mach->vm->vm_map.pmap;
 	struct vmx_machdata *machdata;
 	/* Convert to EPT. */
 	pmap_ept_transform(pmap);
 	/* Fill in pmap info. */
 	pmap->pm_data = (void *)mach;