 @@ -1,425 +1,517 @@
-/*      $NetBSD: xengnt.c,v 1.29.2.1 2020/04/20 11:29:01 bouyer Exp $      */
+/*      $NetBSD: xengnt.c,v 1.29.2.2 2020/04/20 19:42:10 bouyer Exp $      */
 /*
  * Copyright (c) 2006 Manuel Bouyer.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: xengnt.c,v 1.29.2.1 2020/04/20 11:29:01 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: xengnt.c,v 1.29.2.2 2020/04/20 19:42:10 bouyer Exp $");
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kmem.h>
 #include <sys/queue.h>
 #include <sys/extent.h>
 #include <sys/kernel.h>
 #include <sys/mutex.h>
 #include <uvm/uvm.h>
 #include <xen/hypervisor.h>
 #include <xen/xen.h>
 #include <xen/granttables.h>
 #include "opt_xen.h"
 /*
  * grant table v2 is not supported for HVM guests on 4.11 at last.
  * see xen/arch/x86/hvm/hypercall.c in Xen sources (missing
  * GNTTABOP_get_status_frames)
  */
 #ifdef XENPV
 #define USE_GRANT_V2
 #endif
 /* #define XENDEBUG */
 #ifdef XENDEBUG
 #define DPRINTF(x) printf x
 #else
 #define DPRINTF(x)
 #endif
 #define NR_GRANT_ENTRIES_PER_PAGE (PAGE_SIZE / sizeof(grant_entry_v2_t))
 #define NR_GRANT_STATUS_PER_PAGE (PAGE_SIZE / sizeof(grant_status_t))
 /* External tools reserve first few grant table entries. */
 #define NR_RESERVED_ENTRIES 8
 /* Current number of frames making up the grant table */
 int gnt_nr_grant_frames;
 /* Maximum number of frames that can make up the grant table */
 int gnt_max_grant_frames;
 /* Number of grant status frames */
 int gnt_status_frames;
 /* table of free grant entries */
 grant_ref_t *gnt_entries;
 /* last free entry */
 int last_gnt_entry;
 /* empty entry in the list */
 #define XENGNT_NO_ENTRY 0xffffffff
 /* VM address of the grant table */
 #ifdef USE_GRANT_V2
 #define NR_GRANT_ENTRIES_PER_PAGE (PAGE_SIZE / sizeof(grant_entry_v2_t))
 #define NR_GRANT_STATUS_PER_PAGE (PAGE_SIZE / sizeof(grant_status_t))
 grant_entry_v2_t *grant_table;
 /* Number of grant status frames */
 int gnt_status_frames;
 grant_status_t *grant_status;
 #else /* USE_GRANT_V2 */
 #define NR_GRANT_ENTRIES_PER_PAGE (PAGE_SIZE / sizeof(grant_entry_v1_t))
 grant_entry_v1_t *grant_table;
 #endif /* USE_GRANT_V2 */
 kmutex_t grant_lock;
 static grant_ref_t xengnt_get_entry(void);
 static void xengnt_free_entry(grant_ref_t);
 static int xengnt_more_entries(void);
 static int xengnt_map_status(void);
 void
 xengnt_init(void)
+{
 	struct gnttab_query_size query;
 	struct gnttab_set_version gntversion;
 	int rc;
 	int nr_grant_entries;
 	int i;
 	query.dom = DOMID_SELF;
 	rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
 	if ((rc < 0) || (query.status != GNTST_okay))
 		gnt_max_grant_frames = 4; /* Legacy max number of frames */
 	else
 		gnt_max_grant_frames = query.max_nr_frames;
 	/*
 	 * Always allocate max number of grant frames, never expand in runtime
 	 */
 	gnt_nr_grant_frames = gnt_max_grant_frames;
 #ifdef USE_GRANT_V2
 	struct gnttab_set_version gntversion;
 	gntversion.version = 2;
 	rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gntversion, 1);
 	if (rc < 0 || gntversion.version != 2)
 		panic("GNTTABOP_set_version 2 failed %d", rc);
 #endif /* USE_GRANT_V2 */
 	nr_grant_entries =
 	    gnt_max_grant_frames * NR_GRANT_ENTRIES_PER_PAGE;
 	grant_table = (void *)uvm_km_alloc(kernel_map,
 	    gnt_max_grant_frames * PAGE_SIZE, 0, UVM_KMF_VAONLY);
 	if (grant_table == NULL)
 		panic("xengnt_init() table no VM space");
 	gnt_entries = kmem_alloc((nr_grant_entries + 1) * sizeof(grant_ref_t),
 	    KM_SLEEP);
 	for (i = 0; i <= nr_grant_entries; i++)
 		gnt_entries[i] = XENGNT_NO_ENTRY;
 #ifdef USE_GRANT_V2
 	gnt_status_frames =
 	    round_page(nr_grant_entries * sizeof(grant_status_t)) / PAGE_SIZE;
 	grant_status = (void *)uvm_km_alloc(kernel_map,
 	    gnt_status_frames * PAGE_SIZE, 0, UVM_KMF_VAONLY);
 	if (grant_status == NULL)
 		panic("xengnt_init() status no VM space");
 #endif /* USE_GRANT_V2 */
 	mutex_init(&grant_lock, MUTEX_DEFAULT, IPL_VM);
 	xengnt_resume();
+}
 /*
  * Resume grant table state
  */
 bool
 xengnt_resume(void)
+{
 	int previous_nr_grant_frames = gnt_nr_grant_frames;
 	last_gnt_entry = 0;
 	gnt_nr_grant_frames = 0;
 	mutex_enter(&grant_lock);
 	while (gnt_nr_grant_frames < previous_nr_grant_frames) {
 		if (xengnt_more_entries() != 0)
 			panic("xengnt_resume: can't restore grant frames");
+	}
 	xengnt_map_status();
 	mutex_exit(&grant_lock);
 	return true;
+}
 /*
  * Suspend grant table state
  */
 bool
 xengnt_suspend(void) {
 	int i;
 	mutex_enter(&grant_lock);
 	KASSERT(gnt_entries[last_gnt_entry] == XENGNT_NO_ENTRY);
 	for (i = 0; i < last_gnt_entry; i++) {
 		/* invalidate all grant entries (necessary for resume) */
 		gnt_entries[i] = XENGNT_NO_ENTRY;
+	}
 	/* Remove virtual => machine mapping for grant table */
 	pmap_kremove((vaddr_t)grant_table, gnt_nr_grant_frames * PAGE_SIZE);
 #ifdef USE_GRANT_V2
 	/* Remove virtual => machine mapping for status table */
 	pmap_kremove((vaddr_t)grant_status, gnt_status_frames * PAGE_SIZE);
 #endif
 	pmap_update(pmap_kernel());
 	mutex_exit(&grant_lock);
 	return true;
+}
 /*
  * Get status frames and enter them into the VA space.
  */
 static int
 xengnt_map_status(void)
+{
 #ifdef USE_GRANT_V2
 	gnttab_get_status_frames_t getstatus;
 	uint64_t *pages;
 	size_t sz;
 	int err;
 	KASSERT(mutex_owned(&grant_lock));
 	sz = gnt_status_frames * sizeof(*pages);
 	pages = kmem_alloc(sz, KM_NOSLEEP);
 	if (pages == NULL)
 		return ENOMEM;
 	getstatus.dom = DOMID_SELF;
 	getstatus.nr_frames = gnt_status_frames;
 	set_xen_guest_handle(getstatus.frame_list, pages);
 	/*
 	 * get the status frames, and return the list of their virtual
 	 * addresses in 'pages'
 	 */
-	if (HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
+	if ((err = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
-	    &getstatus, 1) != 0)
+	    &getstatus, 1)) != 0)
-		panic("%s: get_status_frames failed", __func__);
+		panic("%s: get_status_frames failed: %d", __func__, err);
 	if (getstatus.status != GNTST_okay) {
 		aprint_error("%s: get_status_frames returned %d\n",
 		    __func__, getstatus.status);
 		kmem_free(pages, sz);
 		return ENOMEM;
+	}
 	/*
 	 * map between status_table addresses and the machine addresses of
 	 * the status table frames
 	 */
 	for (int i = 0; i < gnt_status_frames; i++) {
 		pmap_kenter_ma(((vaddr_t)grant_status) + i * PAGE_SIZE,
 		    ((paddr_t)pages[i]) << PAGE_SHIFT,
 		    VM_PROT_WRITE, 0);
+	}
 	pmap_update(pmap_kernel());
 	kmem_free(pages, sz);
 #endif /* USE_GRANT_V2 */
 	return 0;
+}
 /*
  * Add another page to the grant table
  * Returns 0 on success, ENOMEM on failure
  */
 static int
 xengnt_more_entries(void)
+{
 	gnttab_setup_table_t setup;
 	u_long *pages;
 	int nframes_new = gnt_nr_grant_frames + 1;
 	int i, start_gnt;
 	size_t sz;
 	KASSERT(mutex_owned(&grant_lock));
 	if (gnt_nr_grant_frames == gnt_max_grant_frames)
 		return ENOMEM;
 	sz = nframes_new * sizeof(*pages);
 	pages = kmem_alloc(sz, KM_NOSLEEP);
 	if (pages == NULL)
 		return ENOMEM;
 	if (xen_feature(XENFEAT_auto_translated_physmap)) {
 		/*
 		 * Note: Although we allocate space for the entire
 		 * table, in this mode we only update one entry at a
 		 * time.
 		 */
 		struct vm_page *pg;
 		struct xen_add_to_physmap xmap;
 		pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE|UVM_PGA_ZERO);
 		pages[gnt_nr_grant_frames] = atop(uvm_vm_page_to_phys(pg));
 		xmap.domid = DOMID_SELF;
 		xmap.space = XENMAPSPACE_grant_table;
 		xmap.idx = gnt_nr_grant_frames;
 		xmap.gpfn = pages[gnt_nr_grant_frames];
 		if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xmap) < 0)
 			panic("%s: Unable to register HYPERVISOR_shared_info\n", __func__);
 	} else {
 		setup.dom = DOMID_SELF;
 		setup.nr_frames = nframes_new;
 		set_xen_guest_handle(setup.frame_list, pages);
 		/*
 		 * setup the grant table, made of nframes_new frames
 		 * and return the list of their virtual addresses
 		 * in 'pages'
 		 */
 		if (HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1) != 0)
 			panic("%s: setup table failed", __func__);
 		if (setup.status != GNTST_okay) {
 			aprint_error("%s: setup table returned %d\n",
 			    __func__, setup.status);
 			kmem_free(pages, sz);
 			return ENOMEM;
+		}
+	}
 	DPRINTF(("xengnt_more_entries: map 0x%lx -> %p\n",
 	    pages[gnt_nr_grant_frames],
 	    (char *)grant_table + gnt_nr_grant_frames * PAGE_SIZE));
 	/*
 	 * map between grant_table addresses and the machine addresses of
 	 * the grant table frames
 	 */
 	pmap_kenter_ma(((vaddr_t)grant_table) + gnt_nr_grant_frames * PAGE_SIZE,
 	    ((paddr_t)pages[gnt_nr_grant_frames]) << PAGE_SHIFT,
 	    VM_PROT_WRITE, 0);
 	pmap_update(pmap_kernel());
 	/*
 	 * add the grant entries associated to the last grant table frame
 	 * and mark them as free. Prevent using the first grants (from 0 to 8)
 	 * since they are used by the tools.
 	 */
 	start_gnt = (gnt_nr_grant_frames * NR_GRANT_ENTRIES_PER_PAGE) <
 	            (NR_RESERVED_ENTRIES + 1) ?
 	            (NR_RESERVED_ENTRIES + 1) :
 	            (gnt_nr_grant_frames * NR_GRANT_ENTRIES_PER_PAGE);
 	for (i = start_gnt;
 	    i < nframes_new * NR_GRANT_ENTRIES_PER_PAGE;
 	    i++) {
 		KASSERT(gnt_entries[last_gnt_entry] == XENGNT_NO_ENTRY);
 		gnt_entries[last_gnt_entry] = i;
 		last_gnt_entry++;
+	}
 	gnt_nr_grant_frames = nframes_new;
 	kmem_free(pages, sz);
 	return 0;
+}
 /*
  * Returns a reference to the first free entry in grant table
  */
 static grant_ref_t
 xengnt_get_entry(void)
+{
 	grant_ref_t entry;
 	static struct timeval xengnt_nonmemtime;
 	static const struct timeval xengnt_nonmemintvl = {5,0};
 	KASSERT(mutex_owned(&grant_lock));
 	if (__predict_false(last_gnt_entry == 0)) {
 		if (ratecheck(&xengnt_nonmemtime, &xengnt_nonmemintvl))
 			printf("xengnt_get_entry: out of grant "
 			    "table entries\n");
 		return XENGNT_NO_ENTRY;
+	}
 	KASSERT(gnt_entries[last_gnt_entry] == XENGNT_NO_ENTRY);
 	last_gnt_entry--;
 	entry = gnt_entries[last_gnt_entry];
 	gnt_entries[last_gnt_entry] = XENGNT_NO_ENTRY;
 	KASSERT(entry != XENGNT_NO_ENTRY && entry > NR_RESERVED_ENTRIES);
 	KASSERT(last_gnt_entry >= 0);
 	KASSERT(last_gnt_entry <= gnt_max_grant_frames * NR_GRANT_ENTRIES_PER_PAGE);
 	return entry;
+}
 /*
  * Mark the grant table entry as free
  */
 static void
 xengnt_free_entry(grant_ref_t entry)
+{
 	mutex_enter(&grant_lock);
 	KASSERT(entry > NR_RESERVED_ENTRIES);
 	KASSERT(gnt_entries[last_gnt_entry] == XENGNT_NO_ENTRY);
 	KASSERT(last_gnt_entry >= 0);
 	KASSERT(last_gnt_entry <= gnt_max_grant_frames * NR_GRANT_ENTRIES_PER_PAGE);
 	gnt_entries[last_gnt_entry] = entry;
 	last_gnt_entry++;
 	mutex_exit(&grant_lock);
+}
 #ifdef USE_GRANT_V2
 int
 xengnt_grant_access(domid_t dom, paddr_t ma, int ro, grant_ref_t *entryp)
+{
 	mutex_enter(&grant_lock);
 	*entryp = xengnt_get_entry();
 	if (__predict_false(*entryp == XENGNT_NO_ENTRY)) {
 		mutex_exit(&grant_lock);
 		return ENOMEM;
+	}
 	grant_table[*entryp].full_page.frame = ma >> PAGE_SHIFT;
 	grant_table[*entryp].hdr.domid = dom;
 	/*
 	 * ensure that the above values reach global visibility
 	 * before permitting frame's access (done when we set flags)
 	 */
 	xen_rmb();
 	grant_table[*entryp].hdr.flags =
 	    GTF_permit_access | (ro ? GTF_readonly : 0);
 	mutex_exit(&grant_lock);
 	return 0;
+}
 void
 xengnt_revoke_access(grant_ref_t entry)
+{
 	grant_table[entry].hdr.flags = 0;
 	xen_mb();	/* Concurrent access by hypervisor */
 	if (__predict_false((grant_status[entry] & (GTF_reading|GTF_writing))
 	    != 0))
 		printf("xengnt_revoke_access(%u): still in use\n",
 		    entry);
 	else {
 		/*
 		 * The read of grant_status needs to have acquire semantics.
 		 * Reads already have that on x86, so need only protect
 		 * against compiler reordering. May need full barrier
 		 * on other architectures.
 		 */
 		__insn_barrier();
+	}
 	xengnt_free_entry(entry);
+}
 int
 xengnt_status(grant_ref_t entry)
+{
 	return grant_status[entry] & (GTF_reading|GTF_writing);
+}
 #else /* USE_GRANT_V2 */
 int
 xengnt_grant_access(domid_t dom, paddr_t ma, int ro, grant_ref_t *entryp)
+{
 	mutex_enter(&grant_lock);
 	*entryp = xengnt_get_entry();
 	if (__predict_false(*entryp == XENGNT_NO_ENTRY)) {
 		mutex_exit(&grant_lock);
 		return ENOMEM;
+	}
 	grant_table[*entryp].frame = ma >> PAGE_SHIFT;
 	grant_table[*entryp].domid = dom;
 	/*
 	 * ensure that the above values reach global visibility
 	 * before permitting frame's access (done when we set flags)
 	 */
 	xen_rmb();
 	grant_table[*entryp].flags =
 	    GTF_permit_access | (ro ? GTF_readonly : 0);
 	mutex_exit(&grant_lock);
 	return 0;
+}
 static inline uint16_t
 xen_atomic_cmpxchg16(volatile uint16_t *ptr, uint16_t  val, uint16_t newval)
+{
 	unsigned long result;
 	__asm volatile(__LOCK_PREFIX
 	    "cmpxchgw %w1,%2"
 	    :"=a" (result)
 	    :"q"(newval), "m" (*ptr), "0" (val)
 	    :"memory");
 	return result;
+}
 void
 xengnt_revoke_access(grant_ref_t entry)
+{
 	uint16_t flags, nflags;
 	nflags = grant_table[entry].flags;
 	do {
 		if ((flags = nflags) & (GTF_reading|GTF_writing))
 			panic("xengnt_revoke_access: still in use");
 		nflags = xen_atomic_cmpxchg16(&grant_table[entry].flags,
 		    flags, 0);
 	} while (nflags != flags);
 	xengnt_free_entry(entry);
+}
 int
 xengnt_status(grant_ref_t entry)
+{
 	return (grant_table[entry].flags & (GTF_reading|GTF_writing));
+}
 #endif /* USE_GRANT_V2 */