 @@ -1,14 +1,14 @@
-/*      $NetBSD: xbd_xenbus.c,v 1.110 2020/04/14 13:02:40 jdolecek Exp $      */
+/*      $NetBSD: xbd_xenbus.c,v 1.111 2020/04/14 13:10:43 jdolecek Exp $      */
 /*
  * Copyright (c) 2006 Manuel Bouyer.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
 @@ -40,27 +40,27 @@
  * - initiate request: xbdread/write/open/ioctl/..
  * - depending on operation, it is handled directly by disk(9) subsystem or
  *   goes through physio(9) first.
  * - the request is ultimately processed by xbd_diskstart() that prepares the
  *   xbd requests, post them in the ring I/O queue, then signal the backend.
+ *
  * When a response is available in the queue, the backend signals the frontend
  * via its event channel. This triggers xbd_handler(), which will link back
  * the response to its request through the request ID, and mark the I/O as
  * completed.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: xbd_xenbus.c,v 1.110 2020/04/14 13:02:40 jdolecek Exp $");
+__KERNEL_RCSID(0, "$NetBSD: xbd_xenbus.c,v 1.111 2020/04/14 13:10:43 jdolecek Exp $");
 #include "opt_xen.h"
 #include <sys/param.h>
 #include <sys/buf.h>
 #include <sys/bufq.h>
 #include <sys/device.h>
 #include <sys/disk.h>
 #include <sys/disklabel.h>
 #include <sys/conf.h>
 #include <sys/fcntl.h>
 #include <sys/kernel.h>
 @@ -127,71 +127,76 @@ struct xbd_xenbus_softc {
 	struct intrhand *sc_ih; /* Interrupt handler for this instance. */
 	kmutex_t sc_lock;
 	kcondvar_t sc_cache_flush_cv;
 	kcondvar_t sc_req_cv;
 	kcondvar_t sc_detach_cv;
 	kcondvar_t sc_suspend_cv;
 	blkif_front_ring_t sc_ring;
 	grant_ref_t sc_ring_gntref;
 	struct xbd_req sc_reqs[XBD_RING_SIZE];
 	SLIST_HEAD(,xbd_req) sc_xbdreq_head; /* list of free requests */
 	vmem_addr_t sc_unalign_buffer;
 	bool sc_unalign_free;
 	int sc_backend_status; /* our status with backend */
 #define BLKIF_STATE_DISCONNECTED 0
 #define BLKIF_STATE_CONNECTED    1
 #define BLKIF_STATE_SUSPENDED    2
 	int sc_shutdown;
 #define BLKIF_SHUTDOWN_RUN    0 /* no shutdown */
 #define BLKIF_SHUTDOWN_REMOTE 1 /* backend-initiated shutdown in progress */
 #define BLKIF_SHUTDOWN_LOCAL  2 /* locally-initiated shutdown in progress */
 	uint64_t sc_sectors; /* number of sectors for this device */
 	u_long sc_secsize; /* sector size */
 	uint64_t sc_xbdsize; /* size of disk in DEV_BSIZE */
 	u_long sc_info; /* VDISK_* */
 	u_long sc_handle; /* from backend */
 	int sc_features;
 #define BLKIF_FEATURE_CACHE_FLUSH	0x1
 #define BLKIF_FEATURE_BARRIER		0x2
 #define BLKIF_FEATURE_PERSISTENT	0x4
 #define BLKIF_FEATURE_BITS		\
 	"\20\1CACHE-FLUSH\2BARRIER\3PERSISTENT"
 	struct evcnt sc_cnt_map_unalign;
 	struct evcnt sc_cnt_unalign_busy;
 	struct evcnt sc_cnt_queue_full;
 };
 #if 0
 /* too big to be on stack */
 static multicall_entry_t rq_mcl[XBD_RING_SIZE+1];
 static paddr_t rq_pages[XBD_RING_SIZE];
 #endif
 static int  xbd_xenbus_match(device_t, cfdata_t, void *);
 static void xbd_xenbus_attach(device_t, device_t, void *);
 static int  xbd_xenbus_detach(device_t, int);
 static bool xbd_xenbus_suspend(device_t, const pmf_qual_t *);
 static bool xbd_xenbus_resume(device_t, const pmf_qual_t *);
 static int  xbd_handler(void *);
 static int  xbd_diskstart(device_t, struct buf *);
 static void xbd_iosize(device_t, int *);
 static void xbd_backend_changed(void *, XenbusState);
 static void xbd_connect(struct xbd_xenbus_softc *);
-static int  xbd_map_align(struct xbd_req *);
+static int  xbd_map_align(struct xbd_xenbus_softc *, struct xbd_req *);
-static void xbd_unmap_align(struct xbd_req *);
+static void xbd_unmap_align(struct xbd_xenbus_softc *, struct xbd_req *, bool);
 static void xbdminphys(struct buf *);
 CFATTACH_DECL3_NEW(xbd, sizeof(struct xbd_xenbus_softc),
     xbd_xenbus_match, xbd_xenbus_attach, xbd_xenbus_detach, NULL, NULL, NULL,
     DVF_DETACH_SHUTDOWN);
 dev_type_open(xbdopen);
 dev_type_close(xbdclose);
 dev_type_read(xbdread);
 dev_type_write(xbdwrite);
 dev_type_ioctl(xbdioctl);
 dev_type_strategy(xbdstrategy);
 @@ -287,37 +292,48 @@ xbd_xenbus_attach(device_t parent, devic
 		    req_next);
+	}
 	sc->sc_backend_status = BLKIF_STATE_DISCONNECTED;
 	sc->sc_shutdown = BLKIF_SHUTDOWN_REMOTE;
 	ring = (void *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED);
 	if (ring == NULL)
 		panic("%s: can't alloc ring", device_xname(self));
 	sc->sc_ring.sring = ring;
 	evcnt_attach_dynamic(&sc->sc_cnt_map_unalign, EVCNT_TYPE_MISC,
 	    NULL, device_xname(self), "map unaligned");
 	evcnt_attach_dynamic(&sc->sc_cnt_unalign_busy, EVCNT_TYPE_MISC,
 	    NULL, device_xname(self), "map unaligned");
 	evcnt_attach_dynamic(&sc->sc_cnt_queue_full, EVCNT_TYPE_MISC,
 	    NULL, device_xname(self), "queue full");
 	for (i = 0; i < XBD_RING_SIZE; i++) {
 		if (bus_dmamap_create(sc->sc_xbusd->xbusd_dmat,
 		    XBD_MAX_XFER, BLKIF_MAX_SEGMENTS_PER_REQUEST,
 		    PAGE_SIZE, PAGE_SIZE, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
 		    &sc->sc_reqs[i].req_dmamap) != 0) {
 			aprint_error_dev(self, "can't alloc dma maps\n");
 			return;
+		}
+	}
 	if (uvm_km_kmem_alloc(kmem_va_arena,
 	    MAXPHYS, VM_SLEEP | VM_INSTANTFIT, &sc->sc_unalign_buffer) != 0) {
 		aprint_error_dev(self, "can't alloc align buffer\n");
 		return;
+	}
 	sc->sc_unalign_free = true;
 	/* resume shared structures and tell backend that we are ready */
 	if (xbd_xenbus_resume(self, PMF_Q_NONE) == false) {
 		uvm_km_free(kernel_map, (vaddr_t)ring, PAGE_SIZE,
 		    UVM_KMF_WIRED);
 		return;
+	}
 	if (!pmf_device_register(self, xbd_xenbus_suspend, xbd_xenbus_resume))
 		aprint_error_dev(self, "couldn't establish power handler\n");
+}
 static int
 @@ -397,29 +413,36 @@ xbd_xenbus_detach(device_t dev, int flag
 	xengnt_revoke_access(sc->sc_ring_gntref);
 	uvm_km_free(kernel_map, (vaddr_t)sc->sc_ring.sring,
 	    PAGE_SIZE, UVM_KMF_WIRED);
 	for (i = 0; i < XBD_RING_SIZE; i++) {
 		if (sc->sc_reqs[i].req_dmamap != NULL) {
 			bus_dmamap_destroy(sc->sc_xbusd->xbusd_dmat,
 			    sc->sc_reqs[i].req_dmamap);
 			sc->sc_reqs[i].req_dmamap = NULL;
+		}
+	}
 	if (sc->sc_unalign_buffer != 0) {
 		uvm_km_kmem_free(kmem_va_arena, sc->sc_unalign_buffer, MAXPHYS);
 		sc->sc_unalign_buffer = 0;
+	}
 	mutex_destroy(&sc->sc_lock);
 	evcnt_detach(&sc->sc_cnt_map_unalign);
 	evcnt_detach(&sc->sc_cnt_unalign_busy);
 	evcnt_detach(&sc->sc_cnt_queue_full);
 	pmf_device_deregister(dev);
 	return 0;
+}
 static bool
 xbd_xenbus_suspend(device_t dev, const pmf_qual_t *qual) {
 	struct xbd_xenbus_softc *sc;
 	sc = device_private(dev);
 @@ -730,27 +753,27 @@ again:
 			 */
 			KASSERT(xengnt_status(xbdreq->req_gntref[seg]) == 0);
 			xengnt_revoke_access(xbdreq->req_gntref[seg]);
+		}
 		bus_dmamap_unload(sc->sc_xbusd->xbusd_dmat, xbdreq->req_dmamap);
 		bp = xbdreq->req_bp;
 		KASSERT(bp != NULL && bp->b_data != NULL);
 		DPRINTF(("%s(%p): b_bcount = %ld\n", __func__,
 		    bp, (long)bp->b_bcount));
 		if (__predict_false(bp->b_data != xbdreq->req_data))
-			xbd_unmap_align(xbdreq);
+			xbd_unmap_align(sc, xbdreq, true);
 		xbdreq->req_bp = xbdreq->req_data = NULL;
 		/* b_resid was set in dk_start, only override on error */
 		if (rep->status != BLKIF_RSP_OKAY) {
 			bp->b_error = EIO;
 			bp->b_resid = bp->b_bcount;
+		}
 		dk_done(&sc->sc_dksc, bp);
 		SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, xbdreq, req_next);
+	}
 @@ -995,50 +1018,51 @@ xbd_diskstart(device_t self, struct buf
+	}
 	if (__predict_false(
 	    sc->sc_backend_status == BLKIF_STATE_SUSPENDED)) {
 		/* device is suspended, do not consume buffer */
 		DPRINTF(("%s: (xbd_diskstart) device suspended\n",
 		    sc->sc_dksc.sc_xname));
 		error = EAGAIN;
 		goto out;
+	}
 	xbdreq = SLIST_FIRST(&sc->sc_xbdreq_head);
 	if (__predict_false(xbdreq == NULL)) {
 		sc->sc_cnt_queue_full.ev_count++;
 		DPRINTF(("xbd_diskstart: no req\n"));
 		error = EAGAIN;
 		goto out;
+	}
 	xbdreq->req_bp = bp;
 	xbdreq->req_data = bp->b_data;
 	if (__predict_false((vaddr_t)bp->b_data & (XEN_BSIZE - 1))) {
 		/* Only can get here if this is physio() request */
 		KASSERT(bp->b_saveaddr != NULL);
 		sc->sc_cnt_map_unalign.ev_count++;
-		if (__predict_false(xbd_map_align(xbdreq) != 0)) {
+		if (__predict_false(xbd_map_align(sc, xbdreq) != 0)) {
 			DPRINTF(("xbd_diskstart: no align\n"));
 			error = EAGAIN;
 			goto out;
+		}
+	}
 	if (__predict_false(bus_dmamap_load(sc->sc_xbusd->xbusd_dmat,
 	    xbdreq->req_dmamap, xbdreq->req_data, bp->b_bcount, NULL,
 	    BUS_DMA_NOWAIT) != 0)) {
-		printf("%s: %s: xengnt_grant_access failed",
+		printf("%s: %s: bus_dmamap_load failed",
 		    device_xname(sc->sc_dksc.sc_dev), __func__);
 		error = EINVAL;
 		goto out;
+	}
 	/* We are now committed to the transfer */
 	SLIST_REMOVE_HEAD(&sc->sc_xbdreq_head, req_next);
 	req = RING_GET_REQUEST(&sc->sc_ring, sc->sc_ring.req_prod_pvt);
 	req->id = xbdreq->req_id;
 	req->operation =
 	    bp->b_flags & B_READ ? BLKIF_OP_READ : BLKIF_OP_WRITE;
 	req->sector_number = bp->b_rawblkno;
 	req->handle = sc->sc_handle;
 @@ -1061,57 +1085,60 @@ xbd_diskstart(device_t self, struct buf
 		    sc->sc_xbusd->xbusd_otherend_id,
 		    (ma & ~PAGE_MASK), (bp->b_flags & B_READ) == 0,
 		    &xbdreq->req_gntref[seg]))) {
 			printf("%s: %s: xengnt_grant_access failed",
 			    device_xname(sc->sc_dksc.sc_dev), __func__);
 			if (seg > 0) {
 				for (; --seg >= 0; ) {
 					xengnt_revoke_access(
 					    xbdreq->req_gntref[seg]);
+				}
+			}
 			bus_dmamap_unload(sc->sc_xbusd->xbusd_dmat,
 			    xbdreq->req_dmamap);
 			if (__predict_false(bp->b_data != xbdreq->req_data))
 				xbd_unmap_align(sc, xbdreq, false);
 			SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, xbdreq,
 			    req_next);
 			error = EFAULT;
 			goto out;
+		}
 		req->seg[seg].gref = xbdreq->req_gntref[seg];
+	}
 	req->nr_segments = seg;
 	sc->sc_ring.req_prod_pvt++;
 	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&sc->sc_ring, notify);
 	if (notify)
 		hypervisor_notify_via_evtchn(sc->sc_evtchn);
 out:
 	mutex_exit(&sc->sc_lock);
 	return error;
+}
 static int
-xbd_map_align(struct xbd_req *req)
+xbd_map_align(struct xbd_xenbus_softc *sc, struct xbd_req *req)
+{
-	int rc;
+	if (!sc->sc_unalign_free) {
 		sc->sc_cnt_unalign_busy.ev_count++;
 		return EAGAIN;
+	}
 	sc->sc_unalign_free = false;
-	rc = uvm_km_kmem_alloc(kmem_va_arena,
+	KASSERT(req->req_bp->b_bcount < MAXPHYS);
-	    req->req_bp->b_bcount, (VM_NOSLEEP | VM_INSTANTFIT),
+	req->req_data = (void *)sc->sc_unalign_buffer;
 	    (vmem_addr_t *)&req->req_data);
 	if (__predict_false(rc != 0))
 		return ENOMEM;
 	if ((req->req_bp->b_flags & B_READ) == 0)
 		memcpy(req->req_data, req->req_bp->b_data,
 		    req->req_bp->b_bcount);
 	return 0;
+}
 static void
-xbd_unmap_align(struct xbd_req *req)
+xbd_unmap_align(struct xbd_xenbus_softc *sc, struct xbd_req *req, bool sync)
+{
-	if (req->req_bp->b_flags & B_READ)
+	if (sync && req->req_bp->b_flags & B_READ)
 		memcpy(req->req_bp->b_data, req->req_data,
 		    req->req_bp->b_bcount);
-	uvm_km_kmem_free(kmem_va_arena, (vaddr_t)req->req_data, req->req_bp->b_bcount);
+	sc->sc_unalign_free = true;
+}