Thu Apr 16 09:51:40 2020 UTC ()
when checking for physio request check for B_PHYS in b_flags rather
than an internal field, so this works when the original buf is
wrapped by another one by e.g. dk(4)

fixes misfired assert when doing newfs on a wedge, reported by Manuel Bouyer


(jdolecek)
diff -r1.114 -r1.115 src/sys/arch/xen/xen/xbd_xenbus.c
cvs diff -r1.114 -r1.115 src/sys/arch/xen/xen/xbd_xenbus.c (switch to unified diff)

--- src/sys/arch/xen/xen/xbd_xenbus.c 2020/04/15 10:16:47 1.114
+++ src/sys/arch/xen/xen/xbd_xenbus.c 2020/04/16 09:51:40 1.115
@@ -1,1237 +1,1240 @@ @@ -1,1237 +1,1240 @@
1/* $NetBSD: xbd_xenbus.c,v 1.114 2020/04/15 10:16:47 jdolecek Exp $ */ 1/* $NetBSD: xbd_xenbus.c,v 1.115 2020/04/16 09:51:40 jdolecek Exp $ */
2 2
3/* 3/*
4 * Copyright (c) 2006 Manuel Bouyer. 4 * Copyright (c) 2006 Manuel Bouyer.
5 * 5 *
6 * Redistribution and use in source and binary forms, with or without 6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions 7 * modification, are permitted provided that the following conditions
8 * are met: 8 * are met:
9 * 1. Redistributions of source code must retain the above copyright 9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer. 10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright 11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the 12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution. 13 * documentation and/or other materials provided with the distribution.
14 * 14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 * 25 *
26 */ 26 */
27 27
28/* 28/*
29 * The file contains the xbd frontend code required for block-level 29 * The file contains the xbd frontend code required for block-level
30 * communications (similar to hard disks) between two Xen domains. 30 * communications (similar to hard disks) between two Xen domains.
31 * 31 *
32 * We are not supposed to receive solicitations spontaneously from backend. The 32 * We are not supposed to receive solicitations spontaneously from backend. The
33 * protocol is therefore fairly simple and uses only one ring to communicate 33 * protocol is therefore fairly simple and uses only one ring to communicate
34 * with backend: frontend posts requests to the ring then wait for their 34 * with backend: frontend posts requests to the ring then wait for their
35 * replies asynchronously. 35 * replies asynchronously.
36 * 36 *
37 * xbd follows NetBSD's disk(9) convention. At any time, a LWP can schedule 37 * xbd follows NetBSD's disk(9) convention. At any time, a LWP can schedule
38 * an operation request for the device (be it open(), read(), write(), ...). 38 * an operation request for the device (be it open(), read(), write(), ...).
39 * Calls are typically processed that way: 39 * Calls are typically processed that way:
40 * - initiate request: xbdread/write/open/ioctl/.. 40 * - initiate request: xbdread/write/open/ioctl/..
41 * - depending on operation, it is handled directly by disk(9) subsystem or 41 * - depending on operation, it is handled directly by disk(9) subsystem or
42 * goes through physio(9) first. 42 * goes through physio(9) first.
43 * - the request is ultimately processed by xbd_diskstart() that prepares the 43 * - the request is ultimately processed by xbd_diskstart() that prepares the
44 * xbd requests, post them in the ring I/O queue, then signal the backend. 44 * xbd requests, post them in the ring I/O queue, then signal the backend.
45 * 45 *
46 * When a response is available in the queue, the backend signals the frontend 46 * When a response is available in the queue, the backend signals the frontend
47 * via its event channel. This triggers xbd_handler(), which will link back 47 * via its event channel. This triggers xbd_handler(), which will link back
48 * the response to its request through the request ID, and mark the I/O as 48 * the response to its request through the request ID, and mark the I/O as
49 * completed. 49 * completed.
50 */ 50 */
51 51
52#include <sys/cdefs.h> 52#include <sys/cdefs.h>
53__KERNEL_RCSID(0, "$NetBSD: xbd_xenbus.c,v 1.114 2020/04/15 10:16:47 jdolecek Exp $"); 53__KERNEL_RCSID(0, "$NetBSD: xbd_xenbus.c,v 1.115 2020/04/16 09:51:40 jdolecek Exp $");
54 54
55#include "opt_xen.h" 55#include "opt_xen.h"
56 56
57 57
58#include <sys/param.h> 58#include <sys/param.h>
59#include <sys/buf.h> 59#include <sys/buf.h>
60#include <sys/bufq.h> 60#include <sys/bufq.h>
61#include <sys/device.h> 61#include <sys/device.h>
62#include <sys/disk.h> 62#include <sys/disk.h>
63#include <sys/disklabel.h> 63#include <sys/disklabel.h>
64#include <sys/conf.h> 64#include <sys/conf.h>
65#include <sys/fcntl.h> 65#include <sys/fcntl.h>
66#include <sys/kernel.h> 66#include <sys/kernel.h>
67#include <sys/proc.h> 67#include <sys/proc.h>
68#include <sys/systm.h> 68#include <sys/systm.h>
69#include <sys/stat.h> 69#include <sys/stat.h>
70#include <sys/vnode.h> 70#include <sys/vnode.h>
71#include <sys/mutex.h> 71#include <sys/mutex.h>
72 72
73#include <dev/dkvar.h> 73#include <dev/dkvar.h>
74 74
75#include <uvm/uvm.h> 75#include <uvm/uvm.h>
76 76
77#include <xen/hypervisor.h> 77#include <xen/hypervisor.h>
78#include <xen/evtchn.h> 78#include <xen/evtchn.h>
79#include <xen/granttables.h> 79#include <xen/granttables.h>
80#include <xen/include/public/io/blkif.h> 80#include <xen/include/public/io/blkif.h>
81#include <xen/include/public/io/protocols.h> 81#include <xen/include/public/io/protocols.h>
82 82
83#include <xen/xenbus.h> 83#include <xen/xenbus.h>
84#include "locators.h" 84#include "locators.h"
85 85
86#undef XBD_DEBUG 86#undef XBD_DEBUG
87#ifdef XBD_DEBUG 87#ifdef XBD_DEBUG
88#define DPRINTF(x) printf x; 88#define DPRINTF(x) printf x;
89#else 89#else
90#define DPRINTF(x) 90#define DPRINTF(x)
91#endif 91#endif
92 92
93#define GRANT_INVALID_REF -1 93#define GRANT_INVALID_REF -1
94 94
95#define XBD_RING_SIZE __CONST_RING_SIZE(blkif, PAGE_SIZE) 95#define XBD_RING_SIZE __CONST_RING_SIZE(blkif, PAGE_SIZE)
96#define XBD_MAX_XFER (PAGE_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST) 96#define XBD_MAX_XFER (PAGE_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST)
97#define XBD_MAX_CHUNK 32*1024 /* max I/O size we process in 1 req */ 97#define XBD_MAX_CHUNK 32*1024 /* max I/O size we process in 1 req */
98#define XBD_XFER_LIMIT (2*XBD_MAX_XFER) 98#define XBD_XFER_LIMIT (2*XBD_MAX_XFER)
99 99
100#define XEN_BSHIFT 9 /* log2(XEN_BSIZE) */ 100#define XEN_BSHIFT 9 /* log2(XEN_BSIZE) */
101#define XEN_BSIZE (1 << XEN_BSHIFT)  101#define XEN_BSIZE (1 << XEN_BSHIFT)
102 102
103CTASSERT((MAXPHYS <= 2*XBD_MAX_CHUNK)); 103CTASSERT((MAXPHYS <= 2*XBD_MAX_CHUNK));
104CTASSERT(XEN_BSIZE == DEV_BSIZE); 104CTASSERT(XEN_BSIZE == DEV_BSIZE);
105 105
106struct xbd_req { 106struct xbd_req {
107 SLIST_ENTRY(xbd_req) req_next; 107 SLIST_ENTRY(xbd_req) req_next;
108 uint16_t req_id; /* ID passed to backend */ 108 uint16_t req_id; /* ID passed to backend */
109 bus_dmamap_t req_dmamap; 109 bus_dmamap_t req_dmamap;
110 struct xbd_req *req_parent, *req_child; 110 struct xbd_req *req_parent, *req_child;
111 bool req_parent_done; 111 bool req_parent_done;
112 union { 112 union {
113 struct { 113 struct {
114 grant_ref_t req_gntref[XBD_XFER_LIMIT >> PAGE_SHIFT]; 114 grant_ref_t req_gntref[XBD_XFER_LIMIT >> PAGE_SHIFT];
115 struct buf *req_bp; /* buffer associated with this request */ 115 struct buf *req_bp; /* buffer associated with this request */
116 void *req_data; /* pointer to the data buffer */ 116 void *req_data; /* pointer to the data buffer */
117 } req_rw; 117 } req_rw;
118 struct { 118 struct {
119 int s_error; 119 int s_error;
120 int s_done; 120 int s_done;
121 } req_sync; 121 } req_sync;
122 } u; 122 } u;
123}; 123};
124#define req_gntref u.req_rw.req_gntref 124#define req_gntref u.req_rw.req_gntref
125#define req_bp u.req_rw.req_bp 125#define req_bp u.req_rw.req_bp
126#define req_data u.req_rw.req_data 126#define req_data u.req_rw.req_data
127#define req_sync u.req_sync 127#define req_sync u.req_sync
128 128
129struct xbd_xenbus_softc { 129struct xbd_xenbus_softc {
130 struct dk_softc sc_dksc; /* Must be first in this struct */ 130 struct dk_softc sc_dksc; /* Must be first in this struct */
131 struct xenbus_device *sc_xbusd; 131 struct xenbus_device *sc_xbusd;
132 unsigned int sc_evtchn; 132 unsigned int sc_evtchn;
133 133
134 struct intrhand *sc_ih; /* Interrupt handler for this instance. */ 134 struct intrhand *sc_ih; /* Interrupt handler for this instance. */
135 kmutex_t sc_lock; 135 kmutex_t sc_lock;
136 kcondvar_t sc_cache_flush_cv; 136 kcondvar_t sc_cache_flush_cv;
137 kcondvar_t sc_req_cv; 137 kcondvar_t sc_req_cv;
138 kcondvar_t sc_detach_cv; 138 kcondvar_t sc_detach_cv;
139 kcondvar_t sc_suspend_cv; 139 kcondvar_t sc_suspend_cv;
140 140
141 blkif_front_ring_t sc_ring; 141 blkif_front_ring_t sc_ring;
142 grant_ref_t sc_ring_gntref; 142 grant_ref_t sc_ring_gntref;
143 143
144 struct xbd_req sc_reqs[XBD_RING_SIZE]; 144 struct xbd_req sc_reqs[XBD_RING_SIZE];
145 SLIST_HEAD(,xbd_req) sc_xbdreq_head; /* list of free requests */ 145 SLIST_HEAD(,xbd_req) sc_xbdreq_head; /* list of free requests */
146 146
147 vmem_addr_t sc_unalign_buffer; 147 vmem_addr_t sc_unalign_buffer;
148 struct xbd_req *sc_unalign_used; 148 struct xbd_req *sc_unalign_used;
149 149
150 int sc_backend_status; /* our status with backend */ 150 int sc_backend_status; /* our status with backend */
151#define BLKIF_STATE_DISCONNECTED 0 151#define BLKIF_STATE_DISCONNECTED 0
152#define BLKIF_STATE_CONNECTED 1 152#define BLKIF_STATE_CONNECTED 1
153#define BLKIF_STATE_SUSPENDED 2 153#define BLKIF_STATE_SUSPENDED 2
154 154
155 int sc_shutdown; 155 int sc_shutdown;
156#define BLKIF_SHUTDOWN_RUN 0 /* no shutdown */ 156#define BLKIF_SHUTDOWN_RUN 0 /* no shutdown */
157#define BLKIF_SHUTDOWN_REMOTE 1 /* backend-initiated shutdown in progress */ 157#define BLKIF_SHUTDOWN_REMOTE 1 /* backend-initiated shutdown in progress */
158#define BLKIF_SHUTDOWN_LOCAL 2 /* locally-initiated shutdown in progress */ 158#define BLKIF_SHUTDOWN_LOCAL 2 /* locally-initiated shutdown in progress */
159 159
160 uint64_t sc_sectors; /* number of sectors for this device */ 160 uint64_t sc_sectors; /* number of sectors for this device */
161 u_long sc_secsize; /* sector size */ 161 u_long sc_secsize; /* sector size */
162 uint64_t sc_xbdsize; /* size of disk in DEV_BSIZE */ 162 uint64_t sc_xbdsize; /* size of disk in DEV_BSIZE */
163 u_long sc_info; /* VDISK_* */ 163 u_long sc_info; /* VDISK_* */
164 u_long sc_handle; /* from backend */ 164 u_long sc_handle; /* from backend */
165 int sc_features; 165 int sc_features;
166#define BLKIF_FEATURE_CACHE_FLUSH 0x1 166#define BLKIF_FEATURE_CACHE_FLUSH 0x1
167#define BLKIF_FEATURE_BARRIER 0x2 167#define BLKIF_FEATURE_BARRIER 0x2
168#define BLKIF_FEATURE_PERSISTENT 0x4 168#define BLKIF_FEATURE_PERSISTENT 0x4
169#define BLKIF_FEATURE_BITS \ 169#define BLKIF_FEATURE_BITS \
170 "\20\1CACHE-FLUSH\2BARRIER\3PERSISTENT" 170 "\20\1CACHE-FLUSH\2BARRIER\3PERSISTENT"
171 struct evcnt sc_cnt_map_unalign; 171 struct evcnt sc_cnt_map_unalign;
172 struct evcnt sc_cnt_unalign_busy; 172 struct evcnt sc_cnt_unalign_busy;
173 struct evcnt sc_cnt_queue_full; 173 struct evcnt sc_cnt_queue_full;
174}; 174};
175 175
176#if 0 176#if 0
177/* too big to be on stack */ 177/* too big to be on stack */
178static multicall_entry_t rq_mcl[XBD_RING_SIZE+1]; 178static multicall_entry_t rq_mcl[XBD_RING_SIZE+1];
179static paddr_t rq_pages[XBD_RING_SIZE]; 179static paddr_t rq_pages[XBD_RING_SIZE];
180#endif 180#endif
181 181
182static int xbd_xenbus_match(device_t, cfdata_t, void *); 182static int xbd_xenbus_match(device_t, cfdata_t, void *);
183static void xbd_xenbus_attach(device_t, device_t, void *); 183static void xbd_xenbus_attach(device_t, device_t, void *);
184static int xbd_xenbus_detach(device_t, int); 184static int xbd_xenbus_detach(device_t, int);
185 185
186static bool xbd_xenbus_suspend(device_t, const pmf_qual_t *); 186static bool xbd_xenbus_suspend(device_t, const pmf_qual_t *);
187static bool xbd_xenbus_resume(device_t, const pmf_qual_t *); 187static bool xbd_xenbus_resume(device_t, const pmf_qual_t *);
188 188
189static int xbd_handler(void *); 189static int xbd_handler(void *);
190static int xbd_diskstart(device_t, struct buf *); 190static int xbd_diskstart(device_t, struct buf *);
191static void xbd_iosize(device_t, int *); 191static void xbd_iosize(device_t, int *);
192static void xbd_backend_changed(void *, XenbusState); 192static void xbd_backend_changed(void *, XenbusState);
193static void xbd_connect(struct xbd_xenbus_softc *); 193static void xbd_connect(struct xbd_xenbus_softc *);
194 194
195static void xbd_diskstart_submit(struct xbd_xenbus_softc *, int, 195static void xbd_diskstart_submit(struct xbd_xenbus_softc *, int,
196 struct buf *bp, int, bus_dmamap_t, grant_ref_t *); 196 struct buf *bp, int, bus_dmamap_t, grant_ref_t *);
197static int xbd_map_align(struct xbd_xenbus_softc *, struct xbd_req *); 197static int xbd_map_align(struct xbd_xenbus_softc *, struct xbd_req *);
198static void xbd_unmap_align(struct xbd_xenbus_softc *, struct xbd_req *, bool); 198static void xbd_unmap_align(struct xbd_xenbus_softc *, struct xbd_req *, bool);
199 199
200static void xbdminphys(struct buf *); 200static void xbdminphys(struct buf *);
201 201
202CFATTACH_DECL3_NEW(xbd, sizeof(struct xbd_xenbus_softc), 202CFATTACH_DECL3_NEW(xbd, sizeof(struct xbd_xenbus_softc),
203 xbd_xenbus_match, xbd_xenbus_attach, xbd_xenbus_detach, NULL, NULL, NULL, 203 xbd_xenbus_match, xbd_xenbus_attach, xbd_xenbus_detach, NULL, NULL, NULL,
204 DVF_DETACH_SHUTDOWN); 204 DVF_DETACH_SHUTDOWN);
205 205
206dev_type_open(xbdopen); 206dev_type_open(xbdopen);
207dev_type_close(xbdclose); 207dev_type_close(xbdclose);
208dev_type_read(xbdread); 208dev_type_read(xbdread);
209dev_type_write(xbdwrite); 209dev_type_write(xbdwrite);
210dev_type_ioctl(xbdioctl); 210dev_type_ioctl(xbdioctl);
211dev_type_strategy(xbdstrategy); 211dev_type_strategy(xbdstrategy);
212dev_type_dump(xbddump); 212dev_type_dump(xbddump);
213dev_type_size(xbdsize); 213dev_type_size(xbdsize);
214 214
215const struct bdevsw xbd_bdevsw = { 215const struct bdevsw xbd_bdevsw = {
216 .d_open = xbdopen, 216 .d_open = xbdopen,
217 .d_close = xbdclose, 217 .d_close = xbdclose,
218 .d_strategy = xbdstrategy, 218 .d_strategy = xbdstrategy,
219 .d_ioctl = xbdioctl, 219 .d_ioctl = xbdioctl,
220 .d_dump = xbddump, 220 .d_dump = xbddump,
221 .d_psize = xbdsize, 221 .d_psize = xbdsize,
222 .d_discard = nodiscard, 222 .d_discard = nodiscard,
223 .d_flag = D_DISK | D_MPSAFE 223 .d_flag = D_DISK | D_MPSAFE
224}; 224};
225 225
226const struct cdevsw xbd_cdevsw = { 226const struct cdevsw xbd_cdevsw = {
227 .d_open = xbdopen, 227 .d_open = xbdopen,
228 .d_close = xbdclose, 228 .d_close = xbdclose,
229 .d_read = xbdread, 229 .d_read = xbdread,
230 .d_write = xbdwrite, 230 .d_write = xbdwrite,
231 .d_ioctl = xbdioctl, 231 .d_ioctl = xbdioctl,
232 .d_stop = nostop, 232 .d_stop = nostop,
233 .d_tty = notty, 233 .d_tty = notty,
234 .d_poll = nopoll, 234 .d_poll = nopoll,
235 .d_mmap = nommap, 235 .d_mmap = nommap,
236 .d_kqfilter = nokqfilter, 236 .d_kqfilter = nokqfilter,
237 .d_discard = nodiscard, 237 .d_discard = nodiscard,
238 .d_flag = D_DISK | D_MPSAFE 238 .d_flag = D_DISK | D_MPSAFE
239}; 239};
240 240
241extern struct cfdriver xbd_cd; 241extern struct cfdriver xbd_cd;
242 242
243static struct dkdriver xbddkdriver = { 243static struct dkdriver xbddkdriver = {
244 .d_strategy = xbdstrategy, 244 .d_strategy = xbdstrategy,
245 .d_minphys = xbdminphys, 245 .d_minphys = xbdminphys,
246 .d_open = xbdopen, 246 .d_open = xbdopen,
247 .d_close = xbdclose, 247 .d_close = xbdclose,
248 .d_diskstart = xbd_diskstart, 248 .d_diskstart = xbd_diskstart,
249 .d_iosize = xbd_iosize, 249 .d_iosize = xbd_iosize,
250}; 250};
251 251
252static int 252static int
253xbd_xenbus_match(device_t parent, cfdata_t match, void *aux) 253xbd_xenbus_match(device_t parent, cfdata_t match, void *aux)
254{ 254{
255 struct xenbusdev_attach_args *xa = aux; 255 struct xenbusdev_attach_args *xa = aux;
256 256
257 if (strcmp(xa->xa_type, "vbd") != 0) 257 if (strcmp(xa->xa_type, "vbd") != 0)
258 return 0; 258 return 0;
259 259
260 if (match->cf_loc[XENBUSCF_ID] != XENBUSCF_ID_DEFAULT && 260 if (match->cf_loc[XENBUSCF_ID] != XENBUSCF_ID_DEFAULT &&
261 match->cf_loc[XENBUSCF_ID] != xa->xa_id) 261 match->cf_loc[XENBUSCF_ID] != xa->xa_id)
262 return 0; 262 return 0;
263 263
264 return 1; 264 return 1;
265} 265}
266 266
267static void 267static void
268xbd_xenbus_attach(device_t parent, device_t self, void *aux) 268xbd_xenbus_attach(device_t parent, device_t self, void *aux)
269{ 269{
270 struct xbd_xenbus_softc *sc = device_private(self); 270 struct xbd_xenbus_softc *sc = device_private(self);
271 struct xenbusdev_attach_args *xa = aux; 271 struct xenbusdev_attach_args *xa = aux;
272 blkif_sring_t *ring; 272 blkif_sring_t *ring;
273 RING_IDX i; 273 RING_IDX i;
274#ifdef XBD_DEBUG 274#ifdef XBD_DEBUG
275 char **dir, *val; 275 char **dir, *val;
276 int dir_n = 0; 276 int dir_n = 0;
277 char id_str[20]; 277 char id_str[20];
278 int err; 278 int err;
279#endif 279#endif
280 280
281 config_pending_incr(self); 281 config_pending_incr(self);
282 aprint_normal(": Xen Virtual Block Device Interface\n"); 282 aprint_normal(": Xen Virtual Block Device Interface\n");
283 283
284 dk_init(&sc->sc_dksc, self, DKTYPE_ESDI); 284 dk_init(&sc->sc_dksc, self, DKTYPE_ESDI);
285 disk_init(&sc->sc_dksc.sc_dkdev, device_xname(self), &xbddkdriver); 285 disk_init(&sc->sc_dksc.sc_dkdev, device_xname(self), &xbddkdriver);
286 286
287 sc->sc_xbusd = xa->xa_xbusd; 287 sc->sc_xbusd = xa->xa_xbusd;
288 sc->sc_xbusd->xbusd_otherend_changed = xbd_backend_changed; 288 sc->sc_xbusd->xbusd_otherend_changed = xbd_backend_changed;
289 289
290 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_BIO); 290 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_BIO);
291 cv_init(&sc->sc_cache_flush_cv, "xbdsync"); 291 cv_init(&sc->sc_cache_flush_cv, "xbdsync");
292 cv_init(&sc->sc_req_cv, "xbdreq"); 292 cv_init(&sc->sc_req_cv, "xbdreq");
293 cv_init(&sc->sc_detach_cv, "xbddetach"); 293 cv_init(&sc->sc_detach_cv, "xbddetach");
294 cv_init(&sc->sc_suspend_cv, "xbdsuspend"); 294 cv_init(&sc->sc_suspend_cv, "xbdsuspend");
295 295
296 /* initialize free requests list */ 296 /* initialize free requests list */
297 SLIST_INIT(&sc->sc_xbdreq_head); 297 SLIST_INIT(&sc->sc_xbdreq_head);
298 for (i = 0; i < XBD_RING_SIZE; i++) { 298 for (i = 0; i < XBD_RING_SIZE; i++) {
299 sc->sc_reqs[i].req_id = i; 299 sc->sc_reqs[i].req_id = i;
300 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, &sc->sc_reqs[i], 300 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, &sc->sc_reqs[i],
301 req_next); 301 req_next);
302 } 302 }
303 303
304 sc->sc_backend_status = BLKIF_STATE_DISCONNECTED; 304 sc->sc_backend_status = BLKIF_STATE_DISCONNECTED;
305 sc->sc_shutdown = BLKIF_SHUTDOWN_REMOTE; 305 sc->sc_shutdown = BLKIF_SHUTDOWN_REMOTE;
306 306
307 ring = (void *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED); 307 ring = (void *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED);
308 if (ring == NULL) 308 if (ring == NULL)
309 panic("%s: can't alloc ring", device_xname(self)); 309 panic("%s: can't alloc ring", device_xname(self));
310 sc->sc_ring.sring = ring; 310 sc->sc_ring.sring = ring;
311 311
312 evcnt_attach_dynamic(&sc->sc_cnt_map_unalign, EVCNT_TYPE_MISC, 312 evcnt_attach_dynamic(&sc->sc_cnt_map_unalign, EVCNT_TYPE_MISC,
313 NULL, device_xname(self), "map unaligned"); 313 NULL, device_xname(self), "map unaligned");
314 evcnt_attach_dynamic(&sc->sc_cnt_unalign_busy, EVCNT_TYPE_MISC, 314 evcnt_attach_dynamic(&sc->sc_cnt_unalign_busy, EVCNT_TYPE_MISC,
315 NULL, device_xname(self), "map unaligned"); 315 NULL, device_xname(self), "map unaligned");
316 evcnt_attach_dynamic(&sc->sc_cnt_queue_full, EVCNT_TYPE_MISC, 316 evcnt_attach_dynamic(&sc->sc_cnt_queue_full, EVCNT_TYPE_MISC,
317 NULL, device_xname(self), "queue full"); 317 NULL, device_xname(self), "queue full");
318 318
319 for (i = 0; i < XBD_RING_SIZE; i++) { 319 for (i = 0; i < XBD_RING_SIZE; i++) {
320 if (bus_dmamap_create(sc->sc_xbusd->xbusd_dmat, 320 if (bus_dmamap_create(sc->sc_xbusd->xbusd_dmat,
321 MAXPHYS, XBD_XFER_LIMIT >> PAGE_SHIFT, 321 MAXPHYS, XBD_XFER_LIMIT >> PAGE_SHIFT,
322 PAGE_SIZE, PAGE_SIZE, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 322 PAGE_SIZE, PAGE_SIZE, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
323 &sc->sc_reqs[i].req_dmamap) != 0) { 323 &sc->sc_reqs[i].req_dmamap) != 0) {
324 aprint_error_dev(self, "can't alloc dma maps\n"); 324 aprint_error_dev(self, "can't alloc dma maps\n");
325 return; 325 return;
326 } 326 }
327 } 327 }
328 328
329 if (uvm_km_kmem_alloc(kmem_va_arena, 329 if (uvm_km_kmem_alloc(kmem_va_arena,
330 MAXPHYS, VM_SLEEP | VM_INSTANTFIT, &sc->sc_unalign_buffer) != 0) { 330 MAXPHYS, VM_SLEEP | VM_INSTANTFIT, &sc->sc_unalign_buffer) != 0) {
331 aprint_error_dev(self, "can't alloc align buffer\n"); 331 aprint_error_dev(self, "can't alloc align buffer\n");
332 return; 332 return;
333 } 333 }
334 334
335 /* resume shared structures and tell backend that we are ready */ 335 /* resume shared structures and tell backend that we are ready */
336 if (xbd_xenbus_resume(self, PMF_Q_NONE) == false) { 336 if (xbd_xenbus_resume(self, PMF_Q_NONE) == false) {
337 uvm_km_free(kernel_map, (vaddr_t)ring, PAGE_SIZE, 337 uvm_km_free(kernel_map, (vaddr_t)ring, PAGE_SIZE,
338 UVM_KMF_WIRED); 338 UVM_KMF_WIRED);
339 return; 339 return;
340 } 340 }
341 341
342 if (!pmf_device_register(self, xbd_xenbus_suspend, xbd_xenbus_resume)) 342 if (!pmf_device_register(self, xbd_xenbus_suspend, xbd_xenbus_resume))
343 aprint_error_dev(self, "couldn't establish power handler\n"); 343 aprint_error_dev(self, "couldn't establish power handler\n");
344 344
345} 345}
346 346
347static int 347static int
348xbd_xenbus_detach(device_t dev, int flags) 348xbd_xenbus_detach(device_t dev, int flags)
349{ 349{
350 struct xbd_xenbus_softc *sc = device_private(dev); 350 struct xbd_xenbus_softc *sc = device_private(dev);
351 int bmaj, cmaj, i, mn, rc; 351 int bmaj, cmaj, i, mn, rc;
352 352
353 DPRINTF(("%s: xbd_detach\n", device_xname(dev))); 353 DPRINTF(("%s: xbd_detach\n", device_xname(dev)));
354 354
355 rc = disk_begindetach(&sc->sc_dksc.sc_dkdev, NULL, dev, flags); 355 rc = disk_begindetach(&sc->sc_dksc.sc_dkdev, NULL, dev, flags);
356 if (rc != 0) 356 if (rc != 0)
357 return rc; 357 return rc;
358 358
359 mutex_enter(&sc->sc_lock); 359 mutex_enter(&sc->sc_lock);
360 if (sc->sc_shutdown == BLKIF_SHUTDOWN_RUN) { 360 if (sc->sc_shutdown == BLKIF_SHUTDOWN_RUN) {
361 sc->sc_shutdown = BLKIF_SHUTDOWN_LOCAL; 361 sc->sc_shutdown = BLKIF_SHUTDOWN_LOCAL;
362 362
363 /* wait for requests to complete */ 363 /* wait for requests to complete */
364 while (sc->sc_backend_status == BLKIF_STATE_CONNECTED && 364 while (sc->sc_backend_status == BLKIF_STATE_CONNECTED &&
365 disk_isbusy(&sc->sc_dksc.sc_dkdev)) { 365 disk_isbusy(&sc->sc_dksc.sc_dkdev)) {
366 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2); 366 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2);
367 } 367 }
368 mutex_exit(&sc->sc_lock); 368 mutex_exit(&sc->sc_lock);
369 369
370 /* Trigger state transition with backend */ 370 /* Trigger state transition with backend */
371 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateClosing); 371 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateClosing);
372 372
373 mutex_enter(&sc->sc_lock); 373 mutex_enter(&sc->sc_lock);
374 } 374 }
375 if ((flags & DETACH_FORCE) == 0) { 375 if ((flags & DETACH_FORCE) == 0) {
376 /* xbd_xenbus_detach already in progress */ 376 /* xbd_xenbus_detach already in progress */
377 cv_broadcast(&sc->sc_detach_cv); 377 cv_broadcast(&sc->sc_detach_cv);
378 mutex_exit(&sc->sc_lock); 378 mutex_exit(&sc->sc_lock);
379 return EALREADY; 379 return EALREADY;
380 } 380 }
381 mutex_exit(&sc->sc_lock); 381 mutex_exit(&sc->sc_lock);
382 while (xenbus_read_driver_state(sc->sc_xbusd->xbusd_otherend) 382 while (xenbus_read_driver_state(sc->sc_xbusd->xbusd_otherend)
383 != XenbusStateClosed) { 383 != XenbusStateClosed) {
384 mutex_enter(&sc->sc_lock); 384 mutex_enter(&sc->sc_lock);
385 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2); 385 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2);
386 mutex_exit(&sc->sc_lock); 386 mutex_exit(&sc->sc_lock);
387 } 387 }
388 388
389 /* locate the major number */ 389 /* locate the major number */
390 bmaj = bdevsw_lookup_major(&xbd_bdevsw); 390 bmaj = bdevsw_lookup_major(&xbd_bdevsw);
391 cmaj = cdevsw_lookup_major(&xbd_cdevsw); 391 cmaj = cdevsw_lookup_major(&xbd_cdevsw);
392 392
393 /* Nuke the vnodes for any open instances. */ 393 /* Nuke the vnodes for any open instances. */
394 for (i = 0; i < MAXPARTITIONS; i++) { 394 for (i = 0; i < MAXPARTITIONS; i++) {
395 mn = DISKMINOR(device_unit(dev), i); 395 mn = DISKMINOR(device_unit(dev), i);
396 vdevgone(bmaj, mn, mn, VBLK); 396 vdevgone(bmaj, mn, mn, VBLK);
397 vdevgone(cmaj, mn, mn, VCHR); 397 vdevgone(cmaj, mn, mn, VCHR);
398 } 398 }
399 399
400 if (sc->sc_backend_status == BLKIF_STATE_CONNECTED) { 400 if (sc->sc_backend_status == BLKIF_STATE_CONNECTED) {
401 /* Delete all of our wedges. */ 401 /* Delete all of our wedges. */
402 dkwedge_delall(&sc->sc_dksc.sc_dkdev); 402 dkwedge_delall(&sc->sc_dksc.sc_dkdev);
403 403
404 /* Kill off any queued buffers. */ 404 /* Kill off any queued buffers. */
405 dk_drain(&sc->sc_dksc); 405 dk_drain(&sc->sc_dksc);
406 bufq_free(sc->sc_dksc.sc_bufq); 406 bufq_free(sc->sc_dksc.sc_bufq);
407 407
408 /* detach disk */ 408 /* detach disk */
409 disk_detach(&sc->sc_dksc.sc_dkdev); 409 disk_detach(&sc->sc_dksc.sc_dkdev);
410 disk_destroy(&sc->sc_dksc.sc_dkdev); 410 disk_destroy(&sc->sc_dksc.sc_dkdev);
411 dk_detach(&sc->sc_dksc); 411 dk_detach(&sc->sc_dksc);
412 } 412 }
413 413
414 hypervisor_mask_event(sc->sc_evtchn); 414 hypervisor_mask_event(sc->sc_evtchn);
415 xen_intr_disestablish(sc->sc_ih); 415 xen_intr_disestablish(sc->sc_ih);
416 416
417 mutex_enter(&sc->sc_lock); 417 mutex_enter(&sc->sc_lock);
418 while (xengnt_status(sc->sc_ring_gntref)) 418 while (xengnt_status(sc->sc_ring_gntref))
419 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2); 419 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2);
420 mutex_exit(&sc->sc_lock); 420 mutex_exit(&sc->sc_lock);
421 421
422 xengnt_revoke_access(sc->sc_ring_gntref); 422 xengnt_revoke_access(sc->sc_ring_gntref);
423 uvm_km_free(kernel_map, (vaddr_t)sc->sc_ring.sring, 423 uvm_km_free(kernel_map, (vaddr_t)sc->sc_ring.sring,
424 PAGE_SIZE, UVM_KMF_WIRED); 424 PAGE_SIZE, UVM_KMF_WIRED);
425 425
426 for (i = 0; i < XBD_RING_SIZE; i++) { 426 for (i = 0; i < XBD_RING_SIZE; i++) {
427 if (sc->sc_reqs[i].req_dmamap != NULL) { 427 if (sc->sc_reqs[i].req_dmamap != NULL) {
428 bus_dmamap_destroy(sc->sc_xbusd->xbusd_dmat, 428 bus_dmamap_destroy(sc->sc_xbusd->xbusd_dmat,
429 sc->sc_reqs[i].req_dmamap); 429 sc->sc_reqs[i].req_dmamap);
430 sc->sc_reqs[i].req_dmamap = NULL; 430 sc->sc_reqs[i].req_dmamap = NULL;
431 } 431 }
432 } 432 }
433 433
434 if (sc->sc_unalign_buffer != 0) { 434 if (sc->sc_unalign_buffer != 0) {
435 uvm_km_kmem_free(kmem_va_arena, sc->sc_unalign_buffer, MAXPHYS); 435 uvm_km_kmem_free(kmem_va_arena, sc->sc_unalign_buffer, MAXPHYS);
436 sc->sc_unalign_buffer = 0; 436 sc->sc_unalign_buffer = 0;
437 } 437 }
438 438
439 mutex_destroy(&sc->sc_lock); 439 mutex_destroy(&sc->sc_lock);
440 440
441 evcnt_detach(&sc->sc_cnt_map_unalign); 441 evcnt_detach(&sc->sc_cnt_map_unalign);
442 evcnt_detach(&sc->sc_cnt_unalign_busy); 442 evcnt_detach(&sc->sc_cnt_unalign_busy);
443 evcnt_detach(&sc->sc_cnt_queue_full); 443 evcnt_detach(&sc->sc_cnt_queue_full);
444 444
445 pmf_device_deregister(dev); 445 pmf_device_deregister(dev);
446 446
447 return 0; 447 return 0;
448} 448}
449 449
450static bool 450static bool
451xbd_xenbus_suspend(device_t dev, const pmf_qual_t *qual) { 451xbd_xenbus_suspend(device_t dev, const pmf_qual_t *qual) {
452 452
453 struct xbd_xenbus_softc *sc; 453 struct xbd_xenbus_softc *sc;
454 454
455 sc = device_private(dev); 455 sc = device_private(dev);
456 456
457 mutex_enter(&sc->sc_lock); 457 mutex_enter(&sc->sc_lock);
458 /* wait for requests to complete, then suspend device */ 458 /* wait for requests to complete, then suspend device */
459 while (sc->sc_backend_status == BLKIF_STATE_CONNECTED && 459 while (sc->sc_backend_status == BLKIF_STATE_CONNECTED &&
460 disk_isbusy(&sc->sc_dksc.sc_dkdev)) { 460 disk_isbusy(&sc->sc_dksc.sc_dkdev)) {
461 cv_timedwait(&sc->sc_suspend_cv, &sc->sc_lock, hz/2); 461 cv_timedwait(&sc->sc_suspend_cv, &sc->sc_lock, hz/2);
462 } 462 }
463 463
464 hypervisor_mask_event(sc->sc_evtchn); 464 hypervisor_mask_event(sc->sc_evtchn);
465 sc->sc_backend_status = BLKIF_STATE_SUSPENDED; 465 sc->sc_backend_status = BLKIF_STATE_SUSPENDED;
466 xen_intr_disestablish(sc->sc_ih); 466 xen_intr_disestablish(sc->sc_ih);
467 467
468 mutex_exit(&sc->sc_lock); 468 mutex_exit(&sc->sc_lock);
469 469
470 xenbus_device_suspend(sc->sc_xbusd); 470 xenbus_device_suspend(sc->sc_xbusd);
471 aprint_verbose_dev(dev, "removed event channel %d\n", sc->sc_evtchn); 471 aprint_verbose_dev(dev, "removed event channel %d\n", sc->sc_evtchn);
472 472
473 return true; 473 return true;
474} 474}
475 475
476static bool 476static bool
477xbd_xenbus_resume(device_t dev, const pmf_qual_t *qual) 477xbd_xenbus_resume(device_t dev, const pmf_qual_t *qual)
478{ 478{
479 struct xbd_xenbus_softc *sc; 479 struct xbd_xenbus_softc *sc;
480 struct xenbus_transaction *xbt; 480 struct xenbus_transaction *xbt;
481 int error; 481 int error;
482 blkif_sring_t *ring; 482 blkif_sring_t *ring;
483 paddr_t ma; 483 paddr_t ma;
484 const char *errmsg; 484 const char *errmsg;
485 485
486 sc = device_private(dev); 486 sc = device_private(dev);
487 487
488 if (sc->sc_backend_status == BLKIF_STATE_SUSPENDED) { 488 if (sc->sc_backend_status == BLKIF_STATE_SUSPENDED) {
489 /* 489 /*
490 * Device was suspended, so ensure that access associated to 490 * Device was suspended, so ensure that access associated to
491 * the block I/O ring is revoked. 491 * the block I/O ring is revoked.
492 */ 492 */
493 xengnt_revoke_access(sc->sc_ring_gntref); 493 xengnt_revoke_access(sc->sc_ring_gntref);
494 } 494 }
495 sc->sc_ring_gntref = GRANT_INVALID_REF; 495 sc->sc_ring_gntref = GRANT_INVALID_REF;
496 496
497 /* Initialize ring */ 497 /* Initialize ring */
498 ring = sc->sc_ring.sring; 498 ring = sc->sc_ring.sring;
499 memset(ring, 0, PAGE_SIZE); 499 memset(ring, 0, PAGE_SIZE);
500 SHARED_RING_INIT(ring); 500 SHARED_RING_INIT(ring);
501 FRONT_RING_INIT(&sc->sc_ring, ring, PAGE_SIZE); 501 FRONT_RING_INIT(&sc->sc_ring, ring, PAGE_SIZE);
502 502
503 /* 503 /*
504 * get MA address of the ring, and use it to set up the grant entry 504 * get MA address of the ring, and use it to set up the grant entry
505 * for the block device 505 * for the block device
506 */ 506 */
507 (void)pmap_extract_ma(pmap_kernel(), (vaddr_t)ring, &ma); 507 (void)pmap_extract_ma(pmap_kernel(), (vaddr_t)ring, &ma);
508 error = xenbus_grant_ring(sc->sc_xbusd, ma, &sc->sc_ring_gntref); 508 error = xenbus_grant_ring(sc->sc_xbusd, ma, &sc->sc_ring_gntref);
509 if (error) 509 if (error)
510 goto abort_resume; 510 goto abort_resume;
511 511
512 error = xenbus_alloc_evtchn(sc->sc_xbusd, &sc->sc_evtchn); 512 error = xenbus_alloc_evtchn(sc->sc_xbusd, &sc->sc_evtchn);
513 if (error) 513 if (error)
514 goto abort_resume; 514 goto abort_resume;
515 515
516 aprint_verbose_dev(dev, "using event channel %d\n", 516 aprint_verbose_dev(dev, "using event channel %d\n",
517 sc->sc_evtchn); 517 sc->sc_evtchn);
518 sc->sc_ih = xen_intr_establish_xname(-1, &xen_pic, sc->sc_evtchn, 518 sc->sc_ih = xen_intr_establish_xname(-1, &xen_pic, sc->sc_evtchn,
519 IST_LEVEL, IPL_BIO, &xbd_handler, sc, true, device_xname(dev)); 519 IST_LEVEL, IPL_BIO, &xbd_handler, sc, true, device_xname(dev));
520 KASSERT(sc->sc_ih != NULL); 520 KASSERT(sc->sc_ih != NULL);
521 521
522again: 522again:
523 xbt = xenbus_transaction_start(); 523 xbt = xenbus_transaction_start();
524 if (xbt == NULL) 524 if (xbt == NULL)
525 return false; 525 return false;
526 error = xenbus_printf(xbt, sc->sc_xbusd->xbusd_path, 526 error = xenbus_printf(xbt, sc->sc_xbusd->xbusd_path,
527 "ring-ref","%u", sc->sc_ring_gntref); 527 "ring-ref","%u", sc->sc_ring_gntref);
528 if (error) { 528 if (error) {
529 errmsg = "writing ring-ref"; 529 errmsg = "writing ring-ref";
530 goto abort_transaction; 530 goto abort_transaction;
531 } 531 }
532 error = xenbus_printf(xbt, sc->sc_xbusd->xbusd_path, 532 error = xenbus_printf(xbt, sc->sc_xbusd->xbusd_path,
533 "event-channel", "%u", sc->sc_evtchn); 533 "event-channel", "%u", sc->sc_evtchn);
534 if (error) { 534 if (error) {
535 errmsg = "writing event channel"; 535 errmsg = "writing event channel";
536 goto abort_transaction; 536 goto abort_transaction;
537 } 537 }
538 error = xenbus_printf(xbt, sc->sc_xbusd->xbusd_path, 538 error = xenbus_printf(xbt, sc->sc_xbusd->xbusd_path,
539 "protocol", "%s", XEN_IO_PROTO_ABI_NATIVE); 539 "protocol", "%s", XEN_IO_PROTO_ABI_NATIVE);
540 if (error) { 540 if (error) {
541 errmsg = "writing protocol"; 541 errmsg = "writing protocol";
542 goto abort_transaction; 542 goto abort_transaction;
543 } 543 }
544 error = xenbus_transaction_end(xbt, 0); 544 error = xenbus_transaction_end(xbt, 0);
545 if (error == EAGAIN) 545 if (error == EAGAIN)
546 goto again; 546 goto again;
547 if (error != 0) { 547 if (error != 0) {
548 xenbus_dev_fatal(sc->sc_xbusd, error, 548 xenbus_dev_fatal(sc->sc_xbusd, error,
549 "completing transaction"); 549 "completing transaction");
550 return false; 550 return false;
551 } 551 }
552 552
553 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateInitialised); 553 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateInitialised);
554 554
555 if (sc->sc_backend_status == BLKIF_STATE_SUSPENDED) { 555 if (sc->sc_backend_status == BLKIF_STATE_SUSPENDED) {
556 /* 556 /*
557 * device was suspended, softc structures are 557 * device was suspended, softc structures are
558 * already initialized - we use a shortcut 558 * already initialized - we use a shortcut
559 */ 559 */
560 sc->sc_backend_status = BLKIF_STATE_CONNECTED; 560 sc->sc_backend_status = BLKIF_STATE_CONNECTED;
561 xenbus_device_resume(sc->sc_xbusd); 561 xenbus_device_resume(sc->sc_xbusd);
562 hypervisor_unmask_event(sc->sc_evtchn); 562 hypervisor_unmask_event(sc->sc_evtchn);
563 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateConnected); 563 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateConnected);
564 } 564 }
565 565
566 return true; 566 return true;
567 567
568abort_resume: 568abort_resume:
569 xenbus_dev_fatal(sc->sc_xbusd, error, "resuming device"); 569 xenbus_dev_fatal(sc->sc_xbusd, error, "resuming device");
570 return false; 570 return false;
571 571
572abort_transaction: 572abort_transaction:
573 xenbus_transaction_end(xbt, 1); 573 xenbus_transaction_end(xbt, 1);
574 xenbus_dev_fatal(sc->sc_xbusd, error, "%s", errmsg); 574 xenbus_dev_fatal(sc->sc_xbusd, error, "%s", errmsg);
575 return false; 575 return false;
576} 576}
577 577
578static void 578static void
579xbd_backend_changed(void *arg, XenbusState new_state) 579xbd_backend_changed(void *arg, XenbusState new_state)
580{ 580{
581 struct xbd_xenbus_softc *sc = device_private((device_t)arg); 581 struct xbd_xenbus_softc *sc = device_private((device_t)arg);
582 struct disk_geom *dg; 582 struct disk_geom *dg;
583 583
584 char buf[32]; 584 char buf[32];
585 DPRINTF(("%s: new backend state %d\n", 585 DPRINTF(("%s: new backend state %d\n",
586 device_xname(sc->sc_dksc.sc_dev), new_state)); 586 device_xname(sc->sc_dksc.sc_dev), new_state));
587 587
588 switch (new_state) { 588 switch (new_state) {
589 case XenbusStateUnknown: 589 case XenbusStateUnknown:
590 case XenbusStateInitialising: 590 case XenbusStateInitialising:
591 case XenbusStateInitWait: 591 case XenbusStateInitWait:
592 case XenbusStateInitialised: 592 case XenbusStateInitialised:
593 break; 593 break;
594 case XenbusStateClosing: 594 case XenbusStateClosing:
595 mutex_enter(&sc->sc_lock); 595 mutex_enter(&sc->sc_lock);
596 if (sc->sc_shutdown == BLKIF_SHUTDOWN_RUN) 596 if (sc->sc_shutdown == BLKIF_SHUTDOWN_RUN)
597 sc->sc_shutdown = BLKIF_SHUTDOWN_REMOTE; 597 sc->sc_shutdown = BLKIF_SHUTDOWN_REMOTE;
598 /* wait for requests to complete */ 598 /* wait for requests to complete */
599 while (sc->sc_backend_status == BLKIF_STATE_CONNECTED && 599 while (sc->sc_backend_status == BLKIF_STATE_CONNECTED &&
600 disk_isbusy(&sc->sc_dksc.sc_dkdev)) { 600 disk_isbusy(&sc->sc_dksc.sc_dkdev)) {
601 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2); 601 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2);
602 } 602 }
603 mutex_exit(&sc->sc_lock); 603 mutex_exit(&sc->sc_lock);
604 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateClosed); 604 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateClosed);
605 break; 605 break;
606 case XenbusStateConnected: 606 case XenbusStateConnected:
607 /* 607 /*
608 * note that xbd_backend_changed() can only be called by 608 * note that xbd_backend_changed() can only be called by
609 * the xenbus thread. 609 * the xenbus thread.
610 */ 610 */
611 611
612 if (sc->sc_backend_status == BLKIF_STATE_CONNECTED || 612 if (sc->sc_backend_status == BLKIF_STATE_CONNECTED ||
613 sc->sc_backend_status == BLKIF_STATE_SUSPENDED) 613 sc->sc_backend_status == BLKIF_STATE_SUSPENDED)
614 /* already connected */ 614 /* already connected */
615 return; 615 return;
616 616
617 xbd_connect(sc); 617 xbd_connect(sc);
618 sc->sc_shutdown = BLKIF_SHUTDOWN_RUN; 618 sc->sc_shutdown = BLKIF_SHUTDOWN_RUN;
619 sc->sc_xbdsize = 619 sc->sc_xbdsize =
620 sc->sc_sectors * (uint64_t)sc->sc_secsize / DEV_BSIZE; 620 sc->sc_sectors * (uint64_t)sc->sc_secsize / DEV_BSIZE;
621 dg = &sc->sc_dksc.sc_dkdev.dk_geom; 621 dg = &sc->sc_dksc.sc_dkdev.dk_geom;
622 memset(dg, 0, sizeof(*dg));  622 memset(dg, 0, sizeof(*dg));
623 623
624 dg->dg_secperunit = sc->sc_xbdsize; 624 dg->dg_secperunit = sc->sc_xbdsize;
625 dg->dg_secsize = DEV_BSIZE; 625 dg->dg_secsize = DEV_BSIZE;
626 dg->dg_ntracks = 1; 626 dg->dg_ntracks = 1;
627 // XXX: Ok to hard-code DEV_BSIZE? 627 // XXX: Ok to hard-code DEV_BSIZE?
628 dg->dg_nsectors = 1024 * (1024 / dg->dg_secsize); 628 dg->dg_nsectors = 1024 * (1024 / dg->dg_secsize);
629 dg->dg_ncylinders = dg->dg_secperunit / dg->dg_nsectors; 629 dg->dg_ncylinders = dg->dg_secperunit / dg->dg_nsectors;
630 630
631 bufq_alloc(&sc->sc_dksc.sc_bufq, "fcfs", 0); 631 bufq_alloc(&sc->sc_dksc.sc_bufq, "fcfs", 0);
632 dk_attach(&sc->sc_dksc); 632 dk_attach(&sc->sc_dksc);
633 disk_attach(&sc->sc_dksc.sc_dkdev); 633 disk_attach(&sc->sc_dksc.sc_dkdev);
634 634
635 sc->sc_backend_status = BLKIF_STATE_CONNECTED; 635 sc->sc_backend_status = BLKIF_STATE_CONNECTED;
636 hypervisor_unmask_event(sc->sc_evtchn); 636 hypervisor_unmask_event(sc->sc_evtchn);
637 637
638 format_bytes(buf, sizeof(buf), sc->sc_sectors * sc->sc_secsize); 638 format_bytes(buf, sizeof(buf), sc->sc_sectors * sc->sc_secsize);
639 aprint_normal_dev(sc->sc_dksc.sc_dev, 639 aprint_normal_dev(sc->sc_dksc.sc_dev,
640 "%s, %d bytes/sect x %" PRIu64 " sectors\n", 640 "%s, %d bytes/sect x %" PRIu64 " sectors\n",
641 buf, (int)dg->dg_secsize, sc->sc_xbdsize); 641 buf, (int)dg->dg_secsize, sc->sc_xbdsize);
642 snprintb(buf, sizeof(buf), BLKIF_FEATURE_BITS, 642 snprintb(buf, sizeof(buf), BLKIF_FEATURE_BITS,
643 sc->sc_features); 643 sc->sc_features);
644 aprint_normal_dev(sc->sc_dksc.sc_dev, 644 aprint_normal_dev(sc->sc_dksc.sc_dev,
645 "backend features %s\n", buf); 645 "backend features %s\n", buf);
646 646
647 /* Discover wedges on this disk. */ 647 /* Discover wedges on this disk. */
648 dkwedge_discover(&sc->sc_dksc.sc_dkdev); 648 dkwedge_discover(&sc->sc_dksc.sc_dkdev);
649 649
650 disk_set_info(sc->sc_dksc.sc_dev, &sc->sc_dksc.sc_dkdev, NULL); 650 disk_set_info(sc->sc_dksc.sc_dev, &sc->sc_dksc.sc_dkdev, NULL);
651 651
652 /* the disk should be working now */ 652 /* the disk should be working now */
653 config_pending_decr(sc->sc_dksc.sc_dev); 653 config_pending_decr(sc->sc_dksc.sc_dev);
654 break; 654 break;
655 default: 655 default:
656 panic("bad backend state %d", new_state); 656 panic("bad backend state %d", new_state);
657 } 657 }
658} 658}
659 659
660static void 660static void
661xbd_connect(struct xbd_xenbus_softc *sc) 661xbd_connect(struct xbd_xenbus_softc *sc)
662{ 662{
663 int err; 663 int err;
664 unsigned long long sectors; 664 unsigned long long sectors;
665 u_long val; 665 u_long val;
666 666
667 err = xenbus_read_ul(NULL, 667 err = xenbus_read_ul(NULL,
668 sc->sc_xbusd->xbusd_path, "virtual-device", &sc->sc_handle, 10); 668 sc->sc_xbusd->xbusd_path, "virtual-device", &sc->sc_handle, 10);
669 if (err) 669 if (err)
670 panic("%s: can't read number from %s/virtual-device\n",  670 panic("%s: can't read number from %s/virtual-device\n",
671 device_xname(sc->sc_dksc.sc_dev), 671 device_xname(sc->sc_dksc.sc_dev),
672 sc->sc_xbusd->xbusd_otherend); 672 sc->sc_xbusd->xbusd_otherend);
673 err = xenbus_read_ull(NULL, 673 err = xenbus_read_ull(NULL,
674 sc->sc_xbusd->xbusd_otherend, "sectors", &sectors, 10); 674 sc->sc_xbusd->xbusd_otherend, "sectors", &sectors, 10);
675 if (err) 675 if (err)
676 panic("%s: can't read number from %s/sectors\n",  676 panic("%s: can't read number from %s/sectors\n",
677 device_xname(sc->sc_dksc.sc_dev), 677 device_xname(sc->sc_dksc.sc_dev),
678 sc->sc_xbusd->xbusd_otherend); 678 sc->sc_xbusd->xbusd_otherend);
679 sc->sc_sectors = sectors; 679 sc->sc_sectors = sectors;
680 680
681 err = xenbus_read_ul(NULL, 681 err = xenbus_read_ul(NULL,
682 sc->sc_xbusd->xbusd_otherend, "info", &sc->sc_info, 10); 682 sc->sc_xbusd->xbusd_otherend, "info", &sc->sc_info, 10);
683 if (err) 683 if (err)
684 panic("%s: can't read number from %s/info\n",  684 panic("%s: can't read number from %s/info\n",
685 device_xname(sc->sc_dksc.sc_dev), 685 device_xname(sc->sc_dksc.sc_dev),
686 sc->sc_xbusd->xbusd_otherend); 686 sc->sc_xbusd->xbusd_otherend);
687 err = xenbus_read_ul(NULL, 687 err = xenbus_read_ul(NULL,
688 sc->sc_xbusd->xbusd_otherend, "sector-size", &sc->sc_secsize, 10); 688 sc->sc_xbusd->xbusd_otherend, "sector-size", &sc->sc_secsize, 10);
689 if (err) 689 if (err)
690 panic("%s: can't read number from %s/sector-size\n",  690 panic("%s: can't read number from %s/sector-size\n",
691 device_xname(sc->sc_dksc.sc_dev), 691 device_xname(sc->sc_dksc.sc_dev),
692 sc->sc_xbusd->xbusd_otherend); 692 sc->sc_xbusd->xbusd_otherend);
693 693
694 err = xenbus_read_ul(NULL, sc->sc_xbusd->xbusd_otherend, 694 err = xenbus_read_ul(NULL, sc->sc_xbusd->xbusd_otherend,
695 "feature-flush-cache", &val, 10); 695 "feature-flush-cache", &val, 10);
696 if (err) 696 if (err)
697 val = 0; 697 val = 0;
698 if (val > 0) 698 if (val > 0)
699 sc->sc_features |= BLKIF_FEATURE_CACHE_FLUSH; 699 sc->sc_features |= BLKIF_FEATURE_CACHE_FLUSH;
700 700
701 err = xenbus_read_ul(NULL, sc->sc_xbusd->xbusd_otherend, 701 err = xenbus_read_ul(NULL, sc->sc_xbusd->xbusd_otherend,
702 "feature-barrier", &val, 10); 702 "feature-barrier", &val, 10);
703 if (err) 703 if (err)
704 val = 0; 704 val = 0;
705 if (val > 0) 705 if (val > 0)
706 sc->sc_features |= BLKIF_FEATURE_BARRIER; 706 sc->sc_features |= BLKIF_FEATURE_BARRIER;
707 707
708 err = xenbus_read_ul(NULL, sc->sc_xbusd->xbusd_otherend, 708 err = xenbus_read_ul(NULL, sc->sc_xbusd->xbusd_otherend,
709 "feature-persistent", &val, 10); 709 "feature-persistent", &val, 10);
710 if (err) 710 if (err)
711 val = 0; 711 val = 0;
712 if (val > 0) 712 if (val > 0)
713 sc->sc_features |= BLKIF_FEATURE_PERSISTENT; 713 sc->sc_features |= BLKIF_FEATURE_PERSISTENT;
714 714
715 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateConnected); 715 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateConnected);
716} 716}
717 717
718static int 718static int
719xbd_handler(void *arg) 719xbd_handler(void *arg)
720{ 720{
721 struct xbd_xenbus_softc *sc = arg; 721 struct xbd_xenbus_softc *sc = arg;
722 struct buf *bp; 722 struct buf *bp;
723 RING_IDX resp_prod, i; 723 RING_IDX resp_prod, i;
724 int more_to_do; 724 int more_to_do;
725 int seg; 725 int seg;
726 726
727 DPRINTF(("xbd_handler(%s)\n", device_xname(sc->sc_dksc.sc_dev))); 727 DPRINTF(("xbd_handler(%s)\n", device_xname(sc->sc_dksc.sc_dev)));
728 728
729 if (__predict_false(sc->sc_backend_status != BLKIF_STATE_CONNECTED)) 729 if (__predict_false(sc->sc_backend_status != BLKIF_STATE_CONNECTED))
730 return 0; 730 return 0;
731 731
732 mutex_enter(&sc->sc_lock); 732 mutex_enter(&sc->sc_lock);
733again: 733again:
734 resp_prod = sc->sc_ring.sring->rsp_prod; 734 resp_prod = sc->sc_ring.sring->rsp_prod;
735 xen_rmb(); /* ensure we see replies up to resp_prod */ 735 xen_rmb(); /* ensure we see replies up to resp_prod */
736 for (i = sc->sc_ring.rsp_cons; i != resp_prod; i++) { 736 for (i = sc->sc_ring.rsp_cons; i != resp_prod; i++) {
737 blkif_response_t *rep = RING_GET_RESPONSE(&sc->sc_ring, i); 737 blkif_response_t *rep = RING_GET_RESPONSE(&sc->sc_ring, i);
738 struct xbd_req *xbdreq = &sc->sc_reqs[rep->id]; 738 struct xbd_req *xbdreq = &sc->sc_reqs[rep->id];
739 739
740 if (rep->operation == BLKIF_OP_FLUSH_DISKCACHE) { 740 if (rep->operation == BLKIF_OP_FLUSH_DISKCACHE) {
741 KASSERT(xbdreq->req_bp == NULL); 741 KASSERT(xbdreq->req_bp == NULL);
742 xbdreq->req_sync.s_error = rep->status; 742 xbdreq->req_sync.s_error = rep->status;
743 xbdreq->req_sync.s_done = 1; 743 xbdreq->req_sync.s_done = 1;
744 cv_broadcast(&sc->sc_cache_flush_cv); 744 cv_broadcast(&sc->sc_cache_flush_cv);
745 /* caller will free the req */ 745 /* caller will free the req */
746 continue; 746 continue;
747 } 747 }
748 748
749 if (rep->operation != BLKIF_OP_READ && 749 if (rep->operation != BLKIF_OP_READ &&
750 rep->operation != BLKIF_OP_WRITE) { 750 rep->operation != BLKIF_OP_WRITE) {
751 aprint_error_dev(sc->sc_dksc.sc_dev, 751 aprint_error_dev(sc->sc_dksc.sc_dev,
752 "bad operation %d from backend\n", rep->operation); 752 "bad operation %d from backend\n", rep->operation);
753 continue; 753 continue;
754 } 754 }
755 755
756 bp = xbdreq->req_bp; 756 bp = xbdreq->req_bp;
757 KASSERT(bp != NULL && bp->b_data != NULL); 757 KASSERT(bp != NULL && bp->b_data != NULL);
758 DPRINTF(("%s(%p): b_bcount = %ld\n", __func__, 758 DPRINTF(("%s(%p): b_bcount = %ld\n", __func__,
759 bp, (long)bp->b_bcount)); 759 bp, (long)bp->b_bcount));
760 760
761 if (bp->b_error != 0 || rep->status != BLKIF_RSP_OKAY) { 761 if (bp->b_error != 0 || rep->status != BLKIF_RSP_OKAY) {
762 bp->b_error = EIO; 762 bp->b_error = EIO;
763 bp->b_resid = bp->b_bcount; 763 bp->b_resid = bp->b_bcount;
764 } 764 }
765 765
766 if (xbdreq->req_parent) { 766 if (xbdreq->req_parent) {
767 struct xbd_req *req_parent = xbdreq->req_parent; 767 struct xbd_req *req_parent = xbdreq->req_parent;
768 768
769 /* Unhook and recycle child */ 769 /* Unhook and recycle child */
770 xbdreq->req_parent = NULL; 770 xbdreq->req_parent = NULL;
771 req_parent->req_child = NULL; 771 req_parent->req_child = NULL;
772 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, xbdreq, 772 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, xbdreq,
773 req_next); 773 req_next);
774 774
775 if (!req_parent->req_parent_done) { 775 if (!req_parent->req_parent_done) {
776 /* Finished before parent, nothig else to do */ 776 /* Finished before parent, nothig else to do */
777 continue; 777 continue;
778 } 778 }
779 779
780 /* Must do the cleanup now */ 780 /* Must do the cleanup now */
781 xbdreq = req_parent; 781 xbdreq = req_parent;
782 } 782 }
783 if (xbdreq->req_child) { 783 if (xbdreq->req_child) {
784 /* Finished before child, child will cleanup */ 784 /* Finished before child, child will cleanup */
785 xbdreq->req_parent_done = true; 785 xbdreq->req_parent_done = true;
786 continue; 786 continue;
787 } 787 }
788 788
789 if (bp->b_error == 0) 789 if (bp->b_error == 0)
790 bp->b_resid = 0; 790 bp->b_resid = 0;
791 791
792 for (seg = 0; seg < xbdreq->req_dmamap->dm_nsegs; seg++) { 792 for (seg = 0; seg < xbdreq->req_dmamap->dm_nsegs; seg++) {
793 /* 793 /*
794 * We are not allowing persistent mappings, so 794 * We are not allowing persistent mappings, so
795 * expect the backend to release the grant 795 * expect the backend to release the grant
796 * immediately. 796 * immediately.
797 */ 797 */
798 KASSERT(xengnt_status(xbdreq->req_gntref[seg]) == 0); 798 KASSERT(xengnt_status(xbdreq->req_gntref[seg]) == 0);
799 xengnt_revoke_access(xbdreq->req_gntref[seg]); 799 xengnt_revoke_access(xbdreq->req_gntref[seg]);
800 } 800 }
801 801
802 bus_dmamap_unload(sc->sc_xbusd->xbusd_dmat, xbdreq->req_dmamap); 802 bus_dmamap_unload(sc->sc_xbusd->xbusd_dmat, xbdreq->req_dmamap);
803 803
804 if (__predict_false(bp->b_data != xbdreq->req_data)) 804 if (__predict_false(bp->b_data != xbdreq->req_data))
805 xbd_unmap_align(sc, xbdreq, true); 805 xbd_unmap_align(sc, xbdreq, true);
806 xbdreq->req_bp = xbdreq->req_data = NULL; 806 xbdreq->req_bp = xbdreq->req_data = NULL;
807 807
808 dk_done(&sc->sc_dksc, bp); 808 dk_done(&sc->sc_dksc, bp);
809 809
810 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, xbdreq, req_next); 810 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, xbdreq, req_next);
811 } 811 }
812 812
813 xen_rmb(); 813 xen_rmb();
814 sc->sc_ring.rsp_cons = i; 814 sc->sc_ring.rsp_cons = i;
815 815
816 RING_FINAL_CHECK_FOR_RESPONSES(&sc->sc_ring, more_to_do); 816 RING_FINAL_CHECK_FOR_RESPONSES(&sc->sc_ring, more_to_do);
817 if (more_to_do) 817 if (more_to_do)
818 goto again; 818 goto again;
819 819
820 cv_signal(&sc->sc_req_cv); 820 cv_signal(&sc->sc_req_cv);
821 mutex_exit(&sc->sc_lock); 821 mutex_exit(&sc->sc_lock);
822 822
823 dk_start(&sc->sc_dksc, NULL); 823 dk_start(&sc->sc_dksc, NULL);
824 824
825 return 1; 825 return 1;
826} 826}
827 827
828static void 828static void
829xbdminphys(struct buf *bp) 829xbdminphys(struct buf *bp)
830{ 830{
831 if (bp->b_bcount > XBD_XFER_LIMIT) { 831 if (bp->b_bcount > XBD_XFER_LIMIT) {
832 bp->b_bcount = XBD_XFER_LIMIT; 832 bp->b_bcount = XBD_XFER_LIMIT;
833 } 833 }
834 minphys(bp); 834 minphys(bp);
835} 835}
836 836
837static void 837static void
838xbd_iosize(device_t dev, int *maxxfer) 838xbd_iosize(device_t dev, int *maxxfer)
839{ 839{
840 /* 840 /*
841 * Always restrict dumps to XBD_MAX_XFER to avoid indirect segments, 841 * Always restrict dumps to XBD_MAX_XFER to avoid indirect segments,
842 * so that it uses as little memory as possible.  842 * so that it uses as little memory as possible.
843 */ 843 */
844 if (*maxxfer > XBD_MAX_XFER) 844 if (*maxxfer > XBD_MAX_XFER)
845 *maxxfer = XBD_MAX_XFER; 845 *maxxfer = XBD_MAX_XFER;
846} 846}
847 847
848int 848int
849xbdopen(dev_t dev, int flags, int fmt, struct lwp *l) 849xbdopen(dev_t dev, int flags, int fmt, struct lwp *l)
850{ 850{
851 struct xbd_xenbus_softc *sc; 851 struct xbd_xenbus_softc *sc;
852 852
853 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev)); 853 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev));
854 if (sc == NULL) 854 if (sc == NULL)
855 return (ENXIO); 855 return (ENXIO);
856 if ((flags & FWRITE) && (sc->sc_info & VDISK_READONLY)) 856 if ((flags & FWRITE) && (sc->sc_info & VDISK_READONLY))
857 return EROFS; 857 return EROFS;
858 858
859 DPRINTF(("xbdopen(0x%04x, %d)\n", dev, flags)); 859 DPRINTF(("xbdopen(0x%04x, %d)\n", dev, flags));
860 return dk_open(&sc->sc_dksc, dev, flags, fmt, l); 860 return dk_open(&sc->sc_dksc, dev, flags, fmt, l);
861} 861}
862 862
863int 863int
864xbdclose(dev_t dev, int flags, int fmt, struct lwp *l) 864xbdclose(dev_t dev, int flags, int fmt, struct lwp *l)
865{ 865{
866 struct xbd_xenbus_softc *sc; 866 struct xbd_xenbus_softc *sc;
867 867
868 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev)); 868 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev));
869 869
870 DPRINTF(("xbdclose(%d, %d)\n", dev, flags)); 870 DPRINTF(("xbdclose(%d, %d)\n", dev, flags));
871 return dk_close(&sc->sc_dksc, dev, flags, fmt, l); 871 return dk_close(&sc->sc_dksc, dev, flags, fmt, l);
872} 872}
873 873
874void 874void
875xbdstrategy(struct buf *bp) 875xbdstrategy(struct buf *bp)
876{ 876{
877 struct xbd_xenbus_softc *sc; 877 struct xbd_xenbus_softc *sc;
878 878
879 sc = device_lookup_private(&xbd_cd, DISKUNIT(bp->b_dev)); 879 sc = device_lookup_private(&xbd_cd, DISKUNIT(bp->b_dev));
880 880
881 DPRINTF(("xbdstrategy(%p): b_bcount = %ld\n", bp, 881 DPRINTF(("xbdstrategy(%p): b_bcount = %ld\n", bp,
882 (long)bp->b_bcount)); 882 (long)bp->b_bcount));
883 883
884 if (sc == NULL || sc->sc_shutdown != BLKIF_SHUTDOWN_RUN) { 884 if (sc == NULL || sc->sc_shutdown != BLKIF_SHUTDOWN_RUN) {
885 bp->b_error = EIO; 885 bp->b_error = EIO;
886 biodone(bp); 886 biodone(bp);
887 return; 887 return;
888 } 888 }
889 if (__predict_false((sc->sc_info & VDISK_READONLY) && 889 if (__predict_false((sc->sc_info & VDISK_READONLY) &&
890 (bp->b_flags & B_READ) == 0)) { 890 (bp->b_flags & B_READ) == 0)) {
891 bp->b_error = EROFS; 891 bp->b_error = EROFS;
892 biodone(bp); 892 biodone(bp);
893 return; 893 return;
894 } 894 }
895 895
896 dk_strategy(&sc->sc_dksc, bp); 896 dk_strategy(&sc->sc_dksc, bp);
897 return; 897 return;
898} 898}
899 899
900int 900int
901xbdsize(dev_t dev) 901xbdsize(dev_t dev)
902{ 902{
903 struct xbd_xenbus_softc *sc; 903 struct xbd_xenbus_softc *sc;
904 904
905 DPRINTF(("xbdsize(%d)\n", dev)); 905 DPRINTF(("xbdsize(%d)\n", dev));
906 906
907 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev)); 907 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev));
908 if (sc == NULL || sc->sc_shutdown != BLKIF_SHUTDOWN_RUN) 908 if (sc == NULL || sc->sc_shutdown != BLKIF_SHUTDOWN_RUN)
909 return -1; 909 return -1;
910 return dk_size(&sc->sc_dksc, dev); 910 return dk_size(&sc->sc_dksc, dev);
911} 911}
912 912
913int 913int
914xbdread(dev_t dev, struct uio *uio, int flags) 914xbdread(dev_t dev, struct uio *uio, int flags)
915{ 915{
916 struct xbd_xenbus_softc *sc =  916 struct xbd_xenbus_softc *sc =
917 device_lookup_private(&xbd_cd, DISKUNIT(dev)); 917 device_lookup_private(&xbd_cd, DISKUNIT(dev));
918 struct dk_softc *dksc = &sc->sc_dksc; 918 struct dk_softc *dksc = &sc->sc_dksc;
919 919
920 if (!DK_ATTACHED(dksc)) 920 if (!DK_ATTACHED(dksc))
921 return ENXIO; 921 return ENXIO;
922 return physio(xbdstrategy, NULL, dev, B_READ, xbdminphys, uio); 922 return physio(xbdstrategy, NULL, dev, B_READ, xbdminphys, uio);
923} 923}
924 924
925int 925int
926xbdwrite(dev_t dev, struct uio *uio, int flags) 926xbdwrite(dev_t dev, struct uio *uio, int flags)
927{ 927{
928 struct xbd_xenbus_softc *sc = 928 struct xbd_xenbus_softc *sc =
929 device_lookup_private(&xbd_cd, DISKUNIT(dev)); 929 device_lookup_private(&xbd_cd, DISKUNIT(dev));
930 struct dk_softc *dksc = &sc->sc_dksc; 930 struct dk_softc *dksc = &sc->sc_dksc;
931 931
932 if (!DK_ATTACHED(dksc)) 932 if (!DK_ATTACHED(dksc))
933 return ENXIO; 933 return ENXIO;
934 if (__predict_false(sc->sc_info & VDISK_READONLY)) 934 if (__predict_false(sc->sc_info & VDISK_READONLY))
935 return EROFS; 935 return EROFS;
936 return physio(xbdstrategy, NULL, dev, B_WRITE, xbdminphys, uio); 936 return physio(xbdstrategy, NULL, dev, B_WRITE, xbdminphys, uio);
937} 937}
938 938
939int 939int
940xbdioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 940xbdioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
941{ 941{
942 struct xbd_xenbus_softc *sc = 942 struct xbd_xenbus_softc *sc =
943 device_lookup_private(&xbd_cd, DISKUNIT(dev)); 943 device_lookup_private(&xbd_cd, DISKUNIT(dev));
944 struct dk_softc *dksc; 944 struct dk_softc *dksc;
945 int error; 945 int error;
946 struct xbd_req *xbdreq; 946 struct xbd_req *xbdreq;
947 blkif_request_t *req; 947 blkif_request_t *req;
948 int notify; 948 int notify;
949 949
950 DPRINTF(("xbdioctl(%d, %08lx, %p, %d, %p)\n", 950 DPRINTF(("xbdioctl(%d, %08lx, %p, %d, %p)\n",
951 dev, cmd, data, flag, l)); 951 dev, cmd, data, flag, l));
952 dksc = &sc->sc_dksc; 952 dksc = &sc->sc_dksc;
953 953
954 switch (cmd) { 954 switch (cmd) {
955 case DIOCGCACHE: 955 case DIOCGCACHE:
956 { 956 {
957 /* Assume there is write cache if cache-flush is supported */ 957 /* Assume there is write cache if cache-flush is supported */
958 int *bitsp = (int *)data; 958 int *bitsp = (int *)data;
959 *bitsp = 0; 959 *bitsp = 0;
960 if (sc->sc_features & BLKIF_FEATURE_CACHE_FLUSH) 960 if (sc->sc_features & BLKIF_FEATURE_CACHE_FLUSH)
961 *bitsp |= DKCACHE_WRITE; 961 *bitsp |= DKCACHE_WRITE;
962 error = 0; 962 error = 0;
963 break; 963 break;
964 } 964 }
965 case DIOCCACHESYNC: 965 case DIOCCACHESYNC:
966 if ((sc->sc_features & BLKIF_FEATURE_CACHE_FLUSH) == 0) 966 if ((sc->sc_features & BLKIF_FEATURE_CACHE_FLUSH) == 0)
967 return EOPNOTSUPP; 967 return EOPNOTSUPP;
968 968
969 mutex_enter(&sc->sc_lock); 969 mutex_enter(&sc->sc_lock);
970 while ((xbdreq = SLIST_FIRST(&sc->sc_xbdreq_head)) == NULL) 970 while ((xbdreq = SLIST_FIRST(&sc->sc_xbdreq_head)) == NULL)
971 cv_wait(&sc->sc_req_cv, &sc->sc_lock); 971 cv_wait(&sc->sc_req_cv, &sc->sc_lock);
972 972
973 SLIST_REMOVE_HEAD(&sc->sc_xbdreq_head, req_next); 973 SLIST_REMOVE_HEAD(&sc->sc_xbdreq_head, req_next);
974 req = RING_GET_REQUEST(&sc->sc_ring, 974 req = RING_GET_REQUEST(&sc->sc_ring,
975 sc->sc_ring.req_prod_pvt); 975 sc->sc_ring.req_prod_pvt);
976 req->id = xbdreq->req_id; 976 req->id = xbdreq->req_id;
977 req->operation = BLKIF_OP_FLUSH_DISKCACHE; 977 req->operation = BLKIF_OP_FLUSH_DISKCACHE;
978 req->handle = sc->sc_handle; 978 req->handle = sc->sc_handle;
979 xbdreq->req_sync.s_done = 0; 979 xbdreq->req_sync.s_done = 0;
980 sc->sc_ring.req_prod_pvt++; 980 sc->sc_ring.req_prod_pvt++;
981 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&sc->sc_ring, notify); 981 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&sc->sc_ring, notify);
982 if (notify) 982 if (notify)
983 hypervisor_notify_via_evtchn(sc->sc_evtchn); 983 hypervisor_notify_via_evtchn(sc->sc_evtchn);
984 /* request sent, now wait for completion */ 984 /* request sent, now wait for completion */
985 while (xbdreq->req_sync.s_done == 0) 985 while (xbdreq->req_sync.s_done == 0)
986 cv_wait(&sc->sc_cache_flush_cv, &sc->sc_lock); 986 cv_wait(&sc->sc_cache_flush_cv, &sc->sc_lock);
987 987
988 if (xbdreq->req_sync.s_error == BLKIF_RSP_EOPNOTSUPP) 988 if (xbdreq->req_sync.s_error == BLKIF_RSP_EOPNOTSUPP)
989 error = EOPNOTSUPP; 989 error = EOPNOTSUPP;
990 else if (xbdreq->req_sync.s_error == BLKIF_RSP_OKAY) 990 else if (xbdreq->req_sync.s_error == BLKIF_RSP_OKAY)
991 error = 0; 991 error = 0;
992 else 992 else
993 error = EIO; 993 error = EIO;
994 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, xbdreq, req_next); 994 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, xbdreq, req_next);
995 cv_signal(&sc->sc_req_cv); 995 cv_signal(&sc->sc_req_cv);
996 mutex_exit(&sc->sc_lock); 996 mutex_exit(&sc->sc_lock);
997 997
998 /* Restart I/O if it was waiting for req */ 998 /* Restart I/O if it was waiting for req */
999 dk_start(&sc->sc_dksc, NULL); 999 dk_start(&sc->sc_dksc, NULL);
1000 break; 1000 break;
1001 1001
1002 default: 1002 default:
1003 error = dk_ioctl(dksc, dev, cmd, data, flag, l); 1003 error = dk_ioctl(dksc, dev, cmd, data, flag, l);
1004 break; 1004 break;
1005 } 1005 }
1006 1006
1007 return error; 1007 return error;
1008} 1008}
1009 1009
1010int 1010int
1011xbddump(dev_t dev, daddr_t blkno, void *va, size_t size) 1011xbddump(dev_t dev, daddr_t blkno, void *va, size_t size)
1012{ 1012{
1013 struct xbd_xenbus_softc *sc; 1013 struct xbd_xenbus_softc *sc;
1014 1014
1015 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev)); 1015 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev));
1016 if (sc == NULL) 1016 if (sc == NULL)
1017 return (ENXIO); 1017 return (ENXIO);
1018 1018
1019 DPRINTF(("xbddump(%d, %" PRId64 ", %p, %lu)\n", dev, blkno, va, 1019 DPRINTF(("xbddump(%d, %" PRId64 ", %p, %lu)\n", dev, blkno, va,
1020 (unsigned long)size)); 1020 (unsigned long)size));
1021 return dk_dump(&sc->sc_dksc, dev, blkno, va, size, 0); 1021 return dk_dump(&sc->sc_dksc, dev, blkno, va, size, 0);
1022} 1022}
1023 1023
1024static int 1024static int
1025xbd_diskstart(device_t self, struct buf *bp) 1025xbd_diskstart(device_t self, struct buf *bp)
1026{ 1026{
1027 struct xbd_xenbus_softc *sc = device_private(self); 1027 struct xbd_xenbus_softc *sc = device_private(self);
1028 struct xbd_req *xbdreq; 1028 struct xbd_req *xbdreq;
1029 int error = 0; 1029 int error = 0;
1030 int notify; 1030 int notify;
1031 1031
1032 KASSERT(bp->b_bcount <= MAXPHYS); 1032 KASSERT(bp->b_bcount <= MAXPHYS);
1033 1033
1034 DPRINTF(("xbd_diskstart(%p): b_bcount = %ld\n", 1034 DPRINTF(("xbd_diskstart(%p): b_bcount = %ld\n",
1035 bp, (long)bp->b_bcount)); 1035 bp, (long)bp->b_bcount));
1036 1036
1037 mutex_enter(&sc->sc_lock); 1037 mutex_enter(&sc->sc_lock);
1038 1038
1039 if (sc->sc_shutdown != BLKIF_SHUTDOWN_RUN) { 1039 if (sc->sc_shutdown != BLKIF_SHUTDOWN_RUN) {
1040 error = EIO; 1040 error = EIO;
1041 goto out; 1041 goto out;
1042 } 1042 }
1043 1043
1044 if (bp->b_rawblkno < 0 || bp->b_rawblkno > sc->sc_xbdsize) { 1044 if (bp->b_rawblkno < 0 || bp->b_rawblkno > sc->sc_xbdsize) {
1045 /* invalid block number */ 1045 /* invalid block number */
1046 error = EINVAL; 1046 error = EINVAL;
1047 goto out; 1047 goto out;
1048 } 1048 }
1049 1049
1050 if (__predict_false( 1050 if (__predict_false(
1051 sc->sc_backend_status == BLKIF_STATE_SUSPENDED)) { 1051 sc->sc_backend_status == BLKIF_STATE_SUSPENDED)) {
1052 /* device is suspended, do not consume buffer */ 1052 /* device is suspended, do not consume buffer */
1053 DPRINTF(("%s: (xbd_diskstart) device suspended\n", 1053 DPRINTF(("%s: (xbd_diskstart) device suspended\n",
1054 sc->sc_dksc.sc_xname)); 1054 sc->sc_dksc.sc_xname));
1055 error = EAGAIN; 1055 error = EAGAIN;
1056 goto out; 1056 goto out;
1057 } 1057 }
1058 1058
1059 xbdreq = SLIST_FIRST(&sc->sc_xbdreq_head); 1059 xbdreq = SLIST_FIRST(&sc->sc_xbdreq_head);
1060 if (__predict_false(xbdreq == NULL)) { 1060 if (__predict_false(xbdreq == NULL)) {
1061 sc->sc_cnt_queue_full.ev_count++; 1061 sc->sc_cnt_queue_full.ev_count++;
1062 DPRINTF(("xbd_diskstart: no req\n")); 1062 DPRINTF(("xbd_diskstart: no req\n"));
1063 error = EAGAIN; 1063 error = EAGAIN;
1064 goto out; 1064 goto out;
1065 } 1065 }
1066 1066
1067 if (bp->b_bcount > XBD_MAX_CHUNK) { 1067 if (bp->b_bcount > XBD_MAX_CHUNK) {
1068 if (!SLIST_NEXT(xbdreq, req_next)) { 1068 if (!SLIST_NEXT(xbdreq, req_next)) {
1069 DPRINTF(("%s: need extra req\n", __func__)); 1069 DPRINTF(("%s: need extra req\n", __func__));
1070 error = EAGAIN; 1070 error = EAGAIN;
1071 goto out; 1071 goto out;
1072 } 1072 }
1073 } 1073 }
1074 1074
1075 bp->b_resid = bp->b_bcount; 1075 bp->b_resid = bp->b_bcount;
1076 xbdreq->req_bp = bp; 1076 xbdreq->req_bp = bp;
1077 xbdreq->req_data = bp->b_data; 1077 xbdreq->req_data = bp->b_data;
1078 if (__predict_false((vaddr_t)bp->b_data & (XEN_BSIZE - 1))) { 1078 if (__predict_false((vaddr_t)bp->b_data & (XEN_BSIZE - 1))) {
1079 if (__predict_false(xbd_map_align(sc, xbdreq) != 0)) { 1079 if (__predict_false(xbd_map_align(sc, xbdreq) != 0)) {
1080 DPRINTF(("xbd_diskstart: no align\n")); 1080 DPRINTF(("xbd_diskstart: no align\n"));
1081 error = EAGAIN; 1081 error = EAGAIN;
1082 goto out; 1082 goto out;
1083 } 1083 }
1084 } 1084 }
1085 1085
1086 if (__predict_false(bus_dmamap_load(sc->sc_xbusd->xbusd_dmat, 1086 if (__predict_false(bus_dmamap_load(sc->sc_xbusd->xbusd_dmat,
1087 xbdreq->req_dmamap, xbdreq->req_data, bp->b_bcount, NULL, 1087 xbdreq->req_dmamap, xbdreq->req_data, bp->b_bcount, NULL,
1088 BUS_DMA_NOWAIT) != 0)) { 1088 BUS_DMA_NOWAIT) != 0)) {
1089 printf("%s: %s: bus_dmamap_load failed\n", 1089 printf("%s: %s: bus_dmamap_load failed\n",
1090 device_xname(sc->sc_dksc.sc_dev), __func__); 1090 device_xname(sc->sc_dksc.sc_dev), __func__);
1091 if (__predict_false(bp->b_data != xbdreq->req_data)) 1091 if (__predict_false(bp->b_data != xbdreq->req_data))
1092 xbd_unmap_align(sc, xbdreq, false); 1092 xbd_unmap_align(sc, xbdreq, false);
1093 error = EINVAL; 1093 error = EINVAL;
1094 goto out; 1094 goto out;
1095 } 1095 }
1096 1096
1097 for (int seg = 0; seg < xbdreq->req_dmamap->dm_nsegs; seg++) { 1097 for (int seg = 0; seg < xbdreq->req_dmamap->dm_nsegs; seg++) {
1098 KASSERT(seg < __arraycount(xbdreq->req_gntref)); 1098 KASSERT(seg < __arraycount(xbdreq->req_gntref));
1099 1099
1100 paddr_t ma = xbdreq->req_dmamap->dm_segs[seg].ds_addr; 1100 paddr_t ma = xbdreq->req_dmamap->dm_segs[seg].ds_addr;
1101 if (__predict_false(xengnt_grant_access( 1101 if (__predict_false(xengnt_grant_access(
1102 sc->sc_xbusd->xbusd_otherend_id, 1102 sc->sc_xbusd->xbusd_otherend_id,
1103 (ma & ~PAGE_MASK), (bp->b_flags & B_READ) == 0, 1103 (ma & ~PAGE_MASK), (bp->b_flags & B_READ) == 0,
1104 &xbdreq->req_gntref[seg]))) { 1104 &xbdreq->req_gntref[seg]))) {
1105 printf("%s: %s: xengnt_grant_access failed\n", 1105 printf("%s: %s: xengnt_grant_access failed\n",
1106 device_xname(sc->sc_dksc.sc_dev), __func__); 1106 device_xname(sc->sc_dksc.sc_dev), __func__);
1107 if (seg > 0) { 1107 if (seg > 0) {
1108 for (; --seg >= 0; ) { 1108 for (; --seg >= 0; ) {
1109 xengnt_revoke_access( 1109 xengnt_revoke_access(
1110 xbdreq->req_gntref[seg]); 1110 xbdreq->req_gntref[seg]);
1111 } 1111 }
1112 } 1112 }
1113 bus_dmamap_unload(sc->sc_xbusd->xbusd_dmat, 1113 bus_dmamap_unload(sc->sc_xbusd->xbusd_dmat,
1114 xbdreq->req_dmamap); 1114 xbdreq->req_dmamap);
1115 if (__predict_false(bp->b_data != xbdreq->req_data)) 1115 if (__predict_false(bp->b_data != xbdreq->req_data))
1116 xbd_unmap_align(sc, xbdreq, false); 1116 xbd_unmap_align(sc, xbdreq, false);
1117 error = EAGAIN; 1117 error = EAGAIN;
1118 goto out; 1118 goto out;
1119 } 1119 }
1120 } 1120 }
1121 1121
1122 KASSERT(xbdreq->req_parent == NULL); 1122 KASSERT(xbdreq->req_parent == NULL);
1123 KASSERT(xbdreq->req_child == NULL); 1123 KASSERT(xbdreq->req_child == NULL);
1124 1124
1125 /* We are now committed to the transfer */ 1125 /* We are now committed to the transfer */
1126 SLIST_REMOVE_HEAD(&sc->sc_xbdreq_head, req_next); 1126 SLIST_REMOVE_HEAD(&sc->sc_xbdreq_head, req_next);
1127 xbd_diskstart_submit(sc, xbdreq->req_id, 1127 xbd_diskstart_submit(sc, xbdreq->req_id,
1128 bp, 0, xbdreq->req_dmamap, xbdreq->req_gntref); 1128 bp, 0, xbdreq->req_dmamap, xbdreq->req_gntref);
1129 1129
1130 if (bp->b_bcount > XBD_MAX_CHUNK) { 1130 if (bp->b_bcount > XBD_MAX_CHUNK) {
1131 struct xbd_req *xbdreq2 = SLIST_FIRST(&sc->sc_xbdreq_head); 1131 struct xbd_req *xbdreq2 = SLIST_FIRST(&sc->sc_xbdreq_head);
1132 KASSERT(xbdreq2 != NULL); /* Checked earlier */ 1132 KASSERT(xbdreq2 != NULL); /* Checked earlier */
1133 SLIST_REMOVE_HEAD(&sc->sc_xbdreq_head, req_next); 1133 SLIST_REMOVE_HEAD(&sc->sc_xbdreq_head, req_next);
1134 xbdreq->req_child = xbdreq2; 1134 xbdreq->req_child = xbdreq2;
1135 xbdreq->req_parent_done = false; 1135 xbdreq->req_parent_done = false;
1136 xbdreq2->req_parent = xbdreq; 1136 xbdreq2->req_parent = xbdreq;
1137 xbdreq2->req_bp = bp; 1137 xbdreq2->req_bp = bp;
1138 xbdreq2->req_data = NULL; 1138 xbdreq2->req_data = NULL;
1139 xbd_diskstart_submit(sc, xbdreq2->req_id, 1139 xbd_diskstart_submit(sc, xbdreq2->req_id,
1140 bp, XBD_MAX_CHUNK, xbdreq->req_dmamap, 1140 bp, XBD_MAX_CHUNK, xbdreq->req_dmamap,
1141 xbdreq->req_gntref); 1141 xbdreq->req_gntref);
1142 } 1142 }
1143 1143
1144 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&sc->sc_ring, notify); 1144 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&sc->sc_ring, notify);
1145 if (notify) 1145 if (notify)
1146 hypervisor_notify_via_evtchn(sc->sc_evtchn); 1146 hypervisor_notify_via_evtchn(sc->sc_evtchn);
1147out: 1147out:
1148 mutex_exit(&sc->sc_lock); 1148 mutex_exit(&sc->sc_lock);
1149 return error; 1149 return error;
1150} 1150}
1151 1151
1152static void 1152static void
1153xbd_diskstart_submit(struct xbd_xenbus_softc *sc, 1153xbd_diskstart_submit(struct xbd_xenbus_softc *sc,
1154 int req_id, struct buf *bp, int start, bus_dmamap_t dmamap, 1154 int req_id, struct buf *bp, int start, bus_dmamap_t dmamap,
1155 grant_ref_t *gntref) 1155 grant_ref_t *gntref)
1156{ 1156{
1157 blkif_request_t *req; 1157 blkif_request_t *req;
1158 paddr_t ma; 1158 paddr_t ma;
1159 int nsects, nbytes, dmaseg, first_sect, size, segidx = 0; 1159 int nsects, nbytes, dmaseg, first_sect, size, segidx = 0;
1160 struct blkif_request_segment *reqseg; 1160 struct blkif_request_segment *reqseg;
1161 1161
1162 KASSERT(mutex_owned(&sc->sc_lock)); 1162 KASSERT(mutex_owned(&sc->sc_lock));
1163 1163
1164 req = RING_GET_REQUEST(&sc->sc_ring, sc->sc_ring.req_prod_pvt); 1164 req = RING_GET_REQUEST(&sc->sc_ring, sc->sc_ring.req_prod_pvt);
1165 req->id = req_id; 1165 req->id = req_id;
1166 req->operation = 1166 req->operation =
1167 bp->b_flags & B_READ ? BLKIF_OP_READ : BLKIF_OP_WRITE; 1167 bp->b_flags & B_READ ? BLKIF_OP_READ : BLKIF_OP_WRITE;
1168 req->sector_number = bp->b_rawblkno + (start >> XEN_BSHIFT); 1168 req->sector_number = bp->b_rawblkno + (start >> XEN_BSHIFT);
1169 req->handle = sc->sc_handle; 1169 req->handle = sc->sc_handle;
1170 1170
1171 size = uimin(bp->b_bcount - start, XBD_MAX_CHUNK);  1171 size = uimin(bp->b_bcount - start, XBD_MAX_CHUNK);
1172 for (dmaseg = 0; dmaseg < dmamap->dm_nsegs && size > 0; dmaseg++) { 1172 for (dmaseg = 0; dmaseg < dmamap->dm_nsegs && size > 0; dmaseg++) {
1173 bus_dma_segment_t *ds = &dmamap->dm_segs[dmaseg]; 1173 bus_dma_segment_t *ds = &dmamap->dm_segs[dmaseg];
1174 1174
1175 ma = ds->ds_addr; 1175 ma = ds->ds_addr;
1176 nbytes = imin(ds->ds_len, size); 1176 nbytes = imin(ds->ds_len, size);
1177 1177
1178 if (start > 0) { 1178 if (start > 0) {
1179 if (start >= nbytes) { 1179 if (start >= nbytes) {
1180 start -= nbytes; 1180 start -= nbytes;
1181 continue; 1181 continue;
1182 } 1182 }
1183 ma += start; 1183 ma += start;
1184 nbytes -= start; 1184 nbytes -= start;
1185 start = 0; 1185 start = 0;
1186 } 1186 }
1187 size -= nbytes; 1187 size -= nbytes;
1188 1188
1189 KASSERT(((ma & PAGE_MASK) & (XEN_BSIZE - 1)) == 0); 1189 KASSERT(((ma & PAGE_MASK) & (XEN_BSIZE - 1)) == 0);
1190 KASSERT((nbytes & (XEN_BSIZE - 1)) == 0); 1190 KASSERT((nbytes & (XEN_BSIZE - 1)) == 0);
1191 KASSERT((size & (XEN_BSIZE - 1)) == 0); 1191 KASSERT((size & (XEN_BSIZE - 1)) == 0);
1192 first_sect = (ma & PAGE_MASK) >> XEN_BSHIFT; 1192 first_sect = (ma & PAGE_MASK) >> XEN_BSHIFT;
1193 nsects = nbytes >> XEN_BSHIFT; 1193 nsects = nbytes >> XEN_BSHIFT;
1194 1194
1195 reqseg = &req->seg[segidx++]; 1195 reqseg = &req->seg[segidx++];
1196 reqseg->first_sect = first_sect; 1196 reqseg->first_sect = first_sect;
1197 reqseg->last_sect = first_sect + nsects - 1; 1197 reqseg->last_sect = first_sect + nsects - 1;
1198 KASSERT(reqseg->first_sect <= reqseg->last_sect); 1198 KASSERT(reqseg->first_sect <= reqseg->last_sect);
1199 KASSERT(reqseg->last_sect < (PAGE_SIZE / XEN_BSIZE)); 1199 KASSERT(reqseg->last_sect < (PAGE_SIZE / XEN_BSIZE));
1200 1200
1201 reqseg->gref = gntref[dmaseg]; 1201 reqseg->gref = gntref[dmaseg];
1202 } 1202 }
1203 req->nr_segments = segidx; 1203 req->nr_segments = segidx;
1204 sc->sc_ring.req_prod_pvt++; 1204 sc->sc_ring.req_prod_pvt++;
1205} 1205}
1206 1206
1207static int 1207static int
1208xbd_map_align(struct xbd_xenbus_softc *sc, struct xbd_req *req) 1208xbd_map_align(struct xbd_xenbus_softc *sc, struct xbd_req *req)
1209{ 1209{
1210 /* Only can get here if this is physio() request */ 1210 /*
1211 KASSERT(req->req_bp->b_saveaddr != NULL); 1211 * Only can get here if this is physio() request, block I/O
 1212 * uses DEV_BSIZE-aligned buffers.
 1213 */
 1214 KASSERT((req->req_bp->b_flags & B_PHYS) != 0);
1212 1215
1213 sc->sc_cnt_map_unalign.ev_count++; 1216 sc->sc_cnt_map_unalign.ev_count++;
1214 1217
1215 if (sc->sc_unalign_used) { 1218 if (sc->sc_unalign_used) {
1216 sc->sc_cnt_unalign_busy.ev_count++; 1219 sc->sc_cnt_unalign_busy.ev_count++;
1217 return EAGAIN; 1220 return EAGAIN;
1218 } 1221 }
1219 sc->sc_unalign_used = req; 1222 sc->sc_unalign_used = req;
1220 1223
1221 KASSERT(req->req_bp->b_bcount <= MAXPHYS); 1224 KASSERT(req->req_bp->b_bcount <= MAXPHYS);
1222 req->req_data = (void *)sc->sc_unalign_buffer; 1225 req->req_data = (void *)sc->sc_unalign_buffer;
1223 if ((req->req_bp->b_flags & B_READ) == 0) 1226 if ((req->req_bp->b_flags & B_READ) == 0)
1224 memcpy(req->req_data, req->req_bp->b_data, 1227 memcpy(req->req_data, req->req_bp->b_data,
1225 req->req_bp->b_bcount); 1228 req->req_bp->b_bcount);
1226 return 0; 1229 return 0;
1227} 1230}
1228 1231
1229static void 1232static void
1230xbd_unmap_align(struct xbd_xenbus_softc *sc, struct xbd_req *req, bool sync) 1233xbd_unmap_align(struct xbd_xenbus_softc *sc, struct xbd_req *req, bool sync)
1231{ 1234{
1232 KASSERT(sc->sc_unalign_used == req); 1235 KASSERT(sc->sc_unalign_used == req);
1233 if (sync && req->req_bp->b_flags & B_READ) 1236 if (sync && req->req_bp->b_flags & B_READ)
1234 memcpy(req->req_bp->b_data, req->req_data, 1237 memcpy(req->req_bp->b_data, req->req_data,
1235 req->req_bp->b_bcount); 1238 req->req_bp->b_bcount);
1236 sc->sc_unalign_used = NULL; 1239 sc->sc_unalign_used = NULL;
1237} 1240}