Fri Apr 17 10:35:06 2020 UTC ()
constify xbddkdriver


(jdolecek)
diff -r1.117 -r1.118 src/sys/arch/xen/xen/xbd_xenbus.c
cvs diff -r1.117 -r1.118 src/sys/arch/xen/xen/xbd_xenbus.c (switch to unified diff)

--- src/sys/arch/xen/xen/xbd_xenbus.c 2020/04/17 10:32:19 1.117
+++ src/sys/arch/xen/xen/xbd_xenbus.c 2020/04/17 10:35:06 1.118
@@ -1,1252 +1,1252 @@ @@ -1,1252 +1,1252 @@
1/* $NetBSD: xbd_xenbus.c,v 1.117 2020/04/17 10:32:19 jdolecek Exp $ */ 1/* $NetBSD: xbd_xenbus.c,v 1.118 2020/04/17 10:35:06 jdolecek Exp $ */
2 2
3/* 3/*
4 * Copyright (c) 2006 Manuel Bouyer. 4 * Copyright (c) 2006 Manuel Bouyer.
5 * 5 *
6 * Redistribution and use in source and binary forms, with or without 6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions 7 * modification, are permitted provided that the following conditions
8 * are met: 8 * are met:
9 * 1. Redistributions of source code must retain the above copyright 9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer. 10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright 11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the 12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution. 13 * documentation and/or other materials provided with the distribution.
14 * 14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 * 25 *
26 */ 26 */
27 27
28/* 28/*
29 * The file contains the xbd frontend code required for block-level 29 * The file contains the xbd frontend code required for block-level
30 * communications (similar to hard disks) between two Xen domains. 30 * communications (similar to hard disks) between two Xen domains.
31 * 31 *
32 * We are not supposed to receive solicitations spontaneously from backend. The 32 * We are not supposed to receive solicitations spontaneously from backend. The
33 * protocol is therefore fairly simple and uses only one ring to communicate 33 * protocol is therefore fairly simple and uses only one ring to communicate
34 * with backend: frontend posts requests to the ring then wait for their 34 * with backend: frontend posts requests to the ring then wait for their
35 * replies asynchronously. 35 * replies asynchronously.
36 * 36 *
37 * xbd follows NetBSD's disk(9) convention. At any time, a LWP can schedule 37 * xbd follows NetBSD's disk(9) convention. At any time, a LWP can schedule
38 * an operation request for the device (be it open(), read(), write(), ...). 38 * an operation request for the device (be it open(), read(), write(), ...).
39 * Calls are typically processed that way: 39 * Calls are typically processed that way:
40 * - initiate request: xbdread/write/open/ioctl/.. 40 * - initiate request: xbdread/write/open/ioctl/..
41 * - depending on operation, it is handled directly by disk(9) subsystem or 41 * - depending on operation, it is handled directly by disk(9) subsystem or
42 * goes through physio(9) first. 42 * goes through physio(9) first.
43 * - the request is ultimately processed by xbd_diskstart() that prepares the 43 * - the request is ultimately processed by xbd_diskstart() that prepares the
44 * xbd requests, post them in the ring I/O queue, then signal the backend. 44 * xbd requests, post them in the ring I/O queue, then signal the backend.
45 * 45 *
46 * When a response is available in the queue, the backend signals the frontend 46 * When a response is available in the queue, the backend signals the frontend
47 * via its event channel. This triggers xbd_handler(), which will link back 47 * via its event channel. This triggers xbd_handler(), which will link back
48 * the response to its request through the request ID, and mark the I/O as 48 * the response to its request through the request ID, and mark the I/O as
49 * completed. 49 * completed.
50 */ 50 */
51 51
52#include <sys/cdefs.h> 52#include <sys/cdefs.h>
53__KERNEL_RCSID(0, "$NetBSD: xbd_xenbus.c,v 1.117 2020/04/17 10:32:19 jdolecek Exp $"); 53__KERNEL_RCSID(0, "$NetBSD: xbd_xenbus.c,v 1.118 2020/04/17 10:35:06 jdolecek Exp $");
54 54
55#include "opt_xen.h" 55#include "opt_xen.h"
56 56
57 57
58#include <sys/param.h> 58#include <sys/param.h>
59#include <sys/buf.h> 59#include <sys/buf.h>
60#include <sys/bufq.h> 60#include <sys/bufq.h>
61#include <sys/device.h> 61#include <sys/device.h>
62#include <sys/disk.h> 62#include <sys/disk.h>
63#include <sys/disklabel.h> 63#include <sys/disklabel.h>
64#include <sys/conf.h> 64#include <sys/conf.h>
65#include <sys/fcntl.h> 65#include <sys/fcntl.h>
66#include <sys/kernel.h> 66#include <sys/kernel.h>
67#include <sys/proc.h> 67#include <sys/proc.h>
68#include <sys/systm.h> 68#include <sys/systm.h>
69#include <sys/stat.h> 69#include <sys/stat.h>
70#include <sys/vnode.h> 70#include <sys/vnode.h>
71#include <sys/mutex.h> 71#include <sys/mutex.h>
72 72
73#include <dev/dkvar.h> 73#include <dev/dkvar.h>
74 74
75#include <uvm/uvm.h> 75#include <uvm/uvm.h>
76 76
77#include <xen/hypervisor.h> 77#include <xen/hypervisor.h>
78#include <xen/evtchn.h> 78#include <xen/evtchn.h>
79#include <xen/granttables.h> 79#include <xen/granttables.h>
80#include <xen/include/public/io/blkif.h> 80#include <xen/include/public/io/blkif.h>
81#include <xen/include/public/io/protocols.h> 81#include <xen/include/public/io/protocols.h>
82 82
83#include <xen/xenbus.h> 83#include <xen/xenbus.h>
84#include "locators.h" 84#include "locators.h"
85 85
86#undef XBD_DEBUG 86#undef XBD_DEBUG
87#ifdef XBD_DEBUG 87#ifdef XBD_DEBUG
88#define DPRINTF(x) printf x; 88#define DPRINTF(x) printf x;
89#else 89#else
90#define DPRINTF(x) 90#define DPRINTF(x)
91#endif 91#endif
92 92
93#define GRANT_INVALID_REF -1 93#define GRANT_INVALID_REF -1
94 94
95#define XBD_RING_SIZE __CONST_RING_SIZE(blkif, PAGE_SIZE) 95#define XBD_RING_SIZE __CONST_RING_SIZE(blkif, PAGE_SIZE)
96#define XBD_MAX_XFER (PAGE_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST) 96#define XBD_MAX_XFER (PAGE_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST)
97#define XBD_MAX_CHUNK 32*1024 /* max I/O size we process in 1 req */ 97#define XBD_MAX_CHUNK 32*1024 /* max I/O size we process in 1 req */
98#define XBD_XFER_LIMIT (2*XBD_MAX_XFER) 98#define XBD_XFER_LIMIT (2*XBD_MAX_XFER)
99 99
100#define XEN_BSHIFT 9 /* log2(XEN_BSIZE) */ 100#define XEN_BSHIFT 9 /* log2(XEN_BSIZE) */
101#define XEN_BSIZE (1 << XEN_BSHIFT)  101#define XEN_BSIZE (1 << XEN_BSHIFT)
102 102
103CTASSERT((MAXPHYS <= 2*XBD_MAX_CHUNK)); 103CTASSERT((MAXPHYS <= 2*XBD_MAX_CHUNK));
104CTASSERT(XEN_BSIZE == DEV_BSIZE); 104CTASSERT(XEN_BSIZE == DEV_BSIZE);
105 105
106struct xbd_indirect { 106struct xbd_indirect {
107 SLIST_ENTRY(xbd_indirect) in_next; 107 SLIST_ENTRY(xbd_indirect) in_next;
108 struct blkif_request_segment *in_addr; 108 struct blkif_request_segment *in_addr;
109 grant_ref_t in_gntref; 109 grant_ref_t in_gntref;
110}; 110};
111 111
112struct xbd_req { 112struct xbd_req {
113 SLIST_ENTRY(xbd_req) req_next; 113 SLIST_ENTRY(xbd_req) req_next;
114 uint16_t req_id; /* ID passed to backend */ 114 uint16_t req_id; /* ID passed to backend */
115 bus_dmamap_t req_dmamap; 115 bus_dmamap_t req_dmamap;
116 struct xbd_req *req_parent, *req_child; 116 struct xbd_req *req_parent, *req_child;
117 bool req_parent_done; 117 bool req_parent_done;
118 union { 118 union {
119 struct { 119 struct {
120 grant_ref_t req_gntref[XBD_XFER_LIMIT >> PAGE_SHIFT]; 120 grant_ref_t req_gntref[XBD_XFER_LIMIT >> PAGE_SHIFT];
121 struct buf *req_bp; /* buffer associated with this request */ 121 struct buf *req_bp; /* buffer associated with this request */
122 void *req_data; /* pointer to the data buffer */ 122 void *req_data; /* pointer to the data buffer */
123 struct xbd_indirect *req_indirect; /* indirect page */ 123 struct xbd_indirect *req_indirect; /* indirect page */
124 } req_rw; 124 } req_rw;
125 struct { 125 struct {
126 int s_error; 126 int s_error;
127 int s_done; 127 int s_done;
128 } req_sync; 128 } req_sync;
129 } u; 129 } u;
130}; 130};
131#define req_gntref u.req_rw.req_gntref 131#define req_gntref u.req_rw.req_gntref
132#define req_bp u.req_rw.req_bp 132#define req_bp u.req_rw.req_bp
133#define req_data u.req_rw.req_data 133#define req_data u.req_rw.req_data
134#define req_indirect u.req_rw.req_indirect 134#define req_indirect u.req_rw.req_indirect
135#define req_sync u.req_sync 135#define req_sync u.req_sync
136 136
137struct xbd_xenbus_softc { 137struct xbd_xenbus_softc {
138 struct dk_softc sc_dksc; /* Must be first in this struct */ 138 struct dk_softc sc_dksc; /* Must be first in this struct */
139 struct xenbus_device *sc_xbusd; 139 struct xenbus_device *sc_xbusd;
140 unsigned int sc_evtchn; 140 unsigned int sc_evtchn;
141 141
142 struct intrhand *sc_ih; /* Interrupt handler for this instance. */ 142 struct intrhand *sc_ih; /* Interrupt handler for this instance. */
143 kmutex_t sc_lock; 143 kmutex_t sc_lock;
144 kcondvar_t sc_cache_flush_cv; 144 kcondvar_t sc_cache_flush_cv;
145 kcondvar_t sc_req_cv; 145 kcondvar_t sc_req_cv;
146 kcondvar_t sc_detach_cv; 146 kcondvar_t sc_detach_cv;
147 kcondvar_t sc_suspend_cv; 147 kcondvar_t sc_suspend_cv;
148 148
149 blkif_front_ring_t sc_ring; 149 blkif_front_ring_t sc_ring;
150 grant_ref_t sc_ring_gntref; 150 grant_ref_t sc_ring_gntref;
151 151
152 struct xbd_req sc_reqs[XBD_RING_SIZE]; 152 struct xbd_req sc_reqs[XBD_RING_SIZE];
153 SLIST_HEAD(,xbd_req) sc_xbdreq_head; /* list of free requests */ 153 SLIST_HEAD(,xbd_req) sc_xbdreq_head; /* list of free requests */
154 154
155 struct xbd_indirect sc_indirect[XBD_RING_SIZE]; 155 struct xbd_indirect sc_indirect[XBD_RING_SIZE];
156 SLIST_HEAD(,xbd_indirect) sc_indirect_head; 156 SLIST_HEAD(,xbd_indirect) sc_indirect_head;
157 157
158 vmem_addr_t sc_unalign_buffer; 158 vmem_addr_t sc_unalign_buffer;
159 struct xbd_req *sc_unalign_used; 159 struct xbd_req *sc_unalign_used;
160 160
161 int sc_backend_status; /* our status with backend */ 161 int sc_backend_status; /* our status with backend */
162#define BLKIF_STATE_DISCONNECTED 0 162#define BLKIF_STATE_DISCONNECTED 0
163#define BLKIF_STATE_CONNECTED 1 163#define BLKIF_STATE_CONNECTED 1
164#define BLKIF_STATE_SUSPENDED 2 164#define BLKIF_STATE_SUSPENDED 2
165 165
166 int sc_shutdown; 166 int sc_shutdown;
167#define BLKIF_SHUTDOWN_RUN 0 /* no shutdown */ 167#define BLKIF_SHUTDOWN_RUN 0 /* no shutdown */
168#define BLKIF_SHUTDOWN_REMOTE 1 /* backend-initiated shutdown in progress */ 168#define BLKIF_SHUTDOWN_REMOTE 1 /* backend-initiated shutdown in progress */
169#define BLKIF_SHUTDOWN_LOCAL 2 /* locally-initiated shutdown in progress */ 169#define BLKIF_SHUTDOWN_LOCAL 2 /* locally-initiated shutdown in progress */
170 170
171 uint64_t sc_sectors; /* number of sectors for this device */ 171 uint64_t sc_sectors; /* number of sectors for this device */
172 u_long sc_secsize; /* sector size */ 172 u_long sc_secsize; /* sector size */
173 uint64_t sc_xbdsize; /* size of disk in DEV_BSIZE */ 173 uint64_t sc_xbdsize; /* size of disk in DEV_BSIZE */
174 u_long sc_info; /* VDISK_* */ 174 u_long sc_info; /* VDISK_* */
175 u_long sc_handle; /* from backend */ 175 u_long sc_handle; /* from backend */
176 int sc_features; 176 int sc_features;
177#define BLKIF_FEATURE_CACHE_FLUSH 0x1 177#define BLKIF_FEATURE_CACHE_FLUSH 0x1
178#define BLKIF_FEATURE_BARRIER 0x2 178#define BLKIF_FEATURE_BARRIER 0x2
179#define BLKIF_FEATURE_PERSISTENT 0x4 179#define BLKIF_FEATURE_PERSISTENT 0x4
180#define BLKIF_FEATURE_INDIRECT 0x8 180#define BLKIF_FEATURE_INDIRECT 0x8
181#define BLKIF_FEATURE_BITS \ 181#define BLKIF_FEATURE_BITS \
182 "\20\1CACHE-FLUSH\2BARRIER\3PERSISTENT\4INDIRECT" 182 "\20\1CACHE-FLUSH\2BARRIER\3PERSISTENT\4INDIRECT"
183 struct evcnt sc_cnt_map_unalign; 183 struct evcnt sc_cnt_map_unalign;
184 struct evcnt sc_cnt_unalign_busy; 184 struct evcnt sc_cnt_unalign_busy;
185 struct evcnt sc_cnt_queue_full; 185 struct evcnt sc_cnt_queue_full;
186 struct evcnt sc_cnt_indirect; 186 struct evcnt sc_cnt_indirect;
187}; 187};
188 188
189static int xbd_xenbus_match(device_t, cfdata_t, void *); 189static int xbd_xenbus_match(device_t, cfdata_t, void *);
190static void xbd_xenbus_attach(device_t, device_t, void *); 190static void xbd_xenbus_attach(device_t, device_t, void *);
191static int xbd_xenbus_detach(device_t, int); 191static int xbd_xenbus_detach(device_t, int);
192 192
193static bool xbd_xenbus_suspend(device_t, const pmf_qual_t *); 193static bool xbd_xenbus_suspend(device_t, const pmf_qual_t *);
194static bool xbd_xenbus_resume(device_t, const pmf_qual_t *); 194static bool xbd_xenbus_resume(device_t, const pmf_qual_t *);
195 195
196static int xbd_handler(void *); 196static int xbd_handler(void *);
197static int xbd_diskstart(device_t, struct buf *); 197static int xbd_diskstart(device_t, struct buf *);
198static void xbd_iosize(device_t, int *); 198static void xbd_iosize(device_t, int *);
199static void xbd_backend_changed(void *, XenbusState); 199static void xbd_backend_changed(void *, XenbusState);
200static void xbd_connect(struct xbd_xenbus_softc *); 200static void xbd_connect(struct xbd_xenbus_softc *);
201static void xbd_features(struct xbd_xenbus_softc *); 201static void xbd_features(struct xbd_xenbus_softc *);
202 202
203static void xbd_diskstart_submit(struct xbd_xenbus_softc *, int, 203static void xbd_diskstart_submit(struct xbd_xenbus_softc *, int,
204 struct buf *bp, int, bus_dmamap_t, grant_ref_t *); 204 struct buf *bp, int, bus_dmamap_t, grant_ref_t *);
205static void xbd_diskstart_submit_indirect(struct xbd_xenbus_softc *, 205static void xbd_diskstart_submit_indirect(struct xbd_xenbus_softc *,
206 struct xbd_req *, struct buf *bp); 206 struct xbd_req *, struct buf *bp);
207static int xbd_map_align(struct xbd_xenbus_softc *, struct xbd_req *); 207static int xbd_map_align(struct xbd_xenbus_softc *, struct xbd_req *);
208static void xbd_unmap_align(struct xbd_xenbus_softc *, struct xbd_req *, bool); 208static void xbd_unmap_align(struct xbd_xenbus_softc *, struct xbd_req *, bool);
209 209
210static void xbdminphys(struct buf *); 210static void xbdminphys(struct buf *);
211 211
212CFATTACH_DECL3_NEW(xbd, sizeof(struct xbd_xenbus_softc), 212CFATTACH_DECL3_NEW(xbd, sizeof(struct xbd_xenbus_softc),
213 xbd_xenbus_match, xbd_xenbus_attach, xbd_xenbus_detach, NULL, NULL, NULL, 213 xbd_xenbus_match, xbd_xenbus_attach, xbd_xenbus_detach, NULL, NULL, NULL,
214 DVF_DETACH_SHUTDOWN); 214 DVF_DETACH_SHUTDOWN);
215 215
216dev_type_open(xbdopen); 216dev_type_open(xbdopen);
217dev_type_close(xbdclose); 217dev_type_close(xbdclose);
218dev_type_read(xbdread); 218dev_type_read(xbdread);
219dev_type_write(xbdwrite); 219dev_type_write(xbdwrite);
220dev_type_ioctl(xbdioctl); 220dev_type_ioctl(xbdioctl);
221dev_type_strategy(xbdstrategy); 221dev_type_strategy(xbdstrategy);
222dev_type_dump(xbddump); 222dev_type_dump(xbddump);
223dev_type_size(xbdsize); 223dev_type_size(xbdsize);
224 224
225const struct bdevsw xbd_bdevsw = { 225const struct bdevsw xbd_bdevsw = {
226 .d_open = xbdopen, 226 .d_open = xbdopen,
227 .d_close = xbdclose, 227 .d_close = xbdclose,
228 .d_strategy = xbdstrategy, 228 .d_strategy = xbdstrategy,
229 .d_ioctl = xbdioctl, 229 .d_ioctl = xbdioctl,
230 .d_dump = xbddump, 230 .d_dump = xbddump,
231 .d_psize = xbdsize, 231 .d_psize = xbdsize,
232 .d_discard = nodiscard, 232 .d_discard = nodiscard,
233 .d_flag = D_DISK | D_MPSAFE 233 .d_flag = D_DISK | D_MPSAFE
234}; 234};
235 235
236const struct cdevsw xbd_cdevsw = { 236const struct cdevsw xbd_cdevsw = {
237 .d_open = xbdopen, 237 .d_open = xbdopen,
238 .d_close = xbdclose, 238 .d_close = xbdclose,
239 .d_read = xbdread, 239 .d_read = xbdread,
240 .d_write = xbdwrite, 240 .d_write = xbdwrite,
241 .d_ioctl = xbdioctl, 241 .d_ioctl = xbdioctl,
242 .d_stop = nostop, 242 .d_stop = nostop,
243 .d_tty = notty, 243 .d_tty = notty,
244 .d_poll = nopoll, 244 .d_poll = nopoll,
245 .d_mmap = nommap, 245 .d_mmap = nommap,
246 .d_kqfilter = nokqfilter, 246 .d_kqfilter = nokqfilter,
247 .d_discard = nodiscard, 247 .d_discard = nodiscard,
248 .d_flag = D_DISK | D_MPSAFE 248 .d_flag = D_DISK | D_MPSAFE
249}; 249};
250 250
251extern struct cfdriver xbd_cd; 251extern struct cfdriver xbd_cd;
252 252
253static struct dkdriver xbddkdriver = { 253static const struct dkdriver xbddkdriver = {
254 .d_strategy = xbdstrategy, 254 .d_strategy = xbdstrategy,
255 .d_minphys = xbdminphys, 255 .d_minphys = xbdminphys,
256 .d_open = xbdopen, 256 .d_open = xbdopen,
257 .d_close = xbdclose, 257 .d_close = xbdclose,
258 .d_diskstart = xbd_diskstart, 258 .d_diskstart = xbd_diskstart,
259 .d_iosize = xbd_iosize, 259 .d_iosize = xbd_iosize,
260}; 260};
261 261
262static int 262static int
263xbd_xenbus_match(device_t parent, cfdata_t match, void *aux) 263xbd_xenbus_match(device_t parent, cfdata_t match, void *aux)
264{ 264{
265 struct xenbusdev_attach_args *xa = aux; 265 struct xenbusdev_attach_args *xa = aux;
266 266
267 if (strcmp(xa->xa_type, "vbd") != 0) 267 if (strcmp(xa->xa_type, "vbd") != 0)
268 return 0; 268 return 0;
269 269
270 if (match->cf_loc[XENBUSCF_ID] != XENBUSCF_ID_DEFAULT && 270 if (match->cf_loc[XENBUSCF_ID] != XENBUSCF_ID_DEFAULT &&
271 match->cf_loc[XENBUSCF_ID] != xa->xa_id) 271 match->cf_loc[XENBUSCF_ID] != xa->xa_id)
272 return 0; 272 return 0;
273 273
274 return 1; 274 return 1;
275} 275}
276 276
277static void 277static void
278xbd_xenbus_attach(device_t parent, device_t self, void *aux) 278xbd_xenbus_attach(device_t parent, device_t self, void *aux)
279{ 279{
280 struct xbd_xenbus_softc *sc = device_private(self); 280 struct xbd_xenbus_softc *sc = device_private(self);
281 struct xenbusdev_attach_args *xa = aux; 281 struct xenbusdev_attach_args *xa = aux;
282 blkif_sring_t *ring; 282 blkif_sring_t *ring;
283 RING_IDX i; 283 RING_IDX i;
284#ifdef XBD_DEBUG 284#ifdef XBD_DEBUG
285 char **dir, *val; 285 char **dir, *val;
286 int dir_n = 0; 286 int dir_n = 0;
287 char id_str[20]; 287 char id_str[20];
288 int err; 288 int err;
289#endif 289#endif
290 290
291 config_pending_incr(self); 291 config_pending_incr(self);
292 aprint_normal(": Xen Virtual Block Device Interface\n"); 292 aprint_normal(": Xen Virtual Block Device Interface\n");
293 293
294 dk_init(&sc->sc_dksc, self, DKTYPE_ESDI); 294 dk_init(&sc->sc_dksc, self, DKTYPE_ESDI);
295 disk_init(&sc->sc_dksc.sc_dkdev, device_xname(self), &xbddkdriver); 295 disk_init(&sc->sc_dksc.sc_dkdev, device_xname(self), &xbddkdriver);
296 296
297 sc->sc_xbusd = xa->xa_xbusd; 297 sc->sc_xbusd = xa->xa_xbusd;
298 sc->sc_xbusd->xbusd_otherend_changed = xbd_backend_changed; 298 sc->sc_xbusd->xbusd_otherend_changed = xbd_backend_changed;
299 299
300 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_BIO); 300 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_BIO);
301 cv_init(&sc->sc_cache_flush_cv, "xbdsync"); 301 cv_init(&sc->sc_cache_flush_cv, "xbdsync");
302 cv_init(&sc->sc_req_cv, "xbdreq"); 302 cv_init(&sc->sc_req_cv, "xbdreq");
303 cv_init(&sc->sc_detach_cv, "xbddetach"); 303 cv_init(&sc->sc_detach_cv, "xbddetach");
304 cv_init(&sc->sc_suspend_cv, "xbdsuspend"); 304 cv_init(&sc->sc_suspend_cv, "xbdsuspend");
305 305
306 xbd_features(sc); 306 xbd_features(sc);
307 307
308 /* initialize free requests list */ 308 /* initialize free requests list */
309 SLIST_INIT(&sc->sc_xbdreq_head); 309 SLIST_INIT(&sc->sc_xbdreq_head);
310 for (i = 0; i < XBD_RING_SIZE; i++) { 310 for (i = 0; i < XBD_RING_SIZE; i++) {
311 sc->sc_reqs[i].req_id = i; 311 sc->sc_reqs[i].req_id = i;
312 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, &sc->sc_reqs[i], 312 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, &sc->sc_reqs[i],
313 req_next); 313 req_next);
314 } 314 }
315 315
316 if (sc->sc_features & BLKIF_FEATURE_INDIRECT) { 316 if (sc->sc_features & BLKIF_FEATURE_INDIRECT) {
317 /* initialize indirect page list */ 317 /* initialize indirect page list */
318 for (i = 0; i < XBD_RING_SIZE; i++) { 318 for (i = 0; i < XBD_RING_SIZE; i++) {
319 vmem_addr_t va; 319 vmem_addr_t va;
320 if (uvm_km_kmem_alloc(kmem_va_arena, 320 if (uvm_km_kmem_alloc(kmem_va_arena,
321 PAGE_SIZE, VM_SLEEP | VM_INSTANTFIT, &va) != 0) { 321 PAGE_SIZE, VM_SLEEP | VM_INSTANTFIT, &va) != 0) {
322 aprint_error_dev(self, 322 aprint_error_dev(self,
323 "can't alloc indirect pages\n"); 323 "can't alloc indirect pages\n");
324 return; 324 return;
325 } 325 }
326 sc->sc_indirect[i].in_addr = (void *)va; 326 sc->sc_indirect[i].in_addr = (void *)va;
327 SLIST_INSERT_HEAD(&sc->sc_indirect_head, 327 SLIST_INSERT_HEAD(&sc->sc_indirect_head,
328 &sc->sc_indirect[i], in_next); 328 &sc->sc_indirect[i], in_next);
329 } 329 }
330 } 330 }
331 331
332 sc->sc_backend_status = BLKIF_STATE_DISCONNECTED; 332 sc->sc_backend_status = BLKIF_STATE_DISCONNECTED;
333 sc->sc_shutdown = BLKIF_SHUTDOWN_REMOTE; 333 sc->sc_shutdown = BLKIF_SHUTDOWN_REMOTE;
334 334
335 ring = (void *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED); 335 ring = (void *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED);
336 if (ring == NULL) 336 if (ring == NULL)
337 panic("%s: can't alloc ring", device_xname(self)); 337 panic("%s: can't alloc ring", device_xname(self));
338 sc->sc_ring.sring = ring; 338 sc->sc_ring.sring = ring;
339 339
340 evcnt_attach_dynamic(&sc->sc_cnt_map_unalign, EVCNT_TYPE_MISC, 340 evcnt_attach_dynamic(&sc->sc_cnt_map_unalign, EVCNT_TYPE_MISC,
341 NULL, device_xname(self), "map unaligned"); 341 NULL, device_xname(self), "map unaligned");
342 evcnt_attach_dynamic(&sc->sc_cnt_unalign_busy, EVCNT_TYPE_MISC, 342 evcnt_attach_dynamic(&sc->sc_cnt_unalign_busy, EVCNT_TYPE_MISC,
343 NULL, device_xname(self), "map unaligned"); 343 NULL, device_xname(self), "map unaligned");
344 evcnt_attach_dynamic(&sc->sc_cnt_queue_full, EVCNT_TYPE_MISC, 344 evcnt_attach_dynamic(&sc->sc_cnt_queue_full, EVCNT_TYPE_MISC,
345 NULL, device_xname(self), "queue full"); 345 NULL, device_xname(self), "queue full");
346 evcnt_attach_dynamic(&sc->sc_cnt_indirect, EVCNT_TYPE_MISC, 346 evcnt_attach_dynamic(&sc->sc_cnt_indirect, EVCNT_TYPE_MISC,
347 NULL, device_xname(self), "indirect segment"); 347 NULL, device_xname(self), "indirect segment");
348 348
349 for (i = 0; i < XBD_RING_SIZE; i++) { 349 for (i = 0; i < XBD_RING_SIZE; i++) {
350 if (bus_dmamap_create(sc->sc_xbusd->xbusd_dmat, 350 if (bus_dmamap_create(sc->sc_xbusd->xbusd_dmat,
351 MAXPHYS, XBD_XFER_LIMIT >> PAGE_SHIFT, 351 MAXPHYS, XBD_XFER_LIMIT >> PAGE_SHIFT,
352 PAGE_SIZE, PAGE_SIZE, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 352 PAGE_SIZE, PAGE_SIZE, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
353 &sc->sc_reqs[i].req_dmamap) != 0) { 353 &sc->sc_reqs[i].req_dmamap) != 0) {
354 aprint_error_dev(self, "can't alloc dma maps\n"); 354 aprint_error_dev(self, "can't alloc dma maps\n");
355 return; 355 return;
356 } 356 }
357 } 357 }
358 358
359 if (uvm_km_kmem_alloc(kmem_va_arena, 359 if (uvm_km_kmem_alloc(kmem_va_arena,
360 MAXPHYS, VM_SLEEP | VM_INSTANTFIT, &sc->sc_unalign_buffer) != 0) { 360 MAXPHYS, VM_SLEEP | VM_INSTANTFIT, &sc->sc_unalign_buffer) != 0) {
361 aprint_error_dev(self, "can't alloc align buffer\n"); 361 aprint_error_dev(self, "can't alloc align buffer\n");
362 return; 362 return;
363 } 363 }
364 364
365 /* resume shared structures and tell backend that we are ready */ 365 /* resume shared structures and tell backend that we are ready */
366 if (xbd_xenbus_resume(self, PMF_Q_NONE) == false) { 366 if (xbd_xenbus_resume(self, PMF_Q_NONE) == false) {
367 uvm_km_free(kernel_map, (vaddr_t)ring, PAGE_SIZE, 367 uvm_km_free(kernel_map, (vaddr_t)ring, PAGE_SIZE,
368 UVM_KMF_WIRED); 368 UVM_KMF_WIRED);
369 return; 369 return;
370 } 370 }
371 371
372 if (!pmf_device_register(self, xbd_xenbus_suspend, xbd_xenbus_resume)) 372 if (!pmf_device_register(self, xbd_xenbus_suspend, xbd_xenbus_resume))
373 aprint_error_dev(self, "couldn't establish power handler\n"); 373 aprint_error_dev(self, "couldn't establish power handler\n");
374} 374}
375 375
376static int 376static int
377xbd_xenbus_detach(device_t dev, int flags) 377xbd_xenbus_detach(device_t dev, int flags)
378{ 378{
379 struct xbd_xenbus_softc *sc = device_private(dev); 379 struct xbd_xenbus_softc *sc = device_private(dev);
380 int bmaj, cmaj, i, mn, rc; 380 int bmaj, cmaj, i, mn, rc;
381 381
382 DPRINTF(("%s: xbd_detach\n", device_xname(dev))); 382 DPRINTF(("%s: xbd_detach\n", device_xname(dev)));
383 383
384 rc = disk_begindetach(&sc->sc_dksc.sc_dkdev, NULL, dev, flags); 384 rc = disk_begindetach(&sc->sc_dksc.sc_dkdev, NULL, dev, flags);
385 if (rc != 0) 385 if (rc != 0)
386 return rc; 386 return rc;
387 387
388 mutex_enter(&sc->sc_lock); 388 mutex_enter(&sc->sc_lock);
389 if (sc->sc_shutdown == BLKIF_SHUTDOWN_RUN) { 389 if (sc->sc_shutdown == BLKIF_SHUTDOWN_RUN) {
390 sc->sc_shutdown = BLKIF_SHUTDOWN_LOCAL; 390 sc->sc_shutdown = BLKIF_SHUTDOWN_LOCAL;
391 391
392 /* wait for requests to complete */ 392 /* wait for requests to complete */
393 while (sc->sc_backend_status == BLKIF_STATE_CONNECTED && 393 while (sc->sc_backend_status == BLKIF_STATE_CONNECTED &&
394 disk_isbusy(&sc->sc_dksc.sc_dkdev)) { 394 disk_isbusy(&sc->sc_dksc.sc_dkdev)) {
395 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2); 395 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2);
396 } 396 }
397 mutex_exit(&sc->sc_lock); 397 mutex_exit(&sc->sc_lock);
398 398
399 /* Trigger state transition with backend */ 399 /* Trigger state transition with backend */
400 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateClosing); 400 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateClosing);
401 401
402 mutex_enter(&sc->sc_lock); 402 mutex_enter(&sc->sc_lock);
403 } 403 }
404 if ((flags & DETACH_FORCE) == 0) { 404 if ((flags & DETACH_FORCE) == 0) {
405 /* xbd_xenbus_detach already in progress */ 405 /* xbd_xenbus_detach already in progress */
406 cv_broadcast(&sc->sc_detach_cv); 406 cv_broadcast(&sc->sc_detach_cv);
407 mutex_exit(&sc->sc_lock); 407 mutex_exit(&sc->sc_lock);
408 return EALREADY; 408 return EALREADY;
409 } 409 }
410 mutex_exit(&sc->sc_lock); 410 mutex_exit(&sc->sc_lock);
411 while (xenbus_read_driver_state(sc->sc_xbusd->xbusd_otherend) 411 while (xenbus_read_driver_state(sc->sc_xbusd->xbusd_otherend)
412 != XenbusStateClosed) { 412 != XenbusStateClosed) {
413 mutex_enter(&sc->sc_lock); 413 mutex_enter(&sc->sc_lock);
414 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2); 414 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2);
415 mutex_exit(&sc->sc_lock); 415 mutex_exit(&sc->sc_lock);
416 } 416 }
417 417
418 /* locate the major number */ 418 /* locate the major number */
419 bmaj = bdevsw_lookup_major(&xbd_bdevsw); 419 bmaj = bdevsw_lookup_major(&xbd_bdevsw);
420 cmaj = cdevsw_lookup_major(&xbd_cdevsw); 420 cmaj = cdevsw_lookup_major(&xbd_cdevsw);
421 421
422 /* Nuke the vnodes for any open instances. */ 422 /* Nuke the vnodes for any open instances. */
423 for (i = 0; i < MAXPARTITIONS; i++) { 423 for (i = 0; i < MAXPARTITIONS; i++) {
424 mn = DISKMINOR(device_unit(dev), i); 424 mn = DISKMINOR(device_unit(dev), i);
425 vdevgone(bmaj, mn, mn, VBLK); 425 vdevgone(bmaj, mn, mn, VBLK);
426 vdevgone(cmaj, mn, mn, VCHR); 426 vdevgone(cmaj, mn, mn, VCHR);
427 } 427 }
428 428
429 if (sc->sc_backend_status == BLKIF_STATE_CONNECTED) { 429 if (sc->sc_backend_status == BLKIF_STATE_CONNECTED) {
430 /* Delete all of our wedges. */ 430 /* Delete all of our wedges. */
431 dkwedge_delall(&sc->sc_dksc.sc_dkdev); 431 dkwedge_delall(&sc->sc_dksc.sc_dkdev);
432 432
433 /* Kill off any queued buffers. */ 433 /* Kill off any queued buffers. */
434 dk_drain(&sc->sc_dksc); 434 dk_drain(&sc->sc_dksc);
435 bufq_free(sc->sc_dksc.sc_bufq); 435 bufq_free(sc->sc_dksc.sc_bufq);
436 436
437 /* detach disk */ 437 /* detach disk */
438 disk_detach(&sc->sc_dksc.sc_dkdev); 438 disk_detach(&sc->sc_dksc.sc_dkdev);
439 disk_destroy(&sc->sc_dksc.sc_dkdev); 439 disk_destroy(&sc->sc_dksc.sc_dkdev);
440 dk_detach(&sc->sc_dksc); 440 dk_detach(&sc->sc_dksc);
441 } 441 }
442 442
443 hypervisor_mask_event(sc->sc_evtchn); 443 hypervisor_mask_event(sc->sc_evtchn);
444 xen_intr_disestablish(sc->sc_ih); 444 xen_intr_disestablish(sc->sc_ih);
445 445
446 mutex_enter(&sc->sc_lock); 446 mutex_enter(&sc->sc_lock);
447 while (xengnt_status(sc->sc_ring_gntref)) 447 while (xengnt_status(sc->sc_ring_gntref))
448 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2); 448 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2);
449 mutex_exit(&sc->sc_lock); 449 mutex_exit(&sc->sc_lock);
450 450
451 xengnt_revoke_access(sc->sc_ring_gntref); 451 xengnt_revoke_access(sc->sc_ring_gntref);
452 uvm_km_free(kernel_map, (vaddr_t)sc->sc_ring.sring, 452 uvm_km_free(kernel_map, (vaddr_t)sc->sc_ring.sring,
453 PAGE_SIZE, UVM_KMF_WIRED); 453 PAGE_SIZE, UVM_KMF_WIRED);
454 454
455 for (i = 0; i < XBD_RING_SIZE; i++) { 455 for (i = 0; i < XBD_RING_SIZE; i++) {
456 if (sc->sc_reqs[i].req_dmamap != NULL) { 456 if (sc->sc_reqs[i].req_dmamap != NULL) {
457 bus_dmamap_destroy(sc->sc_xbusd->xbusd_dmat, 457 bus_dmamap_destroy(sc->sc_xbusd->xbusd_dmat,
458 sc->sc_reqs[i].req_dmamap); 458 sc->sc_reqs[i].req_dmamap);
459 sc->sc_reqs[i].req_dmamap = NULL; 459 sc->sc_reqs[i].req_dmamap = NULL;
460 } 460 }
461 } 461 }
462 462
463 if (sc->sc_unalign_buffer != 0) { 463 if (sc->sc_unalign_buffer != 0) {
464 uvm_km_kmem_free(kmem_va_arena, sc->sc_unalign_buffer, MAXPHYS); 464 uvm_km_kmem_free(kmem_va_arena, sc->sc_unalign_buffer, MAXPHYS);
465 sc->sc_unalign_buffer = 0; 465 sc->sc_unalign_buffer = 0;
466 } 466 }
467 467
468 mutex_destroy(&sc->sc_lock); 468 mutex_destroy(&sc->sc_lock);
469 469
470 evcnt_detach(&sc->sc_cnt_map_unalign); 470 evcnt_detach(&sc->sc_cnt_map_unalign);
471 evcnt_detach(&sc->sc_cnt_unalign_busy); 471 evcnt_detach(&sc->sc_cnt_unalign_busy);
472 evcnt_detach(&sc->sc_cnt_queue_full); 472 evcnt_detach(&sc->sc_cnt_queue_full);
473 evcnt_detach(&sc->sc_cnt_indirect); 473 evcnt_detach(&sc->sc_cnt_indirect);
474 474
475 pmf_device_deregister(dev); 475 pmf_device_deregister(dev);
476 476
477 return 0; 477 return 0;
478} 478}
479 479
480static bool 480static bool
481xbd_xenbus_suspend(device_t dev, const pmf_qual_t *qual) { 481xbd_xenbus_suspend(device_t dev, const pmf_qual_t *qual) {
482 482
483 struct xbd_xenbus_softc *sc; 483 struct xbd_xenbus_softc *sc;
484 484
485 sc = device_private(dev); 485 sc = device_private(dev);
486 486
487 mutex_enter(&sc->sc_lock); 487 mutex_enter(&sc->sc_lock);
488 /* wait for requests to complete, then suspend device */ 488 /* wait for requests to complete, then suspend device */
489 while (sc->sc_backend_status == BLKIF_STATE_CONNECTED && 489 while (sc->sc_backend_status == BLKIF_STATE_CONNECTED &&
490 disk_isbusy(&sc->sc_dksc.sc_dkdev)) { 490 disk_isbusy(&sc->sc_dksc.sc_dkdev)) {
491 cv_timedwait(&sc->sc_suspend_cv, &sc->sc_lock, hz/2); 491 cv_timedwait(&sc->sc_suspend_cv, &sc->sc_lock, hz/2);
492 } 492 }
493 493
494 hypervisor_mask_event(sc->sc_evtchn); 494 hypervisor_mask_event(sc->sc_evtchn);
495 sc->sc_backend_status = BLKIF_STATE_SUSPENDED; 495 sc->sc_backend_status = BLKIF_STATE_SUSPENDED;
496 xen_intr_disestablish(sc->sc_ih); 496 xen_intr_disestablish(sc->sc_ih);
497 497
498 mutex_exit(&sc->sc_lock); 498 mutex_exit(&sc->sc_lock);
499 499
500 xenbus_device_suspend(sc->sc_xbusd); 500 xenbus_device_suspend(sc->sc_xbusd);
501 aprint_verbose_dev(dev, "removed event channel %d\n", sc->sc_evtchn); 501 aprint_verbose_dev(dev, "removed event channel %d\n", sc->sc_evtchn);
502 502
503 return true; 503 return true;
504} 504}
505 505
506static bool 506static bool
507xbd_xenbus_resume(device_t dev, const pmf_qual_t *qual) 507xbd_xenbus_resume(device_t dev, const pmf_qual_t *qual)
508{ 508{
509 struct xbd_xenbus_softc *sc; 509 struct xbd_xenbus_softc *sc;
510 struct xenbus_transaction *xbt; 510 struct xenbus_transaction *xbt;
511 int error; 511 int error;
512 blkif_sring_t *ring; 512 blkif_sring_t *ring;
513 paddr_t ma; 513 paddr_t ma;
514 const char *errmsg; 514 const char *errmsg;
515 515
516 sc = device_private(dev); 516 sc = device_private(dev);
517 517
518 if (sc->sc_backend_status == BLKIF_STATE_SUSPENDED) { 518 if (sc->sc_backend_status == BLKIF_STATE_SUSPENDED) {
519 /* 519 /*
520 * Device was suspended, so ensure that access associated to 520 * Device was suspended, so ensure that access associated to
521 * the block I/O ring is revoked. 521 * the block I/O ring is revoked.
522 */ 522 */
523 xengnt_revoke_access(sc->sc_ring_gntref); 523 xengnt_revoke_access(sc->sc_ring_gntref);
524 } 524 }
525 sc->sc_ring_gntref = GRANT_INVALID_REF; 525 sc->sc_ring_gntref = GRANT_INVALID_REF;
526 526
527 /* Initialize ring */ 527 /* Initialize ring */
528 ring = sc->sc_ring.sring; 528 ring = sc->sc_ring.sring;
529 memset(ring, 0, PAGE_SIZE); 529 memset(ring, 0, PAGE_SIZE);
530 SHARED_RING_INIT(ring); 530 SHARED_RING_INIT(ring);
531 FRONT_RING_INIT(&sc->sc_ring, ring, PAGE_SIZE); 531 FRONT_RING_INIT(&sc->sc_ring, ring, PAGE_SIZE);
532 532
533 /* 533 /*
534 * get MA address of the ring, and use it to set up the grant entry 534 * get MA address of the ring, and use it to set up the grant entry
535 * for the block device 535 * for the block device
536 */ 536 */
537 (void)pmap_extract_ma(pmap_kernel(), (vaddr_t)ring, &ma); 537 (void)pmap_extract_ma(pmap_kernel(), (vaddr_t)ring, &ma);
538 error = xenbus_grant_ring(sc->sc_xbusd, ma, &sc->sc_ring_gntref); 538 error = xenbus_grant_ring(sc->sc_xbusd, ma, &sc->sc_ring_gntref);
539 if (error) 539 if (error)
540 goto abort_resume; 540 goto abort_resume;
541 541
542 if (sc->sc_features & BLKIF_FEATURE_INDIRECT) { 542 if (sc->sc_features & BLKIF_FEATURE_INDIRECT) {
543 for (int i = 0; i < XBD_RING_SIZE; i++) { 543 for (int i = 0; i < XBD_RING_SIZE; i++) {
544 vaddr_t va = (vaddr_t)sc->sc_indirect[i].in_addr; 544 vaddr_t va = (vaddr_t)sc->sc_indirect[i].in_addr;
545 KASSERT(va != 0); 545 KASSERT(va != 0);
546 KASSERT((va & PAGE_MASK) == 0); 546 KASSERT((va & PAGE_MASK) == 0);
547 (void)pmap_extract_ma(pmap_kernel(), va, &ma); 547 (void)pmap_extract_ma(pmap_kernel(), va, &ma);
548 if (xengnt_grant_access( 548 if (xengnt_grant_access(
549 sc->sc_xbusd->xbusd_otherend_id, 549 sc->sc_xbusd->xbusd_otherend_id,
550 ma, true, &sc->sc_indirect[i].in_gntref)) { 550 ma, true, &sc->sc_indirect[i].in_gntref)) {
551 aprint_error_dev(dev, 551 aprint_error_dev(dev,
552 "indirect page grant failed\n"); 552 "indirect page grant failed\n");
553 goto abort_resume; 553 goto abort_resume;
554 } 554 }
555 } 555 }
556 } 556 }
557 557
558 error = xenbus_alloc_evtchn(sc->sc_xbusd, &sc->sc_evtchn); 558 error = xenbus_alloc_evtchn(sc->sc_xbusd, &sc->sc_evtchn);
559 if (error) 559 if (error)
560 goto abort_resume; 560 goto abort_resume;
561 561
562 aprint_verbose_dev(dev, "using event channel %d\n", 562 aprint_verbose_dev(dev, "using event channel %d\n",
563 sc->sc_evtchn); 563 sc->sc_evtchn);
564 sc->sc_ih = xen_intr_establish_xname(-1, &xen_pic, sc->sc_evtchn, 564 sc->sc_ih = xen_intr_establish_xname(-1, &xen_pic, sc->sc_evtchn,
565 IST_LEVEL, IPL_BIO, &xbd_handler, sc, true, device_xname(dev)); 565 IST_LEVEL, IPL_BIO, &xbd_handler, sc, true, device_xname(dev));
566 KASSERT(sc->sc_ih != NULL); 566 KASSERT(sc->sc_ih != NULL);
567 567
568again: 568again:
569 xbt = xenbus_transaction_start(); 569 xbt = xenbus_transaction_start();
570 if (xbt == NULL) 570 if (xbt == NULL)
571 return false; 571 return false;
572 error = xenbus_printf(xbt, sc->sc_xbusd->xbusd_path, 572 error = xenbus_printf(xbt, sc->sc_xbusd->xbusd_path,
573 "ring-ref","%u", sc->sc_ring_gntref); 573 "ring-ref","%u", sc->sc_ring_gntref);
574 if (error) { 574 if (error) {
575 errmsg = "writing ring-ref"; 575 errmsg = "writing ring-ref";
576 goto abort_transaction; 576 goto abort_transaction;
577 } 577 }
578 error = xenbus_printf(xbt, sc->sc_xbusd->xbusd_path, 578 error = xenbus_printf(xbt, sc->sc_xbusd->xbusd_path,
579 "event-channel", "%u", sc->sc_evtchn); 579 "event-channel", "%u", sc->sc_evtchn);
580 if (error) { 580 if (error) {
581 errmsg = "writing event channel"; 581 errmsg = "writing event channel";
582 goto abort_transaction; 582 goto abort_transaction;
583 } 583 }
584 error = xenbus_printf(xbt, sc->sc_xbusd->xbusd_path, 584 error = xenbus_printf(xbt, sc->sc_xbusd->xbusd_path,
585 "protocol", "%s", XEN_IO_PROTO_ABI_NATIVE); 585 "protocol", "%s", XEN_IO_PROTO_ABI_NATIVE);
586 if (error) { 586 if (error) {
587 errmsg = "writing protocol"; 587 errmsg = "writing protocol";
588 goto abort_transaction; 588 goto abort_transaction;
589 } 589 }
590 error = xenbus_transaction_end(xbt, 0); 590 error = xenbus_transaction_end(xbt, 0);
591 if (error == EAGAIN) 591 if (error == EAGAIN)
592 goto again; 592 goto again;
593 if (error != 0) { 593 if (error != 0) {
594 xenbus_dev_fatal(sc->sc_xbusd, error, 594 xenbus_dev_fatal(sc->sc_xbusd, error,
595 "completing transaction"); 595 "completing transaction");
596 return false; 596 return false;
597 } 597 }
598 598
599 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateInitialised); 599 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateInitialised);
600 600
601 if (sc->sc_backend_status == BLKIF_STATE_SUSPENDED) { 601 if (sc->sc_backend_status == BLKIF_STATE_SUSPENDED) {
602 /* 602 /*
603 * device was suspended, softc structures are 603 * device was suspended, softc structures are
604 * already initialized - we use a shortcut 604 * already initialized - we use a shortcut
605 */ 605 */
606 sc->sc_backend_status = BLKIF_STATE_CONNECTED; 606 sc->sc_backend_status = BLKIF_STATE_CONNECTED;
607 xenbus_device_resume(sc->sc_xbusd); 607 xenbus_device_resume(sc->sc_xbusd);
608 hypervisor_unmask_event(sc->sc_evtchn); 608 hypervisor_unmask_event(sc->sc_evtchn);
609 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateConnected); 609 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateConnected);
610 } 610 }
611 611
612 return true; 612 return true;
613 613
614abort_resume: 614abort_resume:
615 xenbus_dev_fatal(sc->sc_xbusd, error, "resuming device"); 615 xenbus_dev_fatal(sc->sc_xbusd, error, "resuming device");
616 return false; 616 return false;
617 617
618abort_transaction: 618abort_transaction:
619 xenbus_transaction_end(xbt, 1); 619 xenbus_transaction_end(xbt, 1);
620 xenbus_dev_fatal(sc->sc_xbusd, error, "%s", errmsg); 620 xenbus_dev_fatal(sc->sc_xbusd, error, "%s", errmsg);
621 return false; 621 return false;
622} 622}
623 623
624static void 624static void
625xbd_backend_changed(void *arg, XenbusState new_state) 625xbd_backend_changed(void *arg, XenbusState new_state)
626{ 626{
627 struct xbd_xenbus_softc *sc = device_private((device_t)arg); 627 struct xbd_xenbus_softc *sc = device_private((device_t)arg);
628 struct disk_geom *dg; 628 struct disk_geom *dg;
629 629
630 char buf[64]; 630 char buf[64];
631 DPRINTF(("%s: new backend state %d\n", 631 DPRINTF(("%s: new backend state %d\n",
632 device_xname(sc->sc_dksc.sc_dev), new_state)); 632 device_xname(sc->sc_dksc.sc_dev), new_state));
633 633
634 switch (new_state) { 634 switch (new_state) {
635 case XenbusStateUnknown: 635 case XenbusStateUnknown:
636 case XenbusStateInitialising: 636 case XenbusStateInitialising:
637 case XenbusStateInitWait: 637 case XenbusStateInitWait:
638 case XenbusStateInitialised: 638 case XenbusStateInitialised:
639 break; 639 break;
640 case XenbusStateClosing: 640 case XenbusStateClosing:
641 mutex_enter(&sc->sc_lock); 641 mutex_enter(&sc->sc_lock);
642 if (sc->sc_shutdown == BLKIF_SHUTDOWN_RUN) 642 if (sc->sc_shutdown == BLKIF_SHUTDOWN_RUN)
643 sc->sc_shutdown = BLKIF_SHUTDOWN_REMOTE; 643 sc->sc_shutdown = BLKIF_SHUTDOWN_REMOTE;
644 /* wait for requests to complete */ 644 /* wait for requests to complete */
645 while (sc->sc_backend_status == BLKIF_STATE_CONNECTED && 645 while (sc->sc_backend_status == BLKIF_STATE_CONNECTED &&
646 disk_isbusy(&sc->sc_dksc.sc_dkdev)) { 646 disk_isbusy(&sc->sc_dksc.sc_dkdev)) {
647 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2); 647 cv_timedwait(&sc->sc_detach_cv, &sc->sc_lock, hz/2);
648 } 648 }
649 mutex_exit(&sc->sc_lock); 649 mutex_exit(&sc->sc_lock);
650 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateClosed); 650 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateClosed);
651 break; 651 break;
652 case XenbusStateConnected: 652 case XenbusStateConnected:
653 /* 653 /*
654 * note that xbd_backend_changed() can only be called by 654 * note that xbd_backend_changed() can only be called by
655 * the xenbus thread. 655 * the xenbus thread.
656 */ 656 */
657 657
658 if (sc->sc_backend_status == BLKIF_STATE_CONNECTED || 658 if (sc->sc_backend_status == BLKIF_STATE_CONNECTED ||
659 sc->sc_backend_status == BLKIF_STATE_SUSPENDED) 659 sc->sc_backend_status == BLKIF_STATE_SUSPENDED)
660 /* already connected */ 660 /* already connected */
661 return; 661 return;
662 662
663 xbd_connect(sc); 663 xbd_connect(sc);
664 sc->sc_shutdown = BLKIF_SHUTDOWN_RUN; 664 sc->sc_shutdown = BLKIF_SHUTDOWN_RUN;
665 sc->sc_xbdsize = 665 sc->sc_xbdsize =
666 sc->sc_sectors * (uint64_t)sc->sc_secsize / DEV_BSIZE; 666 sc->sc_sectors * (uint64_t)sc->sc_secsize / DEV_BSIZE;
667 dg = &sc->sc_dksc.sc_dkdev.dk_geom; 667 dg = &sc->sc_dksc.sc_dkdev.dk_geom;
668 memset(dg, 0, sizeof(*dg));  668 memset(dg, 0, sizeof(*dg));
669 669
670 dg->dg_secperunit = sc->sc_xbdsize; 670 dg->dg_secperunit = sc->sc_xbdsize;
671 dg->dg_secsize = DEV_BSIZE; 671 dg->dg_secsize = DEV_BSIZE;
672 dg->dg_ntracks = 1; 672 dg->dg_ntracks = 1;
673 // XXX: Ok to hard-code DEV_BSIZE? 673 // XXX: Ok to hard-code DEV_BSIZE?
674 dg->dg_nsectors = 1024 * (1024 / dg->dg_secsize); 674 dg->dg_nsectors = 1024 * (1024 / dg->dg_secsize);
675 dg->dg_ncylinders = dg->dg_secperunit / dg->dg_nsectors; 675 dg->dg_ncylinders = dg->dg_secperunit / dg->dg_nsectors;
676 676
677 bufq_alloc(&sc->sc_dksc.sc_bufq, "fcfs", 0); 677 bufq_alloc(&sc->sc_dksc.sc_bufq, "fcfs", 0);
678 dk_attach(&sc->sc_dksc); 678 dk_attach(&sc->sc_dksc);
679 disk_attach(&sc->sc_dksc.sc_dkdev); 679 disk_attach(&sc->sc_dksc.sc_dkdev);
680 680
681 sc->sc_backend_status = BLKIF_STATE_CONNECTED; 681 sc->sc_backend_status = BLKIF_STATE_CONNECTED;
682 hypervisor_unmask_event(sc->sc_evtchn); 682 hypervisor_unmask_event(sc->sc_evtchn);
683 683
684 format_bytes(buf, sizeof(buf), sc->sc_sectors * sc->sc_secsize); 684 format_bytes(buf, sizeof(buf), sc->sc_sectors * sc->sc_secsize);
685 aprint_normal_dev(sc->sc_dksc.sc_dev, 685 aprint_normal_dev(sc->sc_dksc.sc_dev,
686 "%s, %d bytes/sect x %" PRIu64 " sectors\n", 686 "%s, %d bytes/sect x %" PRIu64 " sectors\n",
687 buf, (int)dg->dg_secsize, sc->sc_xbdsize); 687 buf, (int)dg->dg_secsize, sc->sc_xbdsize);
688 snprintb(buf, sizeof(buf), BLKIF_FEATURE_BITS, 688 snprintb(buf, sizeof(buf), BLKIF_FEATURE_BITS,
689 sc->sc_features); 689 sc->sc_features);
690 aprint_normal_dev(sc->sc_dksc.sc_dev, 690 aprint_normal_dev(sc->sc_dksc.sc_dev,
691 "backend features %s\n", buf); 691 "backend features %s\n", buf);
692 692
693 /* Discover wedges on this disk. */ 693 /* Discover wedges on this disk. */
694 dkwedge_discover(&sc->sc_dksc.sc_dkdev); 694 dkwedge_discover(&sc->sc_dksc.sc_dkdev);
695 695
696 disk_set_info(sc->sc_dksc.sc_dev, &sc->sc_dksc.sc_dkdev, NULL); 696 disk_set_info(sc->sc_dksc.sc_dev, &sc->sc_dksc.sc_dkdev, NULL);
697 697
698 /* the disk should be working now */ 698 /* the disk should be working now */
699 config_pending_decr(sc->sc_dksc.sc_dev); 699 config_pending_decr(sc->sc_dksc.sc_dev);
700 break; 700 break;
701 default: 701 default:
702 panic("bad backend state %d", new_state); 702 panic("bad backend state %d", new_state);
703 } 703 }
704} 704}
705 705
706static void 706static void
707xbd_connect(struct xbd_xenbus_softc *sc) 707xbd_connect(struct xbd_xenbus_softc *sc)
708{ 708{
709 int err; 709 int err;
710 unsigned long long sectors; 710 unsigned long long sectors;
711 711
712 err = xenbus_read_ul(NULL, 712 err = xenbus_read_ul(NULL,
713 sc->sc_xbusd->xbusd_path, "virtual-device", &sc->sc_handle, 10); 713 sc->sc_xbusd->xbusd_path, "virtual-device", &sc->sc_handle, 10);
714 if (err) 714 if (err)
715 panic("%s: can't read number from %s/virtual-device\n",  715 panic("%s: can't read number from %s/virtual-device\n",
716 device_xname(sc->sc_dksc.sc_dev), 716 device_xname(sc->sc_dksc.sc_dev),
717 sc->sc_xbusd->xbusd_otherend); 717 sc->sc_xbusd->xbusd_otherend);
718 err = xenbus_read_ull(NULL, 718 err = xenbus_read_ull(NULL,
719 sc->sc_xbusd->xbusd_otherend, "sectors", &sectors, 10); 719 sc->sc_xbusd->xbusd_otherend, "sectors", &sectors, 10);
720 if (err) 720 if (err)
721 panic("%s: can't read number from %s/sectors\n",  721 panic("%s: can't read number from %s/sectors\n",
722 device_xname(sc->sc_dksc.sc_dev), 722 device_xname(sc->sc_dksc.sc_dev),
723 sc->sc_xbusd->xbusd_otherend); 723 sc->sc_xbusd->xbusd_otherend);
724 sc->sc_sectors = sectors; 724 sc->sc_sectors = sectors;
725 725
726 err = xenbus_read_ul(NULL, 726 err = xenbus_read_ul(NULL,
727 sc->sc_xbusd->xbusd_otherend, "info", &sc->sc_info, 10); 727 sc->sc_xbusd->xbusd_otherend, "info", &sc->sc_info, 10);
728 if (err) 728 if (err)
729 panic("%s: can't read number from %s/info\n",  729 panic("%s: can't read number from %s/info\n",
730 device_xname(sc->sc_dksc.sc_dev), 730 device_xname(sc->sc_dksc.sc_dev),
731 sc->sc_xbusd->xbusd_otherend); 731 sc->sc_xbusd->xbusd_otherend);
732 err = xenbus_read_ul(NULL, 732 err = xenbus_read_ul(NULL,
733 sc->sc_xbusd->xbusd_otherend, "sector-size", &sc->sc_secsize, 10); 733 sc->sc_xbusd->xbusd_otherend, "sector-size", &sc->sc_secsize, 10);
734 if (err) 734 if (err)
735 panic("%s: can't read number from %s/sector-size\n",  735 panic("%s: can't read number from %s/sector-size\n",
736 device_xname(sc->sc_dksc.sc_dev), 736 device_xname(sc->sc_dksc.sc_dev),
737 sc->sc_xbusd->xbusd_otherend); 737 sc->sc_xbusd->xbusd_otherend);
738 738
739 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateConnected); 739 xenbus_switch_state(sc->sc_xbusd, NULL, XenbusStateConnected);
740} 740}
741 741
742static void 742static void
743xbd_features(struct xbd_xenbus_softc *sc) 743xbd_features(struct xbd_xenbus_softc *sc)
744{ 744{
745 int err; 745 int err;
746 u_long val; 746 u_long val;
747 747
748 err = xenbus_read_ul(NULL, sc->sc_xbusd->xbusd_otherend, 748 err = xenbus_read_ul(NULL, sc->sc_xbusd->xbusd_otherend,
749 "feature-flush-cache", &val, 10); 749 "feature-flush-cache", &val, 10);
750 if (err) 750 if (err)
751 val = 0; 751 val = 0;
752 if (val > 0) 752 if (val > 0)
753 sc->sc_features |= BLKIF_FEATURE_CACHE_FLUSH; 753 sc->sc_features |= BLKIF_FEATURE_CACHE_FLUSH;
754 754
755 err = xenbus_read_ul(NULL, sc->sc_xbusd->xbusd_otherend, 755 err = xenbus_read_ul(NULL, sc->sc_xbusd->xbusd_otherend,
756 "feature-barrier", &val, 10); 756 "feature-barrier", &val, 10);
757 if (err) 757 if (err)
758 val = 0; 758 val = 0;
759 if (val > 0) 759 if (val > 0)
760 sc->sc_features |= BLKIF_FEATURE_BARRIER; 760 sc->sc_features |= BLKIF_FEATURE_BARRIER;
761 761
762 err = xenbus_read_ul(NULL, sc->sc_xbusd->xbusd_otherend, 762 err = xenbus_read_ul(NULL, sc->sc_xbusd->xbusd_otherend,
763 "feature-persistent", &val, 10); 763 "feature-persistent", &val, 10);
764 if (err) 764 if (err)
765 val = 0; 765 val = 0;
766 if (val > 0) 766 if (val > 0)
767 sc->sc_features |= BLKIF_FEATURE_PERSISTENT; 767 sc->sc_features |= BLKIF_FEATURE_PERSISTENT;
768 768
769 err = xenbus_read_ul(NULL, sc->sc_xbusd->xbusd_otherend, 769 err = xenbus_read_ul(NULL, sc->sc_xbusd->xbusd_otherend,
770 "feature-max-indirect-segments", &val, 10); 770 "feature-max-indirect-segments", &val, 10);
771 if (err) 771 if (err)
772 val = 0; 772 val = 0;
773 if (val > (MAXPHYS >> PAGE_SHIFT)) { 773 if (val > (MAXPHYS >> PAGE_SHIFT)) {
774 /* We can use indirect segments, the limit is big enough */ 774 /* We can use indirect segments, the limit is big enough */
775 sc->sc_features |= BLKIF_FEATURE_INDIRECT; 775 sc->sc_features |= BLKIF_FEATURE_INDIRECT;
776 } 776 }
777} 777}
778 778
779static int 779static int
780xbd_handler(void *arg) 780xbd_handler(void *arg)
781{ 781{
782 struct xbd_xenbus_softc *sc = arg; 782 struct xbd_xenbus_softc *sc = arg;
783 struct buf *bp; 783 struct buf *bp;
784 RING_IDX resp_prod, i; 784 RING_IDX resp_prod, i;
785 int more_to_do; 785 int more_to_do;
786 int seg; 786 int seg;
787 grant_ref_t gntref; 787 grant_ref_t gntref;
788 788
789 DPRINTF(("xbd_handler(%s)\n", device_xname(sc->sc_dksc.sc_dev))); 789 DPRINTF(("xbd_handler(%s)\n", device_xname(sc->sc_dksc.sc_dev)));
790 790
791 if (__predict_false(sc->sc_backend_status != BLKIF_STATE_CONNECTED)) 791 if (__predict_false(sc->sc_backend_status != BLKIF_STATE_CONNECTED))
792 return 0; 792 return 0;
793 793
794 mutex_enter(&sc->sc_lock); 794 mutex_enter(&sc->sc_lock);
795again: 795again:
796 resp_prod = sc->sc_ring.sring->rsp_prod; 796 resp_prod = sc->sc_ring.sring->rsp_prod;
797 xen_rmb(); /* ensure we see replies up to resp_prod */ 797 xen_rmb(); /* ensure we see replies up to resp_prod */
798 for (i = sc->sc_ring.rsp_cons; i != resp_prod; i++) { 798 for (i = sc->sc_ring.rsp_cons; i != resp_prod; i++) {
799 blkif_response_t *rep = RING_GET_RESPONSE(&sc->sc_ring, i); 799 blkif_response_t *rep = RING_GET_RESPONSE(&sc->sc_ring, i);
800 struct xbd_req *xbdreq = &sc->sc_reqs[rep->id]; 800 struct xbd_req *xbdreq = &sc->sc_reqs[rep->id];
801 801
802 if (rep->operation == BLKIF_OP_FLUSH_DISKCACHE) { 802 if (rep->operation == BLKIF_OP_FLUSH_DISKCACHE) {
803 KASSERT(xbdreq->req_bp == NULL); 803 KASSERT(xbdreq->req_bp == NULL);
804 xbdreq->req_sync.s_error = rep->status; 804 xbdreq->req_sync.s_error = rep->status;
805 xbdreq->req_sync.s_done = 1; 805 xbdreq->req_sync.s_done = 1;
806 cv_broadcast(&sc->sc_cache_flush_cv); 806 cv_broadcast(&sc->sc_cache_flush_cv);
807 /* caller will free the req */ 807 /* caller will free the req */
808 continue; 808 continue;
809 } 809 }
810 810
811 if (rep->operation != BLKIF_OP_READ && 811 if (rep->operation != BLKIF_OP_READ &&
812 rep->operation != BLKIF_OP_WRITE) { 812 rep->operation != BLKIF_OP_WRITE) {
813 aprint_error_dev(sc->sc_dksc.sc_dev, 813 aprint_error_dev(sc->sc_dksc.sc_dev,
814 "bad operation %d from backend\n", rep->operation); 814 "bad operation %d from backend\n", rep->operation);
815 continue; 815 continue;
816 } 816 }
817 817
818 bp = xbdreq->req_bp; 818 bp = xbdreq->req_bp;
819 KASSERT(bp != NULL && bp->b_data != NULL); 819 KASSERT(bp != NULL && bp->b_data != NULL);
820 DPRINTF(("%s(%p): b_bcount = %ld\n", __func__, 820 DPRINTF(("%s(%p): b_bcount = %ld\n", __func__,
821 bp, (long)bp->b_bcount)); 821 bp, (long)bp->b_bcount));
822 822
823 if (bp->b_error != 0 || rep->status != BLKIF_RSP_OKAY) { 823 if (bp->b_error != 0 || rep->status != BLKIF_RSP_OKAY) {
824 bp->b_error = EIO; 824 bp->b_error = EIO;
825 bp->b_resid = bp->b_bcount; 825 bp->b_resid = bp->b_bcount;
826 } 826 }
827 827
828 if (xbdreq->req_parent) { 828 if (xbdreq->req_parent) {
829 struct xbd_req *req_parent = xbdreq->req_parent; 829 struct xbd_req *req_parent = xbdreq->req_parent;
830 830
831 /* Unhook and recycle child */ 831 /* Unhook and recycle child */
832 xbdreq->req_parent = NULL; 832 xbdreq->req_parent = NULL;
833 req_parent->req_child = NULL; 833 req_parent->req_child = NULL;
834 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, xbdreq, 834 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, xbdreq,
835 req_next); 835 req_next);
836 836
837 if (!req_parent->req_parent_done) { 837 if (!req_parent->req_parent_done) {
838 /* Finished before parent, nothig else to do */ 838 /* Finished before parent, nothig else to do */
839 continue; 839 continue;
840 } 840 }
841 841
842 /* Must do the cleanup now */ 842 /* Must do the cleanup now */
843 xbdreq = req_parent; 843 xbdreq = req_parent;
844 } 844 }
845 if (xbdreq->req_child) { 845 if (xbdreq->req_child) {
846 /* Finished before child, child will cleanup */ 846 /* Finished before child, child will cleanup */
847 xbdreq->req_parent_done = true; 847 xbdreq->req_parent_done = true;
848 continue; 848 continue;
849 } 849 }
850 850
851 if (bp->b_error == 0) 851 if (bp->b_error == 0)
852 bp->b_resid = 0; 852 bp->b_resid = 0;
853 853
854 for (seg = 0; seg < xbdreq->req_dmamap->dm_nsegs; seg++) { 854 for (seg = 0; seg < xbdreq->req_dmamap->dm_nsegs; seg++) {
855 /* 855 /*
856 * We are not allowing persistent mappings, so 856 * We are not allowing persistent mappings, so
857 * expect the backend to release the grant 857 * expect the backend to release the grant
858 * immediately. 858 * immediately.
859 */ 859 */
860 if (xbdreq->req_indirect) { 860 if (xbdreq->req_indirect) {
861 gntref = 861 gntref =
862 xbdreq->req_indirect->in_addr[seg].gref; 862 xbdreq->req_indirect->in_addr[seg].gref;
863 } else 863 } else
864 gntref = xbdreq->req_gntref[seg]; 864 gntref = xbdreq->req_gntref[seg];
865 KASSERT(xengnt_status(gntref) == 0); 865 KASSERT(xengnt_status(gntref) == 0);
866 xengnt_revoke_access(gntref); 866 xengnt_revoke_access(gntref);
867 } 867 }
868 868
869 bus_dmamap_unload(sc->sc_xbusd->xbusd_dmat, xbdreq->req_dmamap); 869 bus_dmamap_unload(sc->sc_xbusd->xbusd_dmat, xbdreq->req_dmamap);
870 870
871 if (__predict_false(bp->b_data != xbdreq->req_data)) 871 if (__predict_false(bp->b_data != xbdreq->req_data))
872 xbd_unmap_align(sc, xbdreq, true); 872 xbd_unmap_align(sc, xbdreq, true);
873 xbdreq->req_bp = xbdreq->req_data = NULL; 873 xbdreq->req_bp = xbdreq->req_data = NULL;
874 874
875 dk_done(&sc->sc_dksc, bp); 875 dk_done(&sc->sc_dksc, bp);
876 876
877 if (xbdreq->req_indirect) { 877 if (xbdreq->req_indirect) {
878 /* No persistent mappings, so check that 878 /* No persistent mappings, so check that
879 * backend unmapped the indirect segment grant too. 879 * backend unmapped the indirect segment grant too.
880 */ 880 */
881 KASSERT(xengnt_status(xbdreq->req_indirect->in_gntref) 881 KASSERT(xengnt_status(xbdreq->req_indirect->in_gntref)
882 == 0); 882 == 0);
883 SLIST_INSERT_HEAD(&sc->sc_indirect_head, 883 SLIST_INSERT_HEAD(&sc->sc_indirect_head,
884 xbdreq->req_indirect, in_next); 884 xbdreq->req_indirect, in_next);
885 xbdreq->req_indirect = NULL; 885 xbdreq->req_indirect = NULL;
886 } 886 }
887 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, xbdreq, req_next); 887 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, xbdreq, req_next);
888 } 888 }
889 889
890 xen_rmb(); 890 xen_rmb();
891 sc->sc_ring.rsp_cons = i; 891 sc->sc_ring.rsp_cons = i;
892 892
893 RING_FINAL_CHECK_FOR_RESPONSES(&sc->sc_ring, more_to_do); 893 RING_FINAL_CHECK_FOR_RESPONSES(&sc->sc_ring, more_to_do);
894 if (more_to_do) 894 if (more_to_do)
895 goto again; 895 goto again;
896 896
897 cv_signal(&sc->sc_req_cv); 897 cv_signal(&sc->sc_req_cv);
898 mutex_exit(&sc->sc_lock); 898 mutex_exit(&sc->sc_lock);
899 899
900 dk_start(&sc->sc_dksc, NULL); 900 dk_start(&sc->sc_dksc, NULL);
901 901
902 return 1; 902 return 1;
903} 903}
904 904
905static void 905static void
906xbdminphys(struct buf *bp) 906xbdminphys(struct buf *bp)
907{ 907{
908 if (bp->b_bcount > XBD_XFER_LIMIT) { 908 if (bp->b_bcount > XBD_XFER_LIMIT) {
909 bp->b_bcount = XBD_XFER_LIMIT; 909 bp->b_bcount = XBD_XFER_LIMIT;
910 } 910 }
911 minphys(bp); 911 minphys(bp);
912} 912}
913 913
914static void 914static void
915xbd_iosize(device_t dev, int *maxxfer) 915xbd_iosize(device_t dev, int *maxxfer)
916{ 916{
917 /* 917 /*
918 * Always restrict dumps to XBD_MAX_XFER to avoid indirect segments, 918 * Always restrict dumps to XBD_MAX_XFER to avoid indirect segments,
919 * so that it uses as little memory as possible.  919 * so that it uses as little memory as possible.
920 */ 920 */
921 if (*maxxfer > XBD_MAX_XFER) 921 if (*maxxfer > XBD_MAX_XFER)
922 *maxxfer = XBD_MAX_XFER; 922 *maxxfer = XBD_MAX_XFER;
923} 923}
924 924
925int 925int
926xbdopen(dev_t dev, int flags, int fmt, struct lwp *l) 926xbdopen(dev_t dev, int flags, int fmt, struct lwp *l)
927{ 927{
928 struct xbd_xenbus_softc *sc; 928 struct xbd_xenbus_softc *sc;
929 929
930 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev)); 930 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev));
931 if (sc == NULL) 931 if (sc == NULL)
932 return (ENXIO); 932 return (ENXIO);
933 if ((flags & FWRITE) && (sc->sc_info & VDISK_READONLY)) 933 if ((flags & FWRITE) && (sc->sc_info & VDISK_READONLY))
934 return EROFS; 934 return EROFS;
935 935
936 DPRINTF(("xbdopen(0x%04x, %d)\n", dev, flags)); 936 DPRINTF(("xbdopen(0x%04x, %d)\n", dev, flags));
937 return dk_open(&sc->sc_dksc, dev, flags, fmt, l); 937 return dk_open(&sc->sc_dksc, dev, flags, fmt, l);
938} 938}
939 939
940int 940int
941xbdclose(dev_t dev, int flags, int fmt, struct lwp *l) 941xbdclose(dev_t dev, int flags, int fmt, struct lwp *l)
942{ 942{
943 struct xbd_xenbus_softc *sc; 943 struct xbd_xenbus_softc *sc;
944 944
945 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev)); 945 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev));
946 946
947 DPRINTF(("xbdclose(%d, %d)\n", dev, flags)); 947 DPRINTF(("xbdclose(%d, %d)\n", dev, flags));
948 return dk_close(&sc->sc_dksc, dev, flags, fmt, l); 948 return dk_close(&sc->sc_dksc, dev, flags, fmt, l);
949} 949}
950 950
951void 951void
952xbdstrategy(struct buf *bp) 952xbdstrategy(struct buf *bp)
953{ 953{
954 struct xbd_xenbus_softc *sc; 954 struct xbd_xenbus_softc *sc;
955 955
956 sc = device_lookup_private(&xbd_cd, DISKUNIT(bp->b_dev)); 956 sc = device_lookup_private(&xbd_cd, DISKUNIT(bp->b_dev));
957 957
958 DPRINTF(("xbdstrategy(%p): b_bcount = %ld\n", bp, 958 DPRINTF(("xbdstrategy(%p): b_bcount = %ld\n", bp,
959 (long)bp->b_bcount)); 959 (long)bp->b_bcount));
960 960
961 if (sc == NULL || sc->sc_shutdown != BLKIF_SHUTDOWN_RUN) { 961 if (sc == NULL || sc->sc_shutdown != BLKIF_SHUTDOWN_RUN) {
962 bp->b_error = EIO; 962 bp->b_error = EIO;
963 biodone(bp); 963 biodone(bp);
964 return; 964 return;
965 } 965 }
966 if (__predict_false((sc->sc_info & VDISK_READONLY) && 966 if (__predict_false((sc->sc_info & VDISK_READONLY) &&
967 (bp->b_flags & B_READ) == 0)) { 967 (bp->b_flags & B_READ) == 0)) {
968 bp->b_error = EROFS; 968 bp->b_error = EROFS;
969 biodone(bp); 969 biodone(bp);
970 return; 970 return;
971 } 971 }
972 972
973 dk_strategy(&sc->sc_dksc, bp); 973 dk_strategy(&sc->sc_dksc, bp);
974 return; 974 return;
975} 975}
976 976
977int 977int
978xbdsize(dev_t dev) 978xbdsize(dev_t dev)
979{ 979{
980 struct xbd_xenbus_softc *sc; 980 struct xbd_xenbus_softc *sc;
981 981
982 DPRINTF(("xbdsize(%d)\n", dev)); 982 DPRINTF(("xbdsize(%d)\n", dev));
983 983
984 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev)); 984 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev));
985 if (sc == NULL || sc->sc_shutdown != BLKIF_SHUTDOWN_RUN) 985 if (sc == NULL || sc->sc_shutdown != BLKIF_SHUTDOWN_RUN)
986 return -1; 986 return -1;
987 return dk_size(&sc->sc_dksc, dev); 987 return dk_size(&sc->sc_dksc, dev);
988} 988}
989 989
990int 990int
991xbdread(dev_t dev, struct uio *uio, int flags) 991xbdread(dev_t dev, struct uio *uio, int flags)
992{ 992{
993 struct xbd_xenbus_softc *sc =  993 struct xbd_xenbus_softc *sc =
994 device_lookup_private(&xbd_cd, DISKUNIT(dev)); 994 device_lookup_private(&xbd_cd, DISKUNIT(dev));
995 struct dk_softc *dksc = &sc->sc_dksc; 995 struct dk_softc *dksc = &sc->sc_dksc;
996 996
997 if (!DK_ATTACHED(dksc)) 997 if (!DK_ATTACHED(dksc))
998 return ENXIO; 998 return ENXIO;
999 return physio(xbdstrategy, NULL, dev, B_READ, xbdminphys, uio); 999 return physio(xbdstrategy, NULL, dev, B_READ, xbdminphys, uio);
1000} 1000}
1001 1001
1002int 1002int
1003xbdwrite(dev_t dev, struct uio *uio, int flags) 1003xbdwrite(dev_t dev, struct uio *uio, int flags)
1004{ 1004{
1005 struct xbd_xenbus_softc *sc = 1005 struct xbd_xenbus_softc *sc =
1006 device_lookup_private(&xbd_cd, DISKUNIT(dev)); 1006 device_lookup_private(&xbd_cd, DISKUNIT(dev));
1007 struct dk_softc *dksc = &sc->sc_dksc; 1007 struct dk_softc *dksc = &sc->sc_dksc;
1008 1008
1009 if (!DK_ATTACHED(dksc)) 1009 if (!DK_ATTACHED(dksc))
1010 return ENXIO; 1010 return ENXIO;
1011 if (__predict_false(sc->sc_info & VDISK_READONLY)) 1011 if (__predict_false(sc->sc_info & VDISK_READONLY))
1012 return EROFS; 1012 return EROFS;
1013 return physio(xbdstrategy, NULL, dev, B_WRITE, xbdminphys, uio); 1013 return physio(xbdstrategy, NULL, dev, B_WRITE, xbdminphys, uio);
1014} 1014}
1015 1015
1016int 1016int
1017xbdioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l) 1017xbdioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
1018{ 1018{
1019 struct xbd_xenbus_softc *sc = 1019 struct xbd_xenbus_softc *sc =
1020 device_lookup_private(&xbd_cd, DISKUNIT(dev)); 1020 device_lookup_private(&xbd_cd, DISKUNIT(dev));
1021 struct dk_softc *dksc; 1021 struct dk_softc *dksc;
1022 int error; 1022 int error;
1023 struct xbd_req *xbdreq; 1023 struct xbd_req *xbdreq;
1024 blkif_request_t *req; 1024 blkif_request_t *req;
1025 int notify; 1025 int notify;
1026 1026
1027 DPRINTF(("xbdioctl(%d, %08lx, %p, %d, %p)\n", 1027 DPRINTF(("xbdioctl(%d, %08lx, %p, %d, %p)\n",
1028 dev, cmd, data, flag, l)); 1028 dev, cmd, data, flag, l));
1029 dksc = &sc->sc_dksc; 1029 dksc = &sc->sc_dksc;
1030 1030
1031 switch (cmd) { 1031 switch (cmd) {
1032 case DIOCGCACHE: 1032 case DIOCGCACHE:
1033 { 1033 {
1034 /* Assume there is write cache if cache-flush is supported */ 1034 /* Assume there is write cache if cache-flush is supported */
1035 int *bitsp = (int *)data; 1035 int *bitsp = (int *)data;
1036 *bitsp = 0; 1036 *bitsp = 0;
1037 if (sc->sc_features & BLKIF_FEATURE_CACHE_FLUSH) 1037 if (sc->sc_features & BLKIF_FEATURE_CACHE_FLUSH)
1038 *bitsp |= DKCACHE_WRITE; 1038 *bitsp |= DKCACHE_WRITE;
1039 error = 0; 1039 error = 0;
1040 break; 1040 break;
1041 } 1041 }
1042 case DIOCCACHESYNC: 1042 case DIOCCACHESYNC:
1043 if ((sc->sc_features & BLKIF_FEATURE_CACHE_FLUSH) == 0) 1043 if ((sc->sc_features & BLKIF_FEATURE_CACHE_FLUSH) == 0)
1044 return EOPNOTSUPP; 1044 return EOPNOTSUPP;
1045 1045
1046 mutex_enter(&sc->sc_lock); 1046 mutex_enter(&sc->sc_lock);
1047 while ((xbdreq = SLIST_FIRST(&sc->sc_xbdreq_head)) == NULL) 1047 while ((xbdreq = SLIST_FIRST(&sc->sc_xbdreq_head)) == NULL)
1048 cv_wait(&sc->sc_req_cv, &sc->sc_lock); 1048 cv_wait(&sc->sc_req_cv, &sc->sc_lock);
1049 1049
1050 SLIST_REMOVE_HEAD(&sc->sc_xbdreq_head, req_next); 1050 SLIST_REMOVE_HEAD(&sc->sc_xbdreq_head, req_next);
1051 req = RING_GET_REQUEST(&sc->sc_ring, 1051 req = RING_GET_REQUEST(&sc->sc_ring,
1052 sc->sc_ring.req_prod_pvt); 1052 sc->sc_ring.req_prod_pvt);
1053 req->id = xbdreq->req_id; 1053 req->id = xbdreq->req_id;
1054 req->operation = BLKIF_OP_FLUSH_DISKCACHE; 1054 req->operation = BLKIF_OP_FLUSH_DISKCACHE;
1055 req->handle = sc->sc_handle; 1055 req->handle = sc->sc_handle;
1056 xbdreq->req_sync.s_done = 0; 1056 xbdreq->req_sync.s_done = 0;
1057 sc->sc_ring.req_prod_pvt++; 1057 sc->sc_ring.req_prod_pvt++;
1058 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&sc->sc_ring, notify); 1058 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&sc->sc_ring, notify);
1059 if (notify) 1059 if (notify)
1060 hypervisor_notify_via_evtchn(sc->sc_evtchn); 1060 hypervisor_notify_via_evtchn(sc->sc_evtchn);
1061 /* request sent, now wait for completion */ 1061 /* request sent, now wait for completion */
1062 while (xbdreq->req_sync.s_done == 0) 1062 while (xbdreq->req_sync.s_done == 0)
1063 cv_wait(&sc->sc_cache_flush_cv, &sc->sc_lock); 1063 cv_wait(&sc->sc_cache_flush_cv, &sc->sc_lock);
1064 1064
1065 if (xbdreq->req_sync.s_error == BLKIF_RSP_EOPNOTSUPP) 1065 if (xbdreq->req_sync.s_error == BLKIF_RSP_EOPNOTSUPP)
1066 error = EOPNOTSUPP; 1066 error = EOPNOTSUPP;
1067 else if (xbdreq->req_sync.s_error == BLKIF_RSP_OKAY) 1067 else if (xbdreq->req_sync.s_error == BLKIF_RSP_OKAY)
1068 error = 0; 1068 error = 0;
1069 else 1069 else
1070 error = EIO; 1070 error = EIO;
1071 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, xbdreq, req_next); 1071 SLIST_INSERT_HEAD(&sc->sc_xbdreq_head, xbdreq, req_next);
1072 cv_signal(&sc->sc_req_cv); 1072 cv_signal(&sc->sc_req_cv);
1073 mutex_exit(&sc->sc_lock); 1073 mutex_exit(&sc->sc_lock);
1074 1074
1075 /* Restart I/O if it was waiting for req */ 1075 /* Restart I/O if it was waiting for req */
1076 dk_start(&sc->sc_dksc, NULL); 1076 dk_start(&sc->sc_dksc, NULL);
1077 break; 1077 break;
1078 1078
1079 default: 1079 default:
1080 error = dk_ioctl(dksc, dev, cmd, data, flag, l); 1080 error = dk_ioctl(dksc, dev, cmd, data, flag, l);
1081 break; 1081 break;
1082 } 1082 }
1083 1083
1084 return error; 1084 return error;
1085} 1085}
1086 1086
1087int 1087int
1088xbddump(dev_t dev, daddr_t blkno, void *va, size_t size) 1088xbddump(dev_t dev, daddr_t blkno, void *va, size_t size)
1089{ 1089{
1090 struct xbd_xenbus_softc *sc; 1090 struct xbd_xenbus_softc *sc;
1091 1091
1092 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev)); 1092 sc = device_lookup_private(&xbd_cd, DISKUNIT(dev));
1093 if (sc == NULL) 1093 if (sc == NULL)
1094 return (ENXIO); 1094 return (ENXIO);
1095 1095
1096 DPRINTF(("xbddump(%d, %" PRId64 ", %p, %lu)\n", dev, blkno, va, 1096 DPRINTF(("xbddump(%d, %" PRId64 ", %p, %lu)\n", dev, blkno, va,
1097 (unsigned long)size)); 1097 (unsigned long)size));
1098 return dk_dump(&sc->sc_dksc, dev, blkno, va, size, 0); 1098 return dk_dump(&sc->sc_dksc, dev, blkno, va, size, 0);
1099} 1099}
1100 1100
1101static int 1101static int
1102xbd_diskstart(device_t self, struct buf *bp) 1102xbd_diskstart(device_t self, struct buf *bp)
1103{ 1103{
1104 struct xbd_xenbus_softc *sc = device_private(self); 1104 struct xbd_xenbus_softc *sc = device_private(self);
1105 struct xbd_req *xbdreq; 1105 struct xbd_req *xbdreq;
1106 int error = 0; 1106 int error = 0;
1107 int notify; 1107 int notify;
1108 1108
1109 KASSERT(bp->b_bcount <= MAXPHYS); 1109 KASSERT(bp->b_bcount <= MAXPHYS);
1110 1110
1111 DPRINTF(("xbd_diskstart(%p): b_bcount = %ld\n", 1111 DPRINTF(("xbd_diskstart(%p): b_bcount = %ld\n",
1112 bp, (long)bp->b_bcount)); 1112 bp, (long)bp->b_bcount));
1113 1113
1114 mutex_enter(&sc->sc_lock); 1114 mutex_enter(&sc->sc_lock);
1115 1115
1116 if (sc->sc_shutdown != BLKIF_SHUTDOWN_RUN) { 1116 if (sc->sc_shutdown != BLKIF_SHUTDOWN_RUN) {
1117 error = EIO; 1117 error = EIO;
1118 goto out; 1118 goto out;
1119 } 1119 }
1120 1120
1121 if (bp->b_rawblkno < 0 || bp->b_rawblkno > sc->sc_xbdsize) { 1121 if (bp->b_rawblkno < 0 || bp->b_rawblkno > sc->sc_xbdsize) {
1122 /* invalid block number */ 1122 /* invalid block number */
1123 error = EINVAL; 1123 error = EINVAL;
1124 goto out; 1124 goto out;
1125 } 1125 }
1126 1126
1127 if (__predict_false( 1127 if (__predict_false(
1128 sc->sc_backend_status == BLKIF_STATE_SUSPENDED)) { 1128 sc->sc_backend_status == BLKIF_STATE_SUSPENDED)) {
1129 /* device is suspended, do not consume buffer */ 1129 /* device is suspended, do not consume buffer */
1130 DPRINTF(("%s: (xbd_diskstart) device suspended\n", 1130 DPRINTF(("%s: (xbd_diskstart) device suspended\n",
1131 sc->sc_dksc.sc_xname)); 1131 sc->sc_dksc.sc_xname));
1132 error = EAGAIN; 1132 error = EAGAIN;
1133 goto out; 1133 goto out;
1134 } 1134 }
1135 1135
1136 xbdreq = SLIST_FIRST(&sc->sc_xbdreq_head); 1136 xbdreq = SLIST_FIRST(&sc->sc_xbdreq_head);
1137 if (__predict_false(xbdreq == NULL)) { 1137 if (__predict_false(xbdreq == NULL)) {
1138 sc->sc_cnt_queue_full.ev_count++; 1138 sc->sc_cnt_queue_full.ev_count++;
1139 DPRINTF(("xbd_diskstart: no req\n")); 1139 DPRINTF(("xbd_diskstart: no req\n"));
1140 error = EAGAIN; 1140 error = EAGAIN;
1141 goto out; 1141 goto out;
1142 } 1142 }
1143 1143
1144 if ((sc->sc_features & BLKIF_FEATURE_INDIRECT) == 0 1144 if ((sc->sc_features & BLKIF_FEATURE_INDIRECT) == 0
1145 && bp->b_bcount > XBD_MAX_CHUNK) { 1145 && bp->b_bcount > XBD_MAX_CHUNK) {
1146 if (!SLIST_NEXT(xbdreq, req_next)) { 1146 if (!SLIST_NEXT(xbdreq, req_next)) {
1147 DPRINTF(("%s: need extra req\n", __func__)); 1147 DPRINTF(("%s: need extra req\n", __func__));
1148 error = EAGAIN; 1148 error = EAGAIN;
1149 goto out; 1149 goto out;
1150 } 1150 }
1151 } 1151 }
1152 1152
1153 bp->b_resid = bp->b_bcount; 1153 bp->b_resid = bp->b_bcount;
1154 xbdreq->req_bp = bp; 1154 xbdreq->req_bp = bp;
1155 xbdreq->req_data = bp->b_data; 1155 xbdreq->req_data = bp->b_data;
1156 if (__predict_false((vaddr_t)bp->b_data & (XEN_BSIZE - 1))) { 1156 if (__predict_false((vaddr_t)bp->b_data & (XEN_BSIZE - 1))) {
1157 if (__predict_false(xbd_map_align(sc, xbdreq) != 0)) { 1157 if (__predict_false(xbd_map_align(sc, xbdreq) != 0)) {
1158 DPRINTF(("xbd_diskstart: no align\n")); 1158 DPRINTF(("xbd_diskstart: no align\n"));
1159 error = EAGAIN; 1159 error = EAGAIN;
1160 goto out; 1160 goto out;
1161 } 1161 }
1162 } 1162 }
1163 1163
1164 if (__predict_false(bus_dmamap_load(sc->sc_xbusd->xbusd_dmat, 1164 if (__predict_false(bus_dmamap_load(sc->sc_xbusd->xbusd_dmat,
1165 xbdreq->req_dmamap, xbdreq->req_data, bp->b_bcount, NULL, 1165 xbdreq->req_dmamap, xbdreq->req_data, bp->b_bcount, NULL,
1166 BUS_DMA_NOWAIT) != 0)) { 1166 BUS_DMA_NOWAIT) != 0)) {
1167 printf("%s: %s: bus_dmamap_load failed\n", 1167 printf("%s: %s: bus_dmamap_load failed\n",
1168 device_xname(sc->sc_dksc.sc_dev), __func__); 1168 device_xname(sc->sc_dksc.sc_dev), __func__);
1169 if (__predict_false(bp->b_data != xbdreq->req_data)) 1169 if (__predict_false(bp->b_data != xbdreq->req_data))
1170 xbd_unmap_align(sc, xbdreq, false); 1170 xbd_unmap_align(sc, xbdreq, false);
1171 error = EINVAL; 1171 error = EINVAL;
1172 goto out; 1172 goto out;
1173 } 1173 }
1174 1174
1175 for (int seg = 0; seg < xbdreq->req_dmamap->dm_nsegs; seg++) { 1175 for (int seg = 0; seg < xbdreq->req_dmamap->dm_nsegs; seg++) {
1176 KASSERT(seg < __arraycount(xbdreq->req_gntref)); 1176 KASSERT(seg < __arraycount(xbdreq->req_gntref));
1177 1177
1178 paddr_t ma = xbdreq->req_dmamap->dm_segs[seg].ds_addr; 1178 paddr_t ma = xbdreq->req_dmamap->dm_segs[seg].ds_addr;
1179 if (__predict_false(xengnt_grant_access( 1179 if (__predict_false(xengnt_grant_access(
1180 sc->sc_xbusd->xbusd_otherend_id, 1180 sc->sc_xbusd->xbusd_otherend_id,
1181 (ma & ~PAGE_MASK), (bp->b_flags & B_READ) == 0, 1181 (ma & ~PAGE_MASK), (bp->b_flags & B_READ) == 0,
1182 &xbdreq->req_gntref[seg]))) { 1182 &xbdreq->req_gntref[seg]))) {
1183 printf("%s: %s: xengnt_grant_access failed\n", 1183 printf("%s: %s: xengnt_grant_access failed\n",
1184 device_xname(sc->sc_dksc.sc_dev), __func__); 1184 device_xname(sc->sc_dksc.sc_dev), __func__);
1185 if (seg > 0) { 1185 if (seg > 0) {
1186 for (; --seg >= 0; ) { 1186 for (; --seg >= 0; ) {
1187 xengnt_revoke_access( 1187 xengnt_revoke_access(
1188 xbdreq->req_gntref[seg]); 1188 xbdreq->req_gntref[seg]);
1189 } 1189 }
1190 } 1190 }
1191 bus_dmamap_unload(sc->sc_xbusd->xbusd_dmat, 1191 bus_dmamap_unload(sc->sc_xbusd->xbusd_dmat,
1192 xbdreq->req_dmamap); 1192 xbdreq->req_dmamap);
1193 if (__predict_false(bp->b_data != xbdreq->req_data)) 1193 if (__predict_false(bp->b_data != xbdreq->req_data))
1194 xbd_unmap_align(sc, xbdreq, false); 1194 xbd_unmap_align(sc, xbdreq, false);
1195 error = EAGAIN; 1195 error = EAGAIN;
1196 goto out; 1196 goto out;
1197 } 1197 }
1198 } 1198 }
1199 1199
1200 KASSERT(xbdreq->req_parent == NULL); 1200 KASSERT(xbdreq->req_parent == NULL);
1201 KASSERT(xbdreq->req_child == NULL); 1201 KASSERT(xbdreq->req_child == NULL);
1202 1202
1203 /* We are now committed to the transfer */ 1203 /* We are now committed to the transfer */
1204 SLIST_REMOVE_HEAD(&sc->sc_xbdreq_head, req_next); 1204 SLIST_REMOVE_HEAD(&sc->sc_xbdreq_head, req_next);
1205 1205
1206 if ((sc->sc_features & BLKIF_FEATURE_INDIRECT) != 0 && 1206 if ((sc->sc_features & BLKIF_FEATURE_INDIRECT) != 0 &&
1207 bp->b_bcount > XBD_MAX_CHUNK) { 1207 bp->b_bcount > XBD_MAX_CHUNK) {
1208 xbd_diskstart_submit_indirect(sc, xbdreq, bp); 1208 xbd_diskstart_submit_indirect(sc, xbdreq, bp);
1209 goto push; 1209 goto push;
1210 } 1210 }
1211 1211
1212 xbd_diskstart_submit(sc, xbdreq->req_id, 1212 xbd_diskstart_submit(sc, xbdreq->req_id,
1213 bp, 0, xbdreq->req_dmamap, xbdreq->req_gntref); 1213 bp, 0, xbdreq->req_dmamap, xbdreq->req_gntref);
1214 1214
1215 if (bp->b_bcount > XBD_MAX_CHUNK) { 1215 if (bp->b_bcount > XBD_MAX_CHUNK) {
1216 struct xbd_req *xbdreq2 = SLIST_FIRST(&sc->sc_xbdreq_head); 1216 struct xbd_req *xbdreq2 = SLIST_FIRST(&sc->sc_xbdreq_head);
1217 KASSERT(xbdreq2 != NULL); /* Checked earlier */ 1217 KASSERT(xbdreq2 != NULL); /* Checked earlier */
1218 SLIST_REMOVE_HEAD(&sc->sc_xbdreq_head, req_next); 1218 SLIST_REMOVE_HEAD(&sc->sc_xbdreq_head, req_next);
1219 xbdreq->req_child = xbdreq2; 1219 xbdreq->req_child = xbdreq2;
1220 xbdreq->req_parent_done = false; 1220 xbdreq->req_parent_done = false;
1221 xbdreq2->req_parent = xbdreq; 1221 xbdreq2->req_parent = xbdreq;
1222 xbdreq2->req_bp = bp; 1222 xbdreq2->req_bp = bp;
1223 xbdreq2->req_data = NULL; 1223 xbdreq2->req_data = NULL;
1224 xbd_diskstart_submit(sc, xbdreq2->req_id, 1224 xbd_diskstart_submit(sc, xbdreq2->req_id,
1225 bp, XBD_MAX_CHUNK, xbdreq->req_dmamap, 1225 bp, XBD_MAX_CHUNK, xbdreq->req_dmamap,
1226 xbdreq->req_gntref); 1226 xbdreq->req_gntref);
1227 } 1227 }
1228 1228
1229push: 1229push:
1230 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&sc->sc_ring, notify); 1230 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&sc->sc_ring, notify);
1231 if (notify) 1231 if (notify)
1232 hypervisor_notify_via_evtchn(sc->sc_evtchn); 1232 hypervisor_notify_via_evtchn(sc->sc_evtchn);
1233out: 1233out:
1234 mutex_exit(&sc->sc_lock); 1234 mutex_exit(&sc->sc_lock);
1235 return error; 1235 return error;
1236} 1236}
1237 1237
1238static void 1238static void
1239xbd_diskstart_submit(struct xbd_xenbus_softc *sc, 1239xbd_diskstart_submit(struct xbd_xenbus_softc *sc,
1240 int req_id, struct buf *bp, int start, bus_dmamap_t dmamap, 1240 int req_id, struct buf *bp, int start, bus_dmamap_t dmamap,
1241 grant_ref_t *gntref) 1241 grant_ref_t *gntref)
1242{ 1242{
1243 blkif_request_t *req; 1243 blkif_request_t *req;
1244 paddr_t ma; 1244 paddr_t ma;
1245 int nsects, nbytes, dmaseg, first_sect, size, segidx = 0; 1245 int nsects, nbytes, dmaseg, first_sect, size, segidx = 0;
1246 struct blkif_request_segment *reqseg; 1246 struct blkif_request_segment *reqseg;
1247 1247
1248 KASSERT(mutex_owned(&sc->sc_lock)); 1248 KASSERT(mutex_owned(&sc->sc_lock));
1249 1249
1250 req = RING_GET_REQUEST(&sc->sc_ring, sc->sc_ring.req_prod_pvt); 1250 req = RING_GET_REQUEST(&sc->sc_ring, sc->sc_ring.req_prod_pvt);
1251 req->id = req_id; 1251 req->id = req_id;
1252 req->operation = 1252 req->operation =