| @@ -1,14 +1,14 @@ | | | @@ -1,14 +1,14 @@ |
1 | /* $NetBSD: kern_physio.c,v 1.88 2008/09/24 08:19:19 hannken Exp $ */ | | 1 | /* $NetBSD: kern_physio.c,v 1.88.4.1 2009/05/27 21:42:08 snj Exp $ */ |
2 | | | 2 | |
3 | /*- | | 3 | /*- |
4 | * Copyright (c) 1982, 1986, 1990, 1993 | | 4 | * Copyright (c) 1982, 1986, 1990, 1993 |
5 | * The Regents of the University of California. All rights reserved. | | 5 | * The Regents of the University of California. All rights reserved. |
6 | * (c) UNIX System Laboratories, Inc. | | 6 | * (c) UNIX System Laboratories, Inc. |
7 | * All or some portions of this file are derived from material licensed | | 7 | * All or some portions of this file are derived from material licensed |
8 | * to the University of California by American Telephone and Telegraph | | 8 | * to the University of California by American Telephone and Telegraph |
9 | * Co. or Unix System Laboratories, Inc. and are reproduced herein with | | 9 | * Co. or Unix System Laboratories, Inc. and are reproduced herein with |
10 | * the permission of UNIX System Laboratories, Inc. | | 10 | * the permission of UNIX System Laboratories, Inc. |
11 | * | | 11 | * |
12 | * Redistribution and use in source and binary forms, with or without | | 12 | * Redistribution and use in source and binary forms, with or without |
13 | * modification, are permitted provided that the following conditions | | 13 | * modification, are permitted provided that the following conditions |
14 | * are met: | | 14 | * are met: |
| @@ -61,27 +61,27 @@ | | | @@ -61,27 +61,27 @@ |
61 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE | | 61 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE |
62 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | | 62 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
63 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | | 63 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
64 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | | 64 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
65 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | | 65 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
66 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | | 66 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
67 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | | 67 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
68 | * SUCH DAMAGE. | | 68 | * SUCH DAMAGE. |
69 | * | | 69 | * |
70 | * @(#)kern_physio.c 8.1 (Berkeley) 6/10/93 | | 70 | * @(#)kern_physio.c 8.1 (Berkeley) 6/10/93 |
71 | */ | | 71 | */ |
72 | | | 72 | |
73 | #include <sys/cdefs.h> | | 73 | #include <sys/cdefs.h> |
74 | __KERNEL_RCSID(0, "$NetBSD: kern_physio.c,v 1.88 2008/09/24 08:19:19 hannken Exp $"); | | 74 | __KERNEL_RCSID(0, "$NetBSD: kern_physio.c,v 1.88.4.1 2009/05/27 21:42:08 snj Exp $"); |
75 | | | 75 | |
76 | #include <sys/param.h> | | 76 | #include <sys/param.h> |
77 | #include <sys/systm.h> | | 77 | #include <sys/systm.h> |
78 | #include <sys/buf.h> | | 78 | #include <sys/buf.h> |
79 | #include <sys/proc.h> | | 79 | #include <sys/proc.h> |
80 | #include <sys/once.h> | | 80 | #include <sys/once.h> |
81 | #include <sys/workqueue.h> | | 81 | #include <sys/workqueue.h> |
82 | #include <sys/kmem.h> | | 82 | #include <sys/kmem.h> |
83 | | | 83 | |
84 | #include <uvm/uvm_extern.h> | | 84 | #include <uvm/uvm_extern.h> |
85 | | | 85 | |
86 | ONCE_DECL(physio_initialized); | | 86 | ONCE_DECL(physio_initialized); |
87 | struct workqueue *physio_workqueue; | | 87 | struct workqueue *physio_workqueue; |
| @@ -107,37 +107,39 @@ struct physio_stat { | | | @@ -107,37 +107,39 @@ struct physio_stat { |
107 | off_t ps_endoffset; | | 107 | off_t ps_endoffset; |
108 | buf_t *ps_orig_bp; | | 108 | buf_t *ps_orig_bp; |
109 | kmutex_t ps_lock; | | 109 | kmutex_t ps_lock; |
110 | kcondvar_t ps_cv; | | 110 | kcondvar_t ps_cv; |
111 | }; | | 111 | }; |
112 | | | 112 | |
113 | static void | | 113 | static void |
114 | physio_done(struct work *wk, void *dummy) | | 114 | physio_done(struct work *wk, void *dummy) |
115 | { | | 115 | { |
116 | struct buf *bp = (void *)wk; | | 116 | struct buf *bp = (void *)wk; |
117 | size_t todo = bp->b_bufsize; | | 117 | size_t todo = bp->b_bufsize; |
118 | size_t done = bp->b_bcount - bp->b_resid; | | 118 | size_t done = bp->b_bcount - bp->b_resid; |
119 | struct physio_stat *ps = bp->b_private; | | 119 | struct physio_stat *ps = bp->b_private; |
| | | 120 | bool is_iobuf; |
120 | | | 121 | |
121 | KASSERT(&bp->b_work == wk); | | 122 | KASSERT(&bp->b_work == wk); |
122 | KASSERT(bp->b_bcount <= todo); | | 123 | KASSERT(bp->b_bcount <= todo); |
123 | KASSERT(bp->b_resid <= bp->b_bcount); | | 124 | KASSERT(bp->b_resid <= bp->b_bcount); |
124 | KASSERT((bp->b_flags & B_PHYS) != 0); | | 125 | KASSERT((bp->b_flags & B_PHYS) != 0); |
125 | KASSERT(dummy == NULL); | | 126 | KASSERT(dummy == NULL); |
126 | | | 127 | |
127 | vunmapbuf(bp, todo); | | 128 | vunmapbuf(bp, todo); |
128 | uvm_vsunlock(bp->b_proc->p_vmspace, bp->b_data, todo); | | 129 | uvm_vsunlock(bp->b_proc->p_vmspace, bp->b_data, todo); |
129 | | | 130 | |
130 | mutex_enter(&ps->ps_lock); | | 131 | mutex_enter(&ps->ps_lock); |
| | | 132 | is_iobuf = (bp != ps->ps_orig_bp); |
131 | if (__predict_false(done != todo)) { | | 133 | if (__predict_false(done != todo)) { |
132 | off_t endoffset = dbtob(bp->b_blkno) + done; | | 134 | off_t endoffset = dbtob(bp->b_blkno) + done; |
133 | | | 135 | |
134 | /* | | 136 | /* |
135 | * we got an error or hit EOM. | | 137 | * we got an error or hit EOM. |
136 | * | | 138 | * |
137 | * we only care about the first one. | | 139 | * we only care about the first one. |
138 | * ie. the one at the lowest offset. | | 140 | * ie. the one at the lowest offset. |
139 | */ | | 141 | */ |
140 | | | 142 | |
141 | KASSERT(ps->ps_endoffset != endoffset); | | 143 | KASSERT(ps->ps_endoffset != endoffset); |
142 | DPRINTF(("%s: error=%d at %" PRIu64 " - %" PRIu64 | | 144 | DPRINTF(("%s: error=%d at %" PRIu64 " - %" PRIu64 |
143 | ", blkno=%" PRIu64 ", bcount=%d, flags=0x%x\n", | | 145 | ", blkno=%" PRIu64 ", bcount=%d, flags=0x%x\n", |
| @@ -153,27 +155,27 @@ physio_done(struct work *wk, void *dummy | | | @@ -153,27 +155,27 @@ physio_done(struct work *wk, void *dummy |
153 | | | 155 | |
154 | ps->ps_endoffset = endoffset; | | 156 | ps->ps_endoffset = endoffset; |
155 | ps->ps_error = bp->b_error; | | 157 | ps->ps_error = bp->b_error; |
156 | } | | 158 | } |
157 | ps->ps_failed++; | | 159 | ps->ps_failed++; |
158 | } else { | | 160 | } else { |
159 | KASSERT(bp->b_error == 0); | | 161 | KASSERT(bp->b_error == 0); |
160 | } | | 162 | } |
161 | | | 163 | |
162 | ps->ps_running--; | | 164 | ps->ps_running--; |
163 | cv_signal(&ps->ps_cv); | | 165 | cv_signal(&ps->ps_cv); |
164 | mutex_exit(&ps->ps_lock); | | 166 | mutex_exit(&ps->ps_lock); |
165 | | | 167 | |
166 | if (bp != ps->ps_orig_bp) | | 168 | if (is_iobuf) |
167 | putiobuf(bp); | | 169 | putiobuf(bp); |
168 | } | | 170 | } |
169 | | | 171 | |
170 | static void | | 172 | static void |
171 | physio_biodone(struct buf *bp) | | 173 | physio_biodone(struct buf *bp) |
172 | { | | 174 | { |
173 | #if defined(DIAGNOSTIC) | | 175 | #if defined(DIAGNOSTIC) |
174 | struct physio_stat *ps = bp->b_private; | | 176 | struct physio_stat *ps = bp->b_private; |
175 | size_t todo = bp->b_bufsize; | | 177 | size_t todo = bp->b_bufsize; |
176 | | | 178 | |
177 | KASSERT(ps->ps_running > 0); | | 179 | KASSERT(ps->ps_running > 0); |
178 | KASSERT(bp->b_bcount <= todo); | | 180 | KASSERT(bp->b_bcount <= todo); |
179 | KASSERT(bp->b_resid <= bp->b_bcount); | | 181 | KASSERT(bp->b_resid <= bp->b_bcount); |