| @@ -1,14 +1,14 @@ | | | @@ -1,14 +1,14 @@ |
1 | /* $NetBSD: genfs_io.c,v 1.53.2.5 2011/11/30 14:31:29 yamt Exp $ */ | | 1 | /* $NetBSD: genfs_io.c,v 1.53.2.6 2011/12/20 13:46:17 yamt Exp $ */ |
2 | | | 2 | |
3 | /* | | 3 | /* |
4 | * Copyright (c) 1982, 1986, 1989, 1993 | | 4 | * Copyright (c) 1982, 1986, 1989, 1993 |
5 | * The Regents of the University of California. All rights reserved. | | 5 | * The Regents of the University of California. All rights reserved. |
6 | * | | 6 | * |
7 | * Redistribution and use in source and binary forms, with or without | | 7 | * Redistribution and use in source and binary forms, with or without |
8 | * modification, are permitted provided that the following conditions | | 8 | * modification, are permitted provided that the following conditions |
9 | * are met: | | 9 | * are met: |
10 | * 1. Redistributions of source code must retain the above copyright | | 10 | * 1. Redistributions of source code must retain the above copyright |
11 | * notice, this list of conditions and the following disclaimer. | | 11 | * notice, this list of conditions and the following disclaimer. |
12 | * 2. Redistributions in binary form must reproduce the above copyright | | 12 | * 2. Redistributions in binary form must reproduce the above copyright |
13 | * notice, this list of conditions and the following disclaimer in the | | 13 | * notice, this list of conditions and the following disclaimer in the |
14 | * documentation and/or other materials provided with the distribution. | | 14 | * documentation and/or other materials provided with the distribution. |
| @@ -21,27 +21,27 @@ | | | @@ -21,27 +21,27 @@ |
21 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | | 21 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
22 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE | | 22 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE |
23 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | | 23 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
24 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | | 24 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
25 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | | 25 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
26 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | | 26 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
27 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | | 27 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
28 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | | 28 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
29 | * SUCH DAMAGE. | | 29 | * SUCH DAMAGE. |
30 | * | | 30 | * |
31 | */ | | 31 | */ |
32 | | | 32 | |
33 | #include <sys/cdefs.h> | | 33 | #include <sys/cdefs.h> |
34 | __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.53.2.5 2011/11/30 14:31:29 yamt Exp $"); | | 34 | __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.53.2.6 2011/12/20 13:46:17 yamt Exp $"); |
35 | | | 35 | |
36 | #include <sys/param.h> | | 36 | #include <sys/param.h> |
37 | #include <sys/systm.h> | | 37 | #include <sys/systm.h> |
38 | #include <sys/proc.h> | | 38 | #include <sys/proc.h> |
39 | #include <sys/kernel.h> | | 39 | #include <sys/kernel.h> |
40 | #include <sys/mount.h> | | 40 | #include <sys/mount.h> |
41 | #include <sys/vnode.h> | | 41 | #include <sys/vnode.h> |
42 | #include <sys/kmem.h> | | 42 | #include <sys/kmem.h> |
43 | #include <sys/kauth.h> | | 43 | #include <sys/kauth.h> |
44 | #include <sys/fstrans.h> | | 44 | #include <sys/fstrans.h> |
45 | #include <sys/buf.h> | | 45 | #include <sys/buf.h> |
46 | #include <sys/radixtree.h> | | 46 | #include <sys/radixtree.h> |
47 | | | 47 | |
| @@ -207,27 +207,28 @@ startover: | | | @@ -207,27 +207,28 @@ startover: |
207 | if (flags & PGO_LOCKED) { | | 207 | if (flags & PGO_LOCKED) { |
208 | int nfound; | | 208 | int nfound; |
209 | struct vm_page *pg; | | 209 | struct vm_page *pg; |
210 | | | 210 | |
211 | KASSERT(!glocked); | | 211 | KASSERT(!glocked); |
212 | npages = *ap->a_count; | | 212 | npages = *ap->a_count; |
213 | #if defined(DEBUG) | | 213 | #if defined(DEBUG) |
214 | for (i = 0; i < npages; i++) { | | 214 | for (i = 0; i < npages; i++) { |
215 | pg = ap->a_m[i]; | | 215 | pg = ap->a_m[i]; |
216 | KASSERT(pg == NULL || pg == PGO_DONTCARE); | | 216 | KASSERT(pg == NULL || pg == PGO_DONTCARE); |
217 | } | | 217 | } |
218 | #endif /* defined(DEBUG) */ | | 218 | #endif /* defined(DEBUG) */ |
219 | nfound = uvn_findpages(uobj, origoffset, &npages, | | 219 | nfound = uvn_findpages(uobj, origoffset, &npages, |
220 | ap->a_m, UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0)); | | 220 | ap->a_m, NULL, |
| | | 221 | UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0)); |
221 | KASSERT(npages == *ap->a_count); | | 222 | KASSERT(npages == *ap->a_count); |
222 | if (nfound == 0) { | | 223 | if (nfound == 0) { |
223 | error = EBUSY; | | 224 | error = EBUSY; |
224 | goto out_err; | | 225 | goto out_err; |
225 | } | | 226 | } |
226 | if (!genfs_node_rdtrylock(vp)) { | | 227 | if (!genfs_node_rdtrylock(vp)) { |
227 | genfs_rel_pages(ap->a_m, npages); | | 228 | genfs_rel_pages(ap->a_m, npages); |
228 | | | 229 | |
229 | /* | | 230 | /* |
230 | * restore the array. | | 231 | * restore the array. |
231 | */ | | 232 | */ |
232 | | | 233 | |
233 | for (i = 0; i < npages; i++) { | | 234 | for (i = 0; i < npages; i++) { |
| @@ -333,27 +334,27 @@ startover: | | | @@ -333,27 +334,27 @@ startover: |
333 | genfs_node_rdlock(vp); | | 334 | genfs_node_rdlock(vp); |
334 | } | | 335 | } |
335 | } | | 336 | } |
336 | mutex_enter(uobj->vmobjlock); | | 337 | mutex_enter(uobj->vmobjlock); |
337 | if (vp->v_size < origvsize) { | | 338 | if (vp->v_size < origvsize) { |
338 | if (!glocked) { | | 339 | if (!glocked) { |
339 | genfs_node_unlock(vp); | | 340 | genfs_node_unlock(vp); |
340 | } | | 341 | } |
341 | if (pgs != pgs_onstack) | | 342 | if (pgs != pgs_onstack) |
342 | kmem_free(pgs, pgs_size); | | 343 | kmem_free(pgs, pgs_size); |
343 | goto startover; | | 344 | goto startover; |
344 | } | | 345 | } |
345 | | | 346 | |
346 | if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], | | 347 | if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], NULL, |
347 | async ? UFP_NOWAIT : UFP_ALL) != orignmempages) { | | 348 | async ? UFP_NOWAIT : UFP_ALL) != orignmempages) { |
348 | if (!glocked) { | | 349 | if (!glocked) { |
349 | genfs_node_unlock(vp); | | 350 | genfs_node_unlock(vp); |
350 | } | | 351 | } |
351 | KASSERT(async != 0); | | 352 | KASSERT(async != 0); |
352 | genfs_rel_pages(&pgs[ridx], orignmempages); | | 353 | genfs_rel_pages(&pgs[ridx], orignmempages); |
353 | mutex_exit(uobj->vmobjlock); | | 354 | mutex_exit(uobj->vmobjlock); |
354 | error = EBUSY; | | 355 | error = EBUSY; |
355 | goto out_err_free; | | 356 | goto out_err_free; |
356 | } | | 357 | } |
357 | | | 358 | |
358 | /* | | 359 | /* |
359 | * if PGO_OVERWRITE is set, don't bother reading the pages. | | 360 | * if PGO_OVERWRITE is set, don't bother reading the pages. |
| @@ -420,27 +421,27 @@ startover: | | | @@ -420,27 +421,27 @@ startover: |
420 | | | 421 | |
421 | /* | | 422 | /* |
422 | * we need to avoid deadlocks caused by locking | | 423 | * we need to avoid deadlocks caused by locking |
423 | * additional pages at lower offsets than pages we | | 424 | * additional pages at lower offsets than pages we |
424 | * already have locked. unlock them all and start over. | | 425 | * already have locked. unlock them all and start over. |
425 | */ | | 426 | */ |
426 | | | 427 | |
427 | genfs_rel_pages(&pgs[ridx], orignmempages); | | 428 | genfs_rel_pages(&pgs[ridx], orignmempages); |
428 | memset(pgs, 0, pgs_size); | | 429 | memset(pgs, 0, pgs_size); |
429 | | | 430 | |
430 | UVMHIST_LOG(ubchist, "reset npages start 0x%x end 0x%x", | | 431 | UVMHIST_LOG(ubchist, "reset npages start 0x%x end 0x%x", |
431 | startoffset, endoffset, 0,0); | | 432 | startoffset, endoffset, 0,0); |
432 | npgs = npages; | | 433 | npgs = npages; |
433 | if (uvn_findpages(uobj, startoffset, &npgs, pgs, | | 434 | if (uvn_findpages(uobj, startoffset, &npgs, pgs, NULL, |
434 | async ? UFP_NOWAIT : UFP_ALL) != npages) { | | 435 | async ? UFP_NOWAIT : UFP_ALL) != npages) { |
435 | if (!glocked) { | | 436 | if (!glocked) { |
436 | genfs_node_unlock(vp); | | 437 | genfs_node_unlock(vp); |
437 | } | | 438 | } |
438 | KASSERT(async != 0); | | 439 | KASSERT(async != 0); |
439 | genfs_rel_pages(pgs, npages); | | 440 | genfs_rel_pages(pgs, npages); |
440 | mutex_exit(uobj->vmobjlock); | | 441 | mutex_exit(uobj->vmobjlock); |
441 | error = EBUSY; | | 442 | error = EBUSY; |
442 | goto out_err_free; | | 443 | goto out_err_free; |
443 | } | | 444 | } |
444 | } | | 445 | } |
445 | | | 446 | |
446 | mutex_exit(uobj->vmobjlock); | | 447 | mutex_exit(uobj->vmobjlock); |
| @@ -846,27 +847,28 @@ genfs_putpages(void *v) | | | @@ -846,27 +847,28 @@ genfs_putpages(void *v) |
846 | return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi, | | 847 | return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi, |
847 | ap->a_flags, NULL); | | 848 | ap->a_flags, NULL); |
848 | } | | 849 | } |
849 | | | 850 | |
850 | int | | 851 | int |
851 | genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff, | | 852 | genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff, |
852 | int origflags, struct vm_page **busypg) | | 853 | int origflags, struct vm_page **busypg) |
853 | { | | 854 | { |
854 | struct uvm_object * const uobj = &vp->v_uobj; | | 855 | struct uvm_object * const uobj = &vp->v_uobj; |
855 | kmutex_t * const slock = uobj->vmobjlock; | | 856 | kmutex_t * const slock = uobj->vmobjlock; |
856 | off_t off; | | 857 | off_t off; |
857 | /* Even for strange MAXPHYS, the shift rounds down to a page */ | | 858 | /* Even for strange MAXPHYS, the shift rounds down to a page */ |
858 | #define maxpages (MAXPHYS >> PAGE_SHIFT) | | 859 | #define maxpages (MAXPHYS >> PAGE_SHIFT) |
859 | int i, error, npages, nback; | | 860 | int i, error; |
| | | 861 | unsigned int npages, nback; |
860 | int freeflag; | | 862 | int freeflag; |
861 | struct vm_page *pgs[maxpages], *pg; | | 863 | struct vm_page *pgs[maxpages], *pg; |
862 | struct uvm_page_array a; | | 864 | struct uvm_page_array a; |
863 | bool wasclean, needs_clean, yld; | | 865 | bool wasclean, needs_clean, yld; |
864 | bool async = (origflags & PGO_SYNCIO) == 0; | | 866 | bool async = (origflags & PGO_SYNCIO) == 0; |
865 | bool pagedaemon = curlwp == uvm.pagedaemon_lwp; | | 867 | bool pagedaemon = curlwp == uvm.pagedaemon_lwp; |
866 | struct lwp * const l = curlwp ? curlwp : &lwp0; | | 868 | struct lwp * const l = curlwp ? curlwp : &lwp0; |
867 | int flags; | | 869 | int flags; |
868 | bool modified; /* if we write out any pages */ | | 870 | bool modified; /* if we write out any pages */ |
869 | bool need_wapbl; | | 871 | bool need_wapbl; |
870 | bool has_trans; | | 872 | bool has_trans; |
871 | bool tryclean; /* try to pull off from the syncer's list */ | | 873 | bool tryclean; /* try to pull off from the syncer's list */ |
872 | bool onworklst; | | 874 | bool onworklst; |
| @@ -1078,116 +1080,84 @@ retry: | | | @@ -1078,116 +1080,84 @@ retry: |
1078 | if (flags & PGO_CLEANIT) { | | 1080 | if (flags & PGO_CLEANIT) { |
1079 | needs_clean = uvm_pagecheckdirty(pg, protected); | | 1081 | needs_clean = uvm_pagecheckdirty(pg, protected); |
1080 | } else { | | 1082 | } else { |
1081 | needs_clean = false; | | 1083 | needs_clean = false; |
1082 | } | | 1084 | } |
1083 | | | 1085 | |
1084 | /* | | 1086 | /* |
1085 | * if we're cleaning, build a cluster. | | 1087 | * if we're cleaning, build a cluster. |
1086 | * the cluster will consist of pages which are currently dirty. | | 1088 | * the cluster will consist of pages which are currently dirty. |
1087 | * if not cleaning, just operate on the one page. | | 1089 | * if not cleaning, just operate on the one page. |
1088 | */ | | 1090 | */ |
1089 | | | 1091 | |
1090 | if (needs_clean) { | | 1092 | if (needs_clean) { |
| | | 1093 | unsigned int nforw; |
| | | 1094 | unsigned int fpflags; |
| | | 1095 | |
1091 | KDASSERT((vp->v_iflag & VI_ONWORKLST)); | | 1096 | KDASSERT((vp->v_iflag & VI_ONWORKLST)); |
1092 | wasclean = false; | | 1097 | wasclean = false; |
1093 | memset(pgs, 0, sizeof(pgs)); | | 1098 | memset(pgs, 0, sizeof(pgs)); |
1094 | pg->flags |= PG_BUSY; | | 1099 | pg->flags |= PG_BUSY; |
1095 | UVM_PAGE_OWN(pg, "genfs_putpages"); | | 1100 | UVM_PAGE_OWN(pg, "genfs_putpages"); |
1096 | | | 1101 | |
1097 | /* | | 1102 | /* |
1098 | * first look backward. | | 1103 | * XXX PG_PAGER1 incompatibility check. |
1099 | * | | 1104 | * this is a kludge for nfs. |
1100 | * XXX implement PG_PAGER1 incompatibility check. | | | |
1101 | * probably it's better to make PG_NEEDCOMMIT a first | | 1105 | * probably it's better to make PG_NEEDCOMMIT a first |
1102 | * level citizen for uvm/genfs. | | 1106 | * level citizen for uvm/genfs. |
1103 | */ | | 1107 | */ |
| | | 1108 | fpflags = UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY; |
| | | 1109 | if ((pg->flags & PG_PAGER1) != 0) { |
| | | 1110 | fpflags |= UFP_ONLYPAGER1; |
| | | 1111 | } else { |
| | | 1112 | fpflags |= UFP_NOPAGER1; |
| | | 1113 | } |
1104 | | | 1114 | |
| | | 1115 | /* |
| | | 1116 | * first look backward. |
| | | 1117 | */ |
1105 | npages = MIN(maxpages >> 1, off >> PAGE_SHIFT); | | 1118 | npages = MIN(maxpages >> 1, off >> PAGE_SHIFT); |
1106 | nback = npages; | | 1119 | nback = npages; |
1107 | uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0], | | 1120 | uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0], |
1108 | UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD); | | 1121 | NULL, fpflags | UFP_BACKWARD); |
1109 | if (nback) { | | 1122 | if (nback) { |
1110 | memmove(&pgs[0], &pgs[npages - nback], | | 1123 | memmove(&pgs[0], &pgs[npages - nback], |
1111 | nback * sizeof(pgs[0])); | | 1124 | nback * sizeof(pgs[0])); |
1112 | if (npages - nback < nback) | | 1125 | if (npages - nback < nback) |
1113 | memset(&pgs[nback], 0, | | 1126 | memset(&pgs[nback], 0, |
1114 | (npages - nback) * sizeof(pgs[0])); | | 1127 | (npages - nback) * sizeof(pgs[0])); |
1115 | else | | 1128 | else |
1116 | memset(&pgs[npages - nback], 0, | | 1129 | memset(&pgs[npages - nback], 0, |
1117 | nback * sizeof(pgs[0])); | | 1130 | nback * sizeof(pgs[0])); |
1118 | } | | 1131 | } |
1119 | | | 1132 | |
1120 | /* | | 1133 | /* |
1121 | * then plug in our page of interest. | | 1134 | * then plug in our page of interest. |
1122 | */ | | 1135 | */ |
1123 | | | 1136 | |
1124 | pgs[nback] = pg; | | 1137 | pgs[nback] = pg; |
1125 | | | 1138 | |
1126 | /* | | 1139 | /* |
1127 | * then look forward to fill in the remaining space in | | 1140 | * then look forward to fill in the remaining space in |
1128 | * the array of pages. | | 1141 | * the array of pages. |
| | | 1142 | * |
| | | 1143 | * pass our cached array of pages so that hopefully |
| | | 1144 | * uvn_findpages can find some good pages in it. |
1129 | */ | | 1145 | */ |
1130 | | | 1146 | |
1131 | for (npages = 1; npages < maxpages; npages++) { | | 1147 | nforw = maxpages - nback - 1; |
1132 | struct vm_page *nextpg; | | 1148 | uvn_findpages(uobj, pg->offset + PAGE_SIZE, |
1133 | | | 1149 | &nforw, &pgs[nback + 1], &a, fpflags); |
1134 | /* | | 1150 | npages = nback + 1 + nforw; |
1135 | * regardless of the value of dirtyonly, | | | |
1136 | * we don't need to care about clean pages here | | | |
1137 | * as we will drop the object lock to call | | | |
1138 | * GOP_WRITE and thus need to clear the array | | | |
1139 | * before the next iteration anyway. | | | |
1140 | */ | | | |
1141 | | | | |
1142 | nextpg = uvm_page_array_fill_and_peek(&a, uobj, | | | |
1143 | pgs[npages - 1]->offset + PAGE_SIZE, | | | |
1144 | maxpages - npages, | | | |
1145 | UVM_PAGE_ARRAY_FILL_DIRTYONLY | | | | |
1146 | UVM_PAGE_ARRAY_FILL_DENSE); | | | |
1147 | if (nextpg == NULL) { | | | |
1148 | break; | | | |
1149 | } | | | |
1150 | KASSERT(nextpg->uobject == pg->uobject); | | | |
1151 | KASSERT(nextpg->offset > pg->offset); | | | |
1152 | KASSERT(nextpg->offset > | | | |
1153 | pgs[npages - 1]->offset); | | | |
1154 | if (pgs[npages - 1]->offset + PAGE_SIZE != | | | |
1155 | nextpg->offset) { | | | |
1156 | break; | | | |
1157 | } | | | |
1158 | if ((nextpg->flags & PG_BUSY) != 0) { | | | |
1159 | break; | | | |
1160 | } | | | |
1161 | | | | |
1162 | /* | | | |
1163 | * don't bother to cluster incompatible pages | | | |
1164 | * together. | | | |
1165 | * | | | |
1166 | * XXX hack for nfs | | | |
1167 | */ | | | |
1168 | | | | |
1169 | if (((nextpg->flags ^ pgs[npages - 1]->flags) & | | | |
1170 | PG_PAGER1) != 0) { | | | |
1171 | break; | | | |
1172 | } | | | |
1173 | if (!uvm_pagecheckdirty(nextpg, false)) { | | | |
1174 | break; | | | |
1175 | } | | | |
1176 | nextpg->flags |= PG_BUSY; | | | |
1177 | UVM_PAGE_OWN(nextpg, "genfs_putpages2"); | | | |
1178 | pgs[npages] = nextpg; | | | |
1179 | uvm_page_array_advance(&a); | | | |
1180 | } | | | |
1181 | } else { | | 1151 | } else { |
1182 | pgs[0] = pg; | | 1152 | pgs[0] = pg; |
1183 | npages = 1; | | 1153 | npages = 1; |
1184 | nback = 0; | | 1154 | nback = 0; |
1185 | } | | 1155 | } |
1186 | | | 1156 | |
1187 | /* | | 1157 | /* |
1188 | * apply FREE or DEACTIVATE options if requested. | | 1158 | * apply FREE or DEACTIVATE options if requested. |
1189 | */ | | 1159 | */ |
1190 | | | 1160 | |
1191 | if (flags & (PGO_DEACTIVATE|PGO_FREE)) { | | 1161 | if (flags & (PGO_DEACTIVATE|PGO_FREE)) { |
1192 | mutex_enter(&uvm_pageqlock); | | 1162 | mutex_enter(&uvm_pageqlock); |
1193 | } | | 1163 | } |
| @@ -1540,45 +1510,45 @@ genfs_compat_getpages(void *v) | | | @@ -1540,45 +1510,45 @@ genfs_compat_getpages(void *v) |
1540 | vaddr_t kva; | | 1510 | vaddr_t kva; |
1541 | int i, error, orignpages, npages; | | 1511 | int i, error, orignpages, npages; |
1542 | struct iovec iov; | | 1512 | struct iovec iov; |
1543 | struct uio uio; | | 1513 | struct uio uio; |
1544 | kauth_cred_t cred = curlwp->l_cred; | | 1514 | kauth_cred_t cred = curlwp->l_cred; |
1545 | const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; | | 1515 | const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; |
1546 | | | 1516 | |
1547 | error = 0; | | 1517 | error = 0; |
1548 | origoffset = ap->a_offset; | | 1518 | origoffset = ap->a_offset; |
1549 | orignpages = *ap->a_count; | | 1519 | orignpages = *ap->a_count; |
1550 | pgs = ap->a_m; | | 1520 | pgs = ap->a_m; |
1551 | | | 1521 | |
1552 | if (ap->a_flags & PGO_LOCKED) { | | 1522 | if (ap->a_flags & PGO_LOCKED) { |
1553 | uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, | | 1523 | uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, NULL, |
1554 | UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0)); | | 1524 | UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0)); |
1555 | | | 1525 | |
1556 | error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0; | | 1526 | error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0; |
1557 | if (error == 0 && memwrite) { | | 1527 | if (error == 0 && memwrite) { |
1558 | genfs_markdirty(vp); | | 1528 | genfs_markdirty(vp); |
1559 | } | | 1529 | } |
1560 | return error; | | 1530 | return error; |
1561 | } | | 1531 | } |
1562 | if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { | | 1532 | if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { |
1563 | mutex_exit(uobj->vmobjlock); | | 1533 | mutex_exit(uobj->vmobjlock); |
1564 | return EINVAL; | | 1534 | return EINVAL; |
1565 | } | | 1535 | } |
1566 | if ((ap->a_flags & PGO_SYNCIO) == 0) { | | 1536 | if ((ap->a_flags & PGO_SYNCIO) == 0) { |
1567 | mutex_exit(uobj->vmobjlock); | | 1537 | mutex_exit(uobj->vmobjlock); |
1568 | return 0; | | 1538 | return 0; |
1569 | } | | 1539 | } |
1570 | npages = orignpages; | | 1540 | npages = orignpages; |
1571 | uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL); | | 1541 | uvn_findpages(uobj, origoffset, &npages, pgs, NULL, UFP_ALL); |
1572 | mutex_exit(uobj->vmobjlock); | | 1542 | mutex_exit(uobj->vmobjlock); |
1573 | kva = uvm_pagermapin(pgs, npages, | | 1543 | kva = uvm_pagermapin(pgs, npages, |
1574 | UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); | | 1544 | UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); |
1575 | for (i = 0; i < npages; i++) { | | 1545 | for (i = 0; i < npages; i++) { |
1576 | pg = pgs[i]; | | 1546 | pg = pgs[i]; |
1577 | if ((pg->flags & PG_FAKE) == 0) { | | 1547 | if ((pg->flags & PG_FAKE) == 0) { |
1578 | continue; | | 1548 | continue; |
1579 | } | | 1549 | } |
1580 | iov.iov_base = (char *)kva + (i << PAGE_SHIFT); | | 1550 | iov.iov_base = (char *)kva + (i << PAGE_SHIFT); |
1581 | iov.iov_len = PAGE_SIZE; | | 1551 | iov.iov_len = PAGE_SIZE; |
1582 | uio.uio_iov = &iov; | | 1552 | uio.uio_iov = &iov; |
1583 | uio.uio_iovcnt = 1; | | 1553 | uio.uio_iovcnt = 1; |
1584 | uio.uio_offset = origoffset + (i << PAGE_SHIFT); | | 1554 | uio.uio_offset = origoffset + (i << PAGE_SHIFT); |