Tue Dec 20 13:46:17 2011 UTC ()
don't inline uvn_findpages in genfs_io.


(yamt)
diff -r1.53.2.5 -r1.53.2.6 src/sys/miscfs/genfs/genfs_io.c
diff -r1.176.2.4 -r1.176.2.5 src/sys/uvm/uvm_extern.h
diff -r1.97.2.2 -r1.97.2.3 src/sys/uvm/uvm_vnode.c

cvs diff -r1.53.2.5 -r1.53.2.6 src/sys/miscfs/genfs/genfs_io.c (expand / switch to unified diff)

--- src/sys/miscfs/genfs/genfs_io.c 2011/11/30 14:31:29 1.53.2.5
+++ src/sys/miscfs/genfs/genfs_io.c 2011/12/20 13:46:17 1.53.2.6
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: genfs_io.c,v 1.53.2.5 2011/11/30 14:31:29 yamt Exp $ */ 1/* $NetBSD: genfs_io.c,v 1.53.2.6 2011/12/20 13:46:17 yamt Exp $ */
2 2
3/* 3/*
4 * Copyright (c) 1982, 1986, 1989, 1993 4 * Copyright (c) 1982, 1986, 1989, 1993
5 * The Regents of the University of California. All rights reserved. 5 * The Regents of the University of California. All rights reserved.
6 * 6 *
7 * Redistribution and use in source and binary forms, with or without 7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions 8 * modification, are permitted provided that the following conditions
9 * are met: 9 * are met:
10 * 1. Redistributions of source code must retain the above copyright 10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer. 11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright 12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the 13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution. 14 * documentation and/or other materials provided with the distribution.
@@ -21,27 +21,27 @@ @@ -21,27 +21,27 @@
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE. 29 * SUCH DAMAGE.
30 * 30 *
31 */ 31 */
32 32
33#include <sys/cdefs.h> 33#include <sys/cdefs.h>
34__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.53.2.5 2011/11/30 14:31:29 yamt Exp $"); 34__KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.53.2.6 2011/12/20 13:46:17 yamt Exp $");
35 35
36#include <sys/param.h> 36#include <sys/param.h>
37#include <sys/systm.h> 37#include <sys/systm.h>
38#include <sys/proc.h> 38#include <sys/proc.h>
39#include <sys/kernel.h> 39#include <sys/kernel.h>
40#include <sys/mount.h> 40#include <sys/mount.h>
41#include <sys/vnode.h> 41#include <sys/vnode.h>
42#include <sys/kmem.h> 42#include <sys/kmem.h>
43#include <sys/kauth.h> 43#include <sys/kauth.h>
44#include <sys/fstrans.h> 44#include <sys/fstrans.h>
45#include <sys/buf.h> 45#include <sys/buf.h>
46#include <sys/radixtree.h> 46#include <sys/radixtree.h>
47 47
@@ -207,27 +207,28 @@ startover: @@ -207,27 +207,28 @@ startover:
207 if (flags & PGO_LOCKED) { 207 if (flags & PGO_LOCKED) {
208 int nfound; 208 int nfound;
209 struct vm_page *pg; 209 struct vm_page *pg;
210 210
211 KASSERT(!glocked); 211 KASSERT(!glocked);
212 npages = *ap->a_count; 212 npages = *ap->a_count;
213#if defined(DEBUG) 213#if defined(DEBUG)
214 for (i = 0; i < npages; i++) { 214 for (i = 0; i < npages; i++) {
215 pg = ap->a_m[i]; 215 pg = ap->a_m[i];
216 KASSERT(pg == NULL || pg == PGO_DONTCARE); 216 KASSERT(pg == NULL || pg == PGO_DONTCARE);
217 } 217 }
218#endif /* defined(DEBUG) */ 218#endif /* defined(DEBUG) */
219 nfound = uvn_findpages(uobj, origoffset, &npages, 219 nfound = uvn_findpages(uobj, origoffset, &npages,
220 ap->a_m, UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0)); 220 ap->a_m, NULL,
 221 UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0));
221 KASSERT(npages == *ap->a_count); 222 KASSERT(npages == *ap->a_count);
222 if (nfound == 0) { 223 if (nfound == 0) {
223 error = EBUSY; 224 error = EBUSY;
224 goto out_err; 225 goto out_err;
225 } 226 }
226 if (!genfs_node_rdtrylock(vp)) { 227 if (!genfs_node_rdtrylock(vp)) {
227 genfs_rel_pages(ap->a_m, npages); 228 genfs_rel_pages(ap->a_m, npages);
228 229
229 /* 230 /*
230 * restore the array. 231 * restore the array.
231 */ 232 */
232 233
233 for (i = 0; i < npages; i++) { 234 for (i = 0; i < npages; i++) {
@@ -333,27 +334,27 @@ startover: @@ -333,27 +334,27 @@ startover:
333 genfs_node_rdlock(vp); 334 genfs_node_rdlock(vp);
334 } 335 }
335 } 336 }
336 mutex_enter(uobj->vmobjlock); 337 mutex_enter(uobj->vmobjlock);
337 if (vp->v_size < origvsize) { 338 if (vp->v_size < origvsize) {
338 if (!glocked) { 339 if (!glocked) {
339 genfs_node_unlock(vp); 340 genfs_node_unlock(vp);
340 } 341 }
341 if (pgs != pgs_onstack) 342 if (pgs != pgs_onstack)
342 kmem_free(pgs, pgs_size); 343 kmem_free(pgs, pgs_size);
343 goto startover; 344 goto startover;
344 } 345 }
345 346
346 if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], 347 if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], NULL,
347 async ? UFP_NOWAIT : UFP_ALL) != orignmempages) { 348 async ? UFP_NOWAIT : UFP_ALL) != orignmempages) {
348 if (!glocked) { 349 if (!glocked) {
349 genfs_node_unlock(vp); 350 genfs_node_unlock(vp);
350 } 351 }
351 KASSERT(async != 0); 352 KASSERT(async != 0);
352 genfs_rel_pages(&pgs[ridx], orignmempages); 353 genfs_rel_pages(&pgs[ridx], orignmempages);
353 mutex_exit(uobj->vmobjlock); 354 mutex_exit(uobj->vmobjlock);
354 error = EBUSY; 355 error = EBUSY;
355 goto out_err_free; 356 goto out_err_free;
356 } 357 }
357 358
358 /* 359 /*
359 * if PGO_OVERWRITE is set, don't bother reading the pages. 360 * if PGO_OVERWRITE is set, don't bother reading the pages.
@@ -420,27 +421,27 @@ startover: @@ -420,27 +421,27 @@ startover:
420 421
421 /* 422 /*
422 * we need to avoid deadlocks caused by locking 423 * we need to avoid deadlocks caused by locking
423 * additional pages at lower offsets than pages we 424 * additional pages at lower offsets than pages we
424 * already have locked. unlock them all and start over. 425 * already have locked. unlock them all and start over.
425 */ 426 */
426 427
427 genfs_rel_pages(&pgs[ridx], orignmempages); 428 genfs_rel_pages(&pgs[ridx], orignmempages);
428 memset(pgs, 0, pgs_size); 429 memset(pgs, 0, pgs_size);
429 430
430 UVMHIST_LOG(ubchist, "reset npages start 0x%x end 0x%x", 431 UVMHIST_LOG(ubchist, "reset npages start 0x%x end 0x%x",
431 startoffset, endoffset, 0,0); 432 startoffset, endoffset, 0,0);
432 npgs = npages; 433 npgs = npages;
433 if (uvn_findpages(uobj, startoffset, &npgs, pgs, 434 if (uvn_findpages(uobj, startoffset, &npgs, pgs, NULL,
434 async ? UFP_NOWAIT : UFP_ALL) != npages) { 435 async ? UFP_NOWAIT : UFP_ALL) != npages) {
435 if (!glocked) { 436 if (!glocked) {
436 genfs_node_unlock(vp); 437 genfs_node_unlock(vp);
437 } 438 }
438 KASSERT(async != 0); 439 KASSERT(async != 0);
439 genfs_rel_pages(pgs, npages); 440 genfs_rel_pages(pgs, npages);
440 mutex_exit(uobj->vmobjlock); 441 mutex_exit(uobj->vmobjlock);
441 error = EBUSY; 442 error = EBUSY;
442 goto out_err_free; 443 goto out_err_free;
443 } 444 }
444 } 445 }
445 446
446 mutex_exit(uobj->vmobjlock); 447 mutex_exit(uobj->vmobjlock);
@@ -846,27 +847,28 @@ genfs_putpages(void *v) @@ -846,27 +847,28 @@ genfs_putpages(void *v)
846 return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi, 847 return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi,
847 ap->a_flags, NULL); 848 ap->a_flags, NULL);
848} 849}
849 850
850int 851int
851genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff, 852genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff,
852 int origflags, struct vm_page **busypg) 853 int origflags, struct vm_page **busypg)
853{ 854{
854 struct uvm_object * const uobj = &vp->v_uobj; 855 struct uvm_object * const uobj = &vp->v_uobj;
855 kmutex_t * const slock = uobj->vmobjlock; 856 kmutex_t * const slock = uobj->vmobjlock;
856 off_t off; 857 off_t off;
857 /* Even for strange MAXPHYS, the shift rounds down to a page */ 858 /* Even for strange MAXPHYS, the shift rounds down to a page */
858#define maxpages (MAXPHYS >> PAGE_SHIFT) 859#define maxpages (MAXPHYS >> PAGE_SHIFT)
859 int i, error, npages, nback; 860 int i, error;
 861 unsigned int npages, nback;
860 int freeflag; 862 int freeflag;
861 struct vm_page *pgs[maxpages], *pg; 863 struct vm_page *pgs[maxpages], *pg;
862 struct uvm_page_array a; 864 struct uvm_page_array a;
863 bool wasclean, needs_clean, yld; 865 bool wasclean, needs_clean, yld;
864 bool async = (origflags & PGO_SYNCIO) == 0; 866 bool async = (origflags & PGO_SYNCIO) == 0;
865 bool pagedaemon = curlwp == uvm.pagedaemon_lwp; 867 bool pagedaemon = curlwp == uvm.pagedaemon_lwp;
866 struct lwp * const l = curlwp ? curlwp : &lwp0; 868 struct lwp * const l = curlwp ? curlwp : &lwp0;
867 int flags; 869 int flags;
868 bool modified; /* if we write out any pages */ 870 bool modified; /* if we write out any pages */
869 bool need_wapbl; 871 bool need_wapbl;
870 bool has_trans; 872 bool has_trans;
871 bool tryclean; /* try to pull off from the syncer's list */ 873 bool tryclean; /* try to pull off from the syncer's list */
872 bool onworklst; 874 bool onworklst;
@@ -1078,116 +1080,84 @@ retry: @@ -1078,116 +1080,84 @@ retry:
1078 if (flags & PGO_CLEANIT) { 1080 if (flags & PGO_CLEANIT) {
1079 needs_clean = uvm_pagecheckdirty(pg, protected); 1081 needs_clean = uvm_pagecheckdirty(pg, protected);
1080 } else { 1082 } else {
1081 needs_clean = false; 1083 needs_clean = false;
1082 } 1084 }
1083 1085
1084 /* 1086 /*
1085 * if we're cleaning, build a cluster. 1087 * if we're cleaning, build a cluster.
1086 * the cluster will consist of pages which are currently dirty. 1088 * the cluster will consist of pages which are currently dirty.
1087 * if not cleaning, just operate on the one page. 1089 * if not cleaning, just operate on the one page.
1088 */ 1090 */
1089 1091
1090 if (needs_clean) { 1092 if (needs_clean) {
 1093 unsigned int nforw;
 1094 unsigned int fpflags;
 1095
1091 KDASSERT((vp->v_iflag & VI_ONWORKLST)); 1096 KDASSERT((vp->v_iflag & VI_ONWORKLST));
1092 wasclean = false; 1097 wasclean = false;
1093 memset(pgs, 0, sizeof(pgs)); 1098 memset(pgs, 0, sizeof(pgs));
1094 pg->flags |= PG_BUSY; 1099 pg->flags |= PG_BUSY;
1095 UVM_PAGE_OWN(pg, "genfs_putpages"); 1100 UVM_PAGE_OWN(pg, "genfs_putpages");
1096 1101
1097 /* 1102 /*
1098 * first look backward. 1103 * XXX PG_PAGER1 incompatibility check.
1099 * 1104 * this is a kludge for nfs.
1100 * XXX implement PG_PAGER1 incompatibility check. 
1101 * probably it's better to make PG_NEEDCOMMIT a first 1105 * probably it's better to make PG_NEEDCOMMIT a first
1102 * level citizen for uvm/genfs. 1106 * level citizen for uvm/genfs.
1103 */ 1107 */
 1108 fpflags = UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY;
 1109 if ((pg->flags & PG_PAGER1) != 0) {
 1110 fpflags |= UFP_ONLYPAGER1;
 1111 } else {
 1112 fpflags |= UFP_NOPAGER1;
 1113 }
1104 1114
 1115 /*
 1116 * first look backward.
 1117 */
1105 npages = MIN(maxpages >> 1, off >> PAGE_SHIFT); 1118 npages = MIN(maxpages >> 1, off >> PAGE_SHIFT);
1106 nback = npages; 1119 nback = npages;
1107 uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0], 1120 uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0],
1108 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD); 1121 NULL, fpflags | UFP_BACKWARD);
1109 if (nback) { 1122 if (nback) {
1110 memmove(&pgs[0], &pgs[npages - nback], 1123 memmove(&pgs[0], &pgs[npages - nback],
1111 nback * sizeof(pgs[0])); 1124 nback * sizeof(pgs[0]));
1112 if (npages - nback < nback) 1125 if (npages - nback < nback)
1113 memset(&pgs[nback], 0, 1126 memset(&pgs[nback], 0,
1114 (npages - nback) * sizeof(pgs[0])); 1127 (npages - nback) * sizeof(pgs[0]));
1115 else 1128 else
1116 memset(&pgs[npages - nback], 0, 1129 memset(&pgs[npages - nback], 0,
1117 nback * sizeof(pgs[0])); 1130 nback * sizeof(pgs[0]));
1118 } 1131 }
1119 1132
1120 /* 1133 /*
1121 * then plug in our page of interest. 1134 * then plug in our page of interest.
1122 */ 1135 */
1123 1136
1124 pgs[nback] = pg; 1137 pgs[nback] = pg;
1125 1138
1126 /* 1139 /*
1127 * then look forward to fill in the remaining space in 1140 * then look forward to fill in the remaining space in
1128 * the array of pages. 1141 * the array of pages.
 1142 *
 1143 * pass our cached array of pages so that hopefully
 1144 * uvn_findpages can find some good pages in it.
1129 */ 1145 */
1130 1146
1131 for (npages = 1; npages < maxpages; npages++) { 1147 nforw = maxpages - nback - 1;
1132 struct vm_page *nextpg; 1148 uvn_findpages(uobj, pg->offset + PAGE_SIZE,
1133 1149 &nforw, &pgs[nback + 1], &a, fpflags);
1134 /* 1150 npages = nback + 1 + nforw;
1135 * regardless of the value of dirtyonly, 
1136 * we don't need to care about clean pages here 
1137 * as we will drop the object lock to call 
1138 * GOP_WRITE and thus need to clear the array 
1139 * before the next iteration anyway. 
1140 */ 
1141 
1142 nextpg = uvm_page_array_fill_and_peek(&a, uobj, 
1143 pgs[npages - 1]->offset + PAGE_SIZE, 
1144 maxpages - npages, 
1145 UVM_PAGE_ARRAY_FILL_DIRTYONLY | 
1146 UVM_PAGE_ARRAY_FILL_DENSE); 
1147 if (nextpg == NULL) { 
1148 break; 
1149 } 
1150 KASSERT(nextpg->uobject == pg->uobject); 
1151 KASSERT(nextpg->offset > pg->offset); 
1152 KASSERT(nextpg->offset > 
1153 pgs[npages - 1]->offset); 
1154 if (pgs[npages - 1]->offset + PAGE_SIZE != 
1155 nextpg->offset) { 
1156 break; 
1157 } 
1158 if ((nextpg->flags & PG_BUSY) != 0) { 
1159 break; 
1160 } 
1161 
1162 /* 
1163 * don't bother to cluster incompatible pages 
1164 * together. 
1165 * 
1166 * XXX hack for nfs 
1167 */ 
1168 
1169 if (((nextpg->flags ^ pgs[npages - 1]->flags) & 
1170 PG_PAGER1) != 0) { 
1171 break; 
1172 } 
1173 if (!uvm_pagecheckdirty(nextpg, false)) { 
1174 break; 
1175 } 
1176 nextpg->flags |= PG_BUSY; 
1177 UVM_PAGE_OWN(nextpg, "genfs_putpages2"); 
1178 pgs[npages] = nextpg; 
1179 uvm_page_array_advance(&a); 
1180 } 
1181 } else { 1151 } else {
1182 pgs[0] = pg; 1152 pgs[0] = pg;
1183 npages = 1; 1153 npages = 1;
1184 nback = 0; 1154 nback = 0;
1185 } 1155 }
1186 1156
1187 /* 1157 /*
1188 * apply FREE or DEACTIVATE options if requested. 1158 * apply FREE or DEACTIVATE options if requested.
1189 */ 1159 */
1190 1160
1191 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1161 if (flags & (PGO_DEACTIVATE|PGO_FREE)) {
1192 mutex_enter(&uvm_pageqlock); 1162 mutex_enter(&uvm_pageqlock);
1193 } 1163 }
@@ -1540,45 +1510,45 @@ genfs_compat_getpages(void *v) @@ -1540,45 +1510,45 @@ genfs_compat_getpages(void *v)
1540 vaddr_t kva; 1510 vaddr_t kva;
1541 int i, error, orignpages, npages; 1511 int i, error, orignpages, npages;
1542 struct iovec iov; 1512 struct iovec iov;
1543 struct uio uio; 1513 struct uio uio;
1544 kauth_cred_t cred = curlwp->l_cred; 1514 kauth_cred_t cred = curlwp->l_cred;
1545 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 1515 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0;
1546 1516
1547 error = 0; 1517 error = 0;
1548 origoffset = ap->a_offset; 1518 origoffset = ap->a_offset;
1549 orignpages = *ap->a_count; 1519 orignpages = *ap->a_count;
1550 pgs = ap->a_m; 1520 pgs = ap->a_m;
1551 1521
1552 if (ap->a_flags & PGO_LOCKED) { 1522 if (ap->a_flags & PGO_LOCKED) {
1553 uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, 1523 uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, NULL,
1554 UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0)); 1524 UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0));
1555 1525
1556 error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0; 1526 error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0;
1557 if (error == 0 && memwrite) { 1527 if (error == 0 && memwrite) {
1558 genfs_markdirty(vp); 1528 genfs_markdirty(vp);
1559 } 1529 }
1560 return error; 1530 return error;
1561 } 1531 }
1562 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { 1532 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) {
1563 mutex_exit(uobj->vmobjlock); 1533 mutex_exit(uobj->vmobjlock);
1564 return EINVAL; 1534 return EINVAL;
1565 } 1535 }
1566 if ((ap->a_flags & PGO_SYNCIO) == 0) { 1536 if ((ap->a_flags & PGO_SYNCIO) == 0) {
1567 mutex_exit(uobj->vmobjlock); 1537 mutex_exit(uobj->vmobjlock);
1568 return 0; 1538 return 0;
1569 } 1539 }
1570 npages = orignpages; 1540 npages = orignpages;
1571 uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL); 1541 uvn_findpages(uobj, origoffset, &npages, pgs, NULL, UFP_ALL);
1572 mutex_exit(uobj->vmobjlock); 1542 mutex_exit(uobj->vmobjlock);
1573 kva = uvm_pagermapin(pgs, npages, 1543 kva = uvm_pagermapin(pgs, npages,
1574 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1544 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK);
1575 for (i = 0; i < npages; i++) { 1545 for (i = 0; i < npages; i++) {
1576 pg = pgs[i]; 1546 pg = pgs[i];
1577 if ((pg->flags & PG_FAKE) == 0) { 1547 if ((pg->flags & PG_FAKE) == 0) {
1578 continue; 1548 continue;
1579 } 1549 }
1580 iov.iov_base = (char *)kva + (i << PAGE_SHIFT); 1550 iov.iov_base = (char *)kva + (i << PAGE_SHIFT);
1581 iov.iov_len = PAGE_SIZE; 1551 iov.iov_len = PAGE_SIZE;
1582 uio.uio_iov = &iov; 1552 uio.uio_iov = &iov;
1583 uio.uio_iovcnt = 1; 1553 uio.uio_iovcnt = 1;
1584 uio.uio_offset = origoffset + (i << PAGE_SHIFT); 1554 uio.uio_offset = origoffset + (i << PAGE_SHIFT);

cvs diff -r1.176.2.4 -r1.176.2.5 src/sys/uvm/uvm_extern.h (expand / switch to unified diff)

--- src/sys/uvm/uvm_extern.h 2011/11/20 10:52:33 1.176.2.4
+++ src/sys/uvm/uvm_extern.h 2011/12/20 13:46:17 1.176.2.5
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: uvm_extern.h,v 1.176.2.4 2011/11/20 10:52:33 yamt Exp $ */ 1/* $NetBSD: uvm_extern.h,v 1.176.2.5 2011/12/20 13:46:17 yamt Exp $ */
2 2
3/* 3/*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * Redistribution and use in source and binary forms, with or without 7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions 8 * modification, are permitted provided that the following conditions
9 * are met: 9 * are met:
10 * 1. Redistributions of source code must retain the above copyright 10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer. 11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright 12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the 13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution. 14 * documentation and/or other materials provided with the distribution.
@@ -200,26 +200,28 @@ typedef voff_t pgoff_t; /* XXX: number  @@ -200,26 +200,28 @@ typedef voff_t pgoff_t; /* XXX: number
200 */ 200 */
201#define UBC_PARTIALOK 0x100 201#define UBC_PARTIALOK 0x100
202 202
203/* 203/*
204 * flags for uvn_findpages(). 204 * flags for uvn_findpages().
205 */ 205 */
206#define UFP_ALL 0x00 206#define UFP_ALL 0x00
207#define UFP_NOWAIT 0x01 207#define UFP_NOWAIT 0x01
208#define UFP_NOALLOC 0x02 208#define UFP_NOALLOC 0x02
209#define UFP_NOCACHE 0x04 209#define UFP_NOCACHE 0x04
210#define UFP_NORDONLY 0x08 210#define UFP_NORDONLY 0x08
211#define UFP_DIRTYONLY 0x10 211#define UFP_DIRTYONLY 0x10
212#define UFP_BACKWARD 0x20 212#define UFP_BACKWARD 0x20
 213#define UFP_ONLYPAGER1 0x40
 214#define UFP_NOPAGER1 0x80
213 215
214/* 216/*
215 * lockflags that control the locking behavior of various functions. 217 * lockflags that control the locking behavior of various functions.
216 */ 218 */
217#define UVM_LK_ENTER 0x00000001 /* map locked on entry */ 219#define UVM_LK_ENTER 0x00000001 /* map locked on entry */
218#define UVM_LK_EXIT 0x00000002 /* leave map locked on exit */ 220#define UVM_LK_EXIT 0x00000002 /* leave map locked on exit */
219 221
220/* 222/*
221 * Default number of pages to allocate on the stack 223 * Default number of pages to allocate on the stack
222 */ 224 */
223#define UBC_MAX_PAGES 8 225#define UBC_MAX_PAGES 8
224 226
225/* 227/*
@@ -765,29 +767,31 @@ int uvm_pglistalloc(psize_t, paddr_t,  @@ -765,29 +767,31 @@ int uvm_pglistalloc(psize_t, paddr_t,
765 paddr_t, paddr_t, struct pglist *, int, int); 767 paddr_t, paddr_t, struct pglist *, int, int);
766void uvm_pglistfree(struct pglist *); 768void uvm_pglistfree(struct pglist *);
767 769
768/* uvm_swap.c */ 770/* uvm_swap.c */
769void uvm_swap_init(void); 771void uvm_swap_init(void);
770 772
771/* uvm_unix.c */ 773/* uvm_unix.c */
772int uvm_grow(struct proc *, vaddr_t); 774int uvm_grow(struct proc *, vaddr_t);
773 775
774/* uvm_user.c */ 776/* uvm_user.c */
775void uvm_deallocate(struct vm_map *, vaddr_t, vsize_t); 777void uvm_deallocate(struct vm_map *, vaddr_t, vsize_t);
776 778
777/* uvm_vnode.c */ 779/* uvm_vnode.c */
 780struct uvm_page_array;
778void uvm_vnp_setsize(struct vnode *, voff_t); 781void uvm_vnp_setsize(struct vnode *, voff_t);
779void uvm_vnp_setwritesize(struct vnode *, voff_t); 782void uvm_vnp_setwritesize(struct vnode *, voff_t);
780int uvn_findpages(struct uvm_object *, voff_t, 783int uvn_findpages(struct uvm_object *, voff_t,
781 int *, struct vm_page **, int); 784 unsigned int *, struct vm_page **,
 785 struct uvm_page_array *, unsigned int);
782bool uvn_text_p(struct uvm_object *); 786bool uvn_text_p(struct uvm_object *);
783bool uvn_clean_p(struct uvm_object *); 787bool uvn_clean_p(struct uvm_object *);
784bool uvn_needs_writefault_p(struct uvm_object *); 788bool uvn_needs_writefault_p(struct uvm_object *);
785 789
786/* kern_malloc.c */ 790/* kern_malloc.c */
787void kmeminit_nkmempages(void); 791void kmeminit_nkmempages(void);
788void kmeminit(void); 792void kmeminit(void);
789extern int nkmempages; 793extern int nkmempages;
790 794
791#endif /* _KERNEL */ 795#endif /* _KERNEL */
792 796
793#endif /* _UVM_UVM_EXTERN_H_ */ 797#endif /* _UVM_UVM_EXTERN_H_ */

cvs diff -r1.97.2.2 -r1.97.2.3 src/sys/uvm/uvm_vnode.c (expand / switch to unified diff)

--- src/sys/uvm/uvm_vnode.c 2011/11/26 15:19:06 1.97.2.2
+++ src/sys/uvm/uvm_vnode.c 2011/12/20 13:46:17 1.97.2.3
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: uvm_vnode.c,v 1.97.2.2 2011/11/26 15:19:06 yamt Exp $ */ 1/* $NetBSD: uvm_vnode.c,v 1.97.2.3 2011/12/20 13:46:17 yamt Exp $ */
2 2
3/* 3/*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * Copyright (c) 1991, 1993 5 * Copyright (c) 1991, 1993
6 * The Regents of the University of California. 6 * The Regents of the University of California.
7 * Copyright (c) 1990 University of Utah. 7 * Copyright (c) 1990 University of Utah.
8 * 8 *
9 * All rights reserved. 9 * All rights reserved.
10 * 10 *
11 * This code is derived from software contributed to Berkeley by 11 * This code is derived from software contributed to Berkeley by
12 * the Systems Programming Group of the University of Utah Computer 12 * the Systems Programming Group of the University of Utah Computer
13 * Science Department. 13 * Science Department.
14 * 14 *
@@ -35,27 +35,27 @@ @@ -35,27 +35,27 @@
35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
37 * SUCH DAMAGE. 37 * SUCH DAMAGE.
38 * 38 *
39 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94 39 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94
40 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp 40 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp
41 */ 41 */
42 42
43/* 43/*
44 * uvm_vnode.c: the vnode pager. 44 * uvm_vnode.c: the vnode pager.
45 */ 45 */
46 46
47#include <sys/cdefs.h> 47#include <sys/cdefs.h>
48__KERNEL_RCSID(0, "$NetBSD: uvm_vnode.c,v 1.97.2.2 2011/11/26 15:19:06 yamt Exp $"); 48__KERNEL_RCSID(0, "$NetBSD: uvm_vnode.c,v 1.97.2.3 2011/12/20 13:46:17 yamt Exp $");
49 49
50#include "opt_uvmhist.h" 50#include "opt_uvmhist.h"
51 51
52#include <sys/param.h> 52#include <sys/param.h>
53#include <sys/systm.h> 53#include <sys/systm.h>
54#include <sys/kernel.h> 54#include <sys/kernel.h>
55#include <sys/vnode.h> 55#include <sys/vnode.h>
56#include <sys/disklabel.h> 56#include <sys/disklabel.h>
57#include <sys/ioctl.h> 57#include <sys/ioctl.h>
58#include <sys/fcntl.h> 58#include <sys/fcntl.h>
59#include <sys/conf.h> 59#include <sys/conf.h>
60#include <sys/pool.h> 60#include <sys/pool.h>
61#include <sys/mount.h> 61#include <sys/mount.h>
@@ -67,27 +67,28 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_vnode.c, @@ -67,27 +67,28 @@ __KERNEL_RCSID(0, "$NetBSD: uvm_vnode.c,
67#include <uvm/uvm_page_array.h> 67#include <uvm/uvm_page_array.h>
68 68
69/* 69/*
70 * functions 70 * functions
71 */ 71 */
72 72
73static void uvn_detach(struct uvm_object *); 73static void uvn_detach(struct uvm_object *);
74static int uvn_get(struct uvm_object *, voff_t, struct vm_page **, int *, 74static int uvn_get(struct uvm_object *, voff_t, struct vm_page **, int *,
75 int, vm_prot_t, int, int); 75 int, vm_prot_t, int, int);
76static int uvn_put(struct uvm_object *, voff_t, voff_t, int); 76static int uvn_put(struct uvm_object *, voff_t, voff_t, int);
77static void uvn_reference(struct uvm_object *); 77static void uvn_reference(struct uvm_object *);
78 78
79static int uvn_findpage(struct uvm_object *, voff_t, struct vm_page **, 79static int uvn_findpage(struct uvm_object *, voff_t, struct vm_page **,
80 int, struct uvm_page_array *a, unsigned int); 80 unsigned int, struct uvm_page_array *a,
 81 unsigned int);
81 82
82/* 83/*
83 * master pager structure 84 * master pager structure
84 */ 85 */
85 86
86const struct uvm_pagerops uvm_vnodeops = { 87const struct uvm_pagerops uvm_vnodeops = {
87 .pgo_reference = uvn_reference, 88 .pgo_reference = uvn_reference,
88 .pgo_detach = uvn_detach, 89 .pgo_detach = uvn_detach,
89 .pgo_get = uvn_get, 90 .pgo_get = uvn_get,
90 .pgo_put = uvn_put, 91 .pgo_put = uvn_put,
91}; 92};
92 93
93/* 94/*
@@ -183,66 +184,72 @@ uvn_get(struct uvm_object *uobj, voff_t  @@ -183,66 +184,72 @@ uvn_get(struct uvm_object *uobj, voff_t
183 (flags & PGO_LOCKED) == 0); 184 (flags & PGO_LOCKED) == 0);
184 return error; 185 return error;
185} 186}
186 187
187 188
188/* 189/*
189 * uvn_findpages: 190 * uvn_findpages:
190 * return the page for the uobj and offset requested, allocating if needed. 191 * return the page for the uobj and offset requested, allocating if needed.
191 * => uobj must be locked. 192 * => uobj must be locked.
192 * => returned pages will be BUSY. 193 * => returned pages will be BUSY.
193 */ 194 */
194 195
195int 196int
196uvn_findpages(struct uvm_object *uobj, voff_t offset, int *npagesp, 197uvn_findpages(struct uvm_object *uobj, voff_t offset, unsigned int *npagesp,
197 struct vm_page **pgs, int flags) 198 struct vm_page **pgs, struct uvm_page_array *a, unsigned int flags)
198{ 199{
199 int i, count, found, npages, rv; 200 unsigned int count, found, npages;
200 struct uvm_page_array a; 201 int i, rv;
201 202 struct uvm_page_array a_store;
202 uvm_page_array_init(&a); 203
 204 if (a == NULL) {
 205 a = &a_store;
 206 uvm_page_array_init(a);
 207 }
203 count = found = 0; 208 count = found = 0;
204 npages = *npagesp; 209 npages = *npagesp;
205 if (flags & UFP_BACKWARD) { 210 if (flags & UFP_BACKWARD) {
206 for (i = npages - 1; i >= 0; i--, offset -= PAGE_SIZE) { 211 for (i = npages - 1; i >= 0; i--, offset -= PAGE_SIZE) {
207 rv = uvn_findpage(uobj, offset, &pgs[i], flags, &a, 212 rv = uvn_findpage(uobj, offset, &pgs[i], flags, a,
208 npages - i); 213 npages - i);
209 if (rv == 0) { 214 if (rv == 0) {
210 if (flags & UFP_DIRTYONLY) 215 if (flags & UFP_DIRTYONLY)
211 break; 216 break;
212 } else 217 } else
213 found++; 218 found++;
214 count++; 219 count++;
215 } 220 }
216 } else { 221 } else {
217 for (i = 0; i < npages; i++, offset += PAGE_SIZE) { 222 for (i = 0; i < npages; i++, offset += PAGE_SIZE) {
218 rv = uvn_findpage(uobj, offset, &pgs[i], flags, &a, 223 rv = uvn_findpage(uobj, offset, &pgs[i], flags, a,
219 npages - i); 224 npages - i);
220 if (rv == 0) { 225 if (rv == 0) {
221 if (flags & UFP_DIRTYONLY) 226 if (flags & UFP_DIRTYONLY)
222 break; 227 break;
223 } else 228 } else
224 found++; 229 found++;
225 count++; 230 count++;
226 } 231 }
227 } 232 }
228 uvm_page_array_fini(&a); 233 if (a == &a_store) {
 234 uvm_page_array_fini(a);
 235 }
229 *npagesp = count; 236 *npagesp = count;
230 return (found); 237 return (found);
231} 238}
232 239
233static int 240static int
234uvn_findpage(struct uvm_object *uobj, voff_t offset, struct vm_page **pgp, 241uvn_findpage(struct uvm_object *uobj, voff_t offset, struct vm_page **pgp,
235 int flags, struct uvm_page_array *a, unsigned int nleft) 242 unsigned int flags, struct uvm_page_array *a, unsigned int nleft)
236{ 243{
237 struct vm_page *pg; 244 struct vm_page *pg;
238 bool dirty; 245 bool dirty;
239 const unsigned int fillflags = 246 const unsigned int fillflags =
240 ((flags & UFP_BACKWARD) ? UVM_PAGE_ARRAY_FILL_BACKWARD : 0) || 247 ((flags & UFP_BACKWARD) ? UVM_PAGE_ARRAY_FILL_BACKWARD : 0) ||
241 ((flags & UFP_DIRTYONLY) ? 248 ((flags & UFP_DIRTYONLY) ?
242 (UVM_PAGE_ARRAY_FILL_DIRTYONLY|UVM_PAGE_ARRAY_FILL_DENSE) : 0); 249 (UVM_PAGE_ARRAY_FILL_DIRTYONLY|UVM_PAGE_ARRAY_FILL_DENSE) : 0);
243 UVMHIST_FUNC("uvn_findpage"); UVMHIST_CALLED(ubchist); 250 UVMHIST_FUNC("uvn_findpage"); UVMHIST_CALLED(ubchist);
244 UVMHIST_LOG(ubchist, "vp %p off 0x%lx", uobj, offset,0,0); 251 UVMHIST_LOG(ubchist, "vp %p off 0x%lx", uobj, offset,0,0);
245 252
246 KASSERT(mutex_owned(uobj->vmobjlock)); 253 KASSERT(mutex_owned(uobj->vmobjlock));
247 254
248 if (*pgp != NULL) { 255 if (*pgp != NULL) {
@@ -301,26 +308,40 @@ uvn_findpage(struct uvm_object *uobj, vo @@ -301,26 +308,40 @@ uvn_findpage(struct uvm_object *uobj, vo
301 "uvn_fp2", 0); 308 "uvn_fp2", 0);
302 uvm_page_array_clear(a); 309 uvm_page_array_clear(a);
303 mutex_enter(uobj->vmobjlock); 310 mutex_enter(uobj->vmobjlock);
304 continue; 311 continue;
305 } 312 }
306 313
307 /* skip PG_RDONLY and PG_HOLE pages if requested */ 314 /* skip PG_RDONLY and PG_HOLE pages if requested */
308 if ((flags & UFP_NORDONLY) && 315 if ((flags & UFP_NORDONLY) &&
309 (pg->flags & (PG_RDONLY|PG_HOLE))) { 316 (pg->flags & (PG_RDONLY|PG_HOLE))) {
310 UVMHIST_LOG(ubchist, "nordonly",0,0,0,0); 317 UVMHIST_LOG(ubchist, "nordonly",0,0,0,0);
311 goto skip; 318 goto skip;
312 } 319 }
313 320
 321 /*
 322 * check for PG_PAGER1 requests
 323 */
 324 if ((flags & UFP_NOPAGER1) != 0 &&
 325 (pg->flags & PG_PAGER1) != 0) {
 326 UVMHIST_LOG(ubchist, "nopager1",0,0,0,0);
 327 goto skip;
 328 }
 329 if ((flags & UFP_ONLYPAGER1) != 0 &&
 330 (pg->flags & PG_PAGER1) == 0) {
 331 UVMHIST_LOG(ubchist, "onlypager1",0,0,0,0);
 332 goto skip;
 333 }
 334
314 /* stop on clean pages if requested */ 335 /* stop on clean pages if requested */
315 if (flags & UFP_DIRTYONLY) { 336 if (flags & UFP_DIRTYONLY) {
316 dirty = uvm_pagecheckdirty(pg, false); 337 dirty = uvm_pagecheckdirty(pg, false);
317 if (!dirty) { 338 if (!dirty) {
318 UVMHIST_LOG(ubchist, "dirtonly", 0,0,0,0); 339 UVMHIST_LOG(ubchist, "dirtonly", 0,0,0,0);
319 return 0; 340 return 0;
320 } 341 }
321 } 342 }
322 343
323 /* mark the page BUSY and we're done. */ 344 /* mark the page BUSY and we're done. */
324 pg->flags |= PG_BUSY; 345 pg->flags |= PG_BUSY;
325 UVM_PAGE_OWN(pg, "uvn_findpage"); 346 UVM_PAGE_OWN(pg, "uvn_findpage");
326 UVMHIST_LOG(ubchist, "found %p (color %u)", 347 UVMHIST_LOG(ubchist, "found %p (color %u)",