Mon Dec 30 18:08:38 2019 UTC ()
pagedaemon:

- Use marker pages to keep place in the queue when scanning, rather than
  relying on assumptions.

- In uvmpdpol_balancequeue(), lock the object once instead of twice.

- When draining pools, the situation is getting desperate, but try to avoid
  saturating the system with xcall, lock and interrupt activity by sleeping
  for 1 clock tick if being continually awoken and all pools have been
  cycled through at least once.

- Pause & resume the freelist cache during pool draining.

PR kern/54209: NetBSD 8 large memory performance extremely low
PR kern/54210: NetBSD-8 processes presumably not exiting
PR kern/54727: writing a large file causes unreasonable system behaviour


(ad)
diff -r1.118 -r1.119 src/sys/uvm/uvm_pdaemon.c
diff -r1.17 -r1.18 src/sys/uvm/uvm_pdaemon.h
diff -r1.4 -r1.5 src/sys/uvm/uvm_pdpolicy.h
diff -r1.23 -r1.24 src/sys/uvm/uvm_pdpolicy_clock.c
diff -r1.19 -r1.20 src/sys/uvm/uvm_pdpolicy_clockpro.c

cvs diff -r1.118 -r1.119 src/sys/uvm/uvm_pdaemon.c (expand / switch to unified diff)

--- src/sys/uvm/uvm_pdaemon.c 2019/12/21 16:10:20 1.118
+++ src/sys/uvm/uvm_pdaemon.c 2019/12/30 18:08:37 1.119
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: uvm_pdaemon.c,v 1.118 2019/12/21 16:10:20 ad Exp $ */ 1/* $NetBSD: uvm_pdaemon.c,v 1.119 2019/12/30 18:08:37 ad Exp $ */
2 2
3/* 3/*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * Copyright (c) 1991, 1993, The Regents of the University of California. 5 * Copyright (c) 1991, 1993, The Regents of the University of California.
6 * 6 *
7 * All rights reserved. 7 * All rights reserved.
8 * 8 *
9 * This code is derived from software contributed to Berkeley by 9 * This code is derived from software contributed to Berkeley by
10 * The Mach Operating System project at Carnegie-Mellon University. 10 * The Mach Operating System project at Carnegie-Mellon University.
11 * 11 *
12 * Redistribution and use in source and binary forms, with or without 12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions 13 * modification, are permitted provided that the following conditions
14 * are met: 14 * are met:
@@ -56,43 +56,44 @@ @@ -56,43 +56,44 @@
56 * School of Computer Science 56 * School of Computer Science
57 * Carnegie Mellon University 57 * Carnegie Mellon University
58 * Pittsburgh PA 15213-3890 58 * Pittsburgh PA 15213-3890
59 * 59 *
60 * any improvements or extensions that they make and grant Carnegie the 60 * any improvements or extensions that they make and grant Carnegie the
61 * rights to redistribute these changes. 61 * rights to redistribute these changes.
62 */ 62 */
63 63
64/* 64/*
65 * uvm_pdaemon.c: the page daemon 65 * uvm_pdaemon.c: the page daemon
66 */ 66 */
67 67
68#include <sys/cdefs.h> 68#include <sys/cdefs.h>
69__KERNEL_RCSID(0, "$NetBSD: uvm_pdaemon.c,v 1.118 2019/12/21 16:10:20 ad Exp $"); 69__KERNEL_RCSID(0, "$NetBSD: uvm_pdaemon.c,v 1.119 2019/12/30 18:08:37 ad Exp $");
70 70
71#include "opt_uvmhist.h" 71#include "opt_uvmhist.h"
72#include "opt_readahead.h" 72#include "opt_readahead.h"
73 73
74#include <sys/param.h> 74#include <sys/param.h>
75#include <sys/proc.h> 75#include <sys/proc.h>
76#include <sys/systm.h> 76#include <sys/systm.h>
77#include <sys/kernel.h> 77#include <sys/kernel.h>
78#include <sys/pool.h> 78#include <sys/pool.h>
79#include <sys/buf.h> 79#include <sys/buf.h>
80#include <sys/module.h> 80#include <sys/module.h>
81#include <sys/atomic.h> 81#include <sys/atomic.h>
82#include <sys/kthread.h> 82#include <sys/kthread.h>
83 83
84#include <uvm/uvm.h> 84#include <uvm/uvm.h>
85#include <uvm/uvm_pdpolicy.h> 85#include <uvm/uvm_pdpolicy.h>
 86#include <uvm/uvm_pgflcache.h>
86 87
87#ifdef UVMHIST 88#ifdef UVMHIST
88UVMHIST_DEFINE(pdhist); 89UVMHIST_DEFINE(pdhist);
89#endif 90#endif
90 91
91/* 92/*
92 * UVMPD_NUMDIRTYREACTS is how many dirty pages the pagedaemon will reactivate 93 * UVMPD_NUMDIRTYREACTS is how many dirty pages the pagedaemon will reactivate
93 * in a pass thru the inactive list when swap is full. the value should be 94 * in a pass thru the inactive list when swap is full. the value should be
94 * "small"... if it's too large we'll cycle the active pages thru the inactive 95 * "small"... if it's too large we'll cycle the active pages thru the inactive
95 * queue too quickly to for them to be referenced and avoid being freed. 96 * queue too quickly to for them to be referenced and avoid being freed.
96 */ 97 */
97 98
98#define UVMPD_NUMDIRTYREACTS 16 99#define UVMPD_NUMDIRTYREACTS 16
@@ -588,94 +589,50 @@ static int @@ -588,94 +589,50 @@ static int
588swapcluster_nused(struct swapcluster *swc) 589swapcluster_nused(struct swapcluster *swc)
589{ 590{
590 591
591 return swc->swc_nused; 592 return swc->swc_nused;
592} 593}
593 594
594/* 595/*
595 * uvmpd_dropswap: free any swap allocated to this page. 596 * uvmpd_dropswap: free any swap allocated to this page.
596 * 597 *
597 * => called with owner locked. 598 * => called with owner locked.
598 * => return true if a page had an associated slot. 599 * => return true if a page had an associated slot.
599 */ 600 */
600 601
601static bool 602bool
602uvmpd_dropswap(struct vm_page *pg) 603uvmpd_dropswap(struct vm_page *pg)
603{ 604{
604 bool result = false; 605 bool result = false;
605 struct vm_anon *anon = pg->uanon; 606 struct vm_anon *anon = pg->uanon;
606 607
607 if ((pg->flags & PG_ANON) && anon->an_swslot) { 608 if ((pg->flags & PG_ANON) && anon->an_swslot) {
608 uvm_swap_free(anon->an_swslot, 1); 609 uvm_swap_free(anon->an_swslot, 1);
609 anon->an_swslot = 0; 610 anon->an_swslot = 0;
610 pg->flags &= ~PG_CLEAN; 611 pg->flags &= ~PG_CLEAN;
611 result = true; 612 result = true;
612 } else if (pg->flags & PG_AOBJ) { 613 } else if (pg->flags & PG_AOBJ) {
613 int slot = uao_set_swslot(pg->uobject, 614 int slot = uao_set_swslot(pg->uobject,
614 pg->offset >> PAGE_SHIFT, 0); 615 pg->offset >> PAGE_SHIFT, 0);
615 if (slot) { 616 if (slot) {
616 uvm_swap_free(slot, 1); 617 uvm_swap_free(slot, 1);
617 pg->flags &= ~PG_CLEAN; 618 pg->flags &= ~PG_CLEAN;
618 result = true; 619 result = true;
619 } 620 }
620 } 621 }
621 622
622 return result; 623 return result;
623} 624}
624 625
625/* 
626 * uvmpd_trydropswap: try to free any swap allocated to this page. 
627 * 
628 * => return true if a slot is successfully freed. 
629 * => page interlock must be held, and will be dropped. 
630 */ 
631 
632bool 
633uvmpd_trydropswap(struct vm_page *pg) 
634{ 
635 kmutex_t *slock; 
636 bool result; 
637 
638 if ((pg->flags & PG_BUSY) != 0) { 
639 mutex_exit(&pg->interlock); 
640 return false; 
641 } 
642 
643 /* 
644 * lock the page's owner. 
645 * this will drop pg->interlock. 
646 */ 
647 
648 slock = uvmpd_trylockowner(pg); 
649 if (slock == NULL) { 
650 return false; 
651 } 
652 
653 /* 
654 * skip this page if it's busy. 
655 */ 
656 
657 if ((pg->flags & PG_BUSY) != 0) { 
658 mutex_exit(slock); 
659 return false; 
660 } 
661 
662 result = uvmpd_dropswap(pg); 
663 
664 mutex_exit(slock); 
665 
666 return result; 
667} 
668 
669#endif /* defined(VMSWAP) */ 626#endif /* defined(VMSWAP) */
670 627
671/* 628/*
672 * uvmpd_scan_queue: scan an replace candidate list for pages 629 * uvmpd_scan_queue: scan an replace candidate list for pages
673 * to clean or free. 630 * to clean or free.
674 * 631 *
675 * => we work on meeting our free target by converting inactive pages 632 * => we work on meeting our free target by converting inactive pages
676 * into free pages. 633 * into free pages.
677 * => we handle the building of swap-backed clusters 634 * => we handle the building of swap-backed clusters
678 */ 635 */
679 636
680static void 637static void
681uvmpd_scan_queue(void) 638uvmpd_scan_queue(void)
@@ -899,26 +856,28 @@ uvmpd_scan_queue(void) @@ -899,26 +856,28 @@ uvmpd_scan_queue(void)
899 /* 856 /*
900 * the pageout is in progress. bump counters and set up 857 * the pageout is in progress. bump counters and set up
901 * for the next loop. 858 * for the next loop.
902 */ 859 */
903 860
904 atomic_inc_uint(&uvmexp.pdpending); 861 atomic_inc_uint(&uvmexp.pdpending);
905 862
906#else /* defined(VMSWAP) */ 863#else /* defined(VMSWAP) */
907 uvm_pageactivate(p); 864 uvm_pageactivate(p);
908 mutex_exit(slock); 865 mutex_exit(slock);
909#endif /* defined(VMSWAP) */ 866#endif /* defined(VMSWAP) */
910 } 867 }
911 868
 869 uvmpdpol_scanfini();
 870
912#if defined(VMSWAP) 871#if defined(VMSWAP)
913 swapcluster_flush(&swc, true); 872 swapcluster_flush(&swc, true);
914#endif /* defined(VMSWAP) */ 873#endif /* defined(VMSWAP) */
915} 874}
916 875
917/* 876/*
918 * uvmpd_scan: scan the page queues and attempt to meet our targets. 877 * uvmpd_scan: scan the page queues and attempt to meet our targets.
919 */ 878 */
920 879
921static void 880static void
922uvmpd_scan(void) 881uvmpd_scan(void)
923{ 882{
924 int swap_shortage, pages_freed, fpages; 883 int swap_shortage, pages_freed, fpages;
@@ -1021,51 +980,85 @@ uvm_estimatepageable(int *active, int *i @@ -1021,51 +980,85 @@ uvm_estimatepageable(int *active, int *i
1021 uvmpdpol_estimatepageable(active, inactive); 980 uvmpdpol_estimatepageable(active, inactive);
1022} 981}
1023 982
1024 983
1025/* 984/*
1026 * Use a separate thread for draining pools. 985 * Use a separate thread for draining pools.
1027 * This work can't done from the main pagedaemon thread because 986 * This work can't done from the main pagedaemon thread because
1028 * some pool allocators need to take vm_map locks. 987 * some pool allocators need to take vm_map locks.
1029 */ 988 */
1030 989
1031static void 990static void
1032uvmpd_pool_drain_thread(void *arg) 991uvmpd_pool_drain_thread(void *arg)
1033{ 992{
1034 int bufcnt; 993 struct pool *firstpool, *curpool;
 994 int bufcnt, lastslept;
 995 bool cycled;
1035 996
 997 firstpool = NULL;
 998 cycled = true;
1036 for (;;) { 999 for (;;) {
 1000 /*
 1001 * sleep until awoken by the pagedaemon.
 1002 */
1037 mutex_enter(&uvmpd_lock); 1003 mutex_enter(&uvmpd_lock);
1038 if (!uvmpd_pool_drain_run) { 1004 if (!uvmpd_pool_drain_run) {
 1005 lastslept = hardclock_ticks;
1039 cv_wait(&uvmpd_pool_drain_cv, &uvmpd_lock); 1006 cv_wait(&uvmpd_pool_drain_cv, &uvmpd_lock);
 1007 if (hardclock_ticks != lastslept) {
 1008 cycled = false;
 1009 firstpool = NULL;
 1010 }
1040 } 1011 }
1041 uvmpd_pool_drain_run = false; 1012 uvmpd_pool_drain_run = false;
1042 mutex_exit(&uvmpd_lock); 1013 mutex_exit(&uvmpd_lock);
1043 1014
1044 /* 1015 /*
 1016 * rate limit draining, otherwise in desperate circumstances
 1017 * this can totally saturate the system with xcall activity.
 1018 */
 1019 if (cycled) {
 1020 kpause("uvmpdlmt", false, 1, NULL);
 1021 cycled = false;
 1022 firstpool = NULL;
 1023 }
 1024
 1025 /*
 1026 * drain and temporarily disable the freelist cache.
 1027 */
 1028 uvm_pgflcache_pause();
 1029
 1030 /*
1045 * kill unused metadata buffers. 1031 * kill unused metadata buffers.
1046 */ 1032 */
1047 bufcnt = uvmexp.freetarg - uvm_free(); 1033 bufcnt = uvmexp.freetarg - uvm_free();
1048 if (bufcnt < 0) 1034 if (bufcnt < 0)
1049 bufcnt = 0; 1035 bufcnt = 0;
1050 1036
1051 mutex_enter(&bufcache_lock); 1037 mutex_enter(&bufcache_lock);
1052 buf_drain(bufcnt << PAGE_SHIFT); 1038 buf_drain(bufcnt << PAGE_SHIFT);
1053 mutex_exit(&bufcache_lock); 1039 mutex_exit(&bufcache_lock);
1054 1040
1055 /* 1041 /*
1056 * drain a pool. 1042 * drain a pool, and then re-enable the freelist cache.
1057 */ 1043 */
1058 pool_drain(NULL); 1044 (void)pool_drain(&curpool);
 1045 KASSERT(curpool != NULL);
 1046 if (firstpool == NULL) {
 1047 firstpool = curpool;
 1048 } else if (firstpool == curpool) {
 1049 cycled = true;
 1050 }
 1051 uvm_pgflcache_resume();
1059 } 1052 }
1060 /*NOTREACHED*/ 1053 /*NOTREACHED*/
1061} 1054}
1062 1055
1063static void 1056static void
1064uvmpd_pool_drain_wakeup(void) 1057uvmpd_pool_drain_wakeup(void)
1065{ 1058{
1066 1059
1067 mutex_enter(&uvmpd_lock); 1060 mutex_enter(&uvmpd_lock);
1068 uvmpd_pool_drain_run = true; 1061 uvmpd_pool_drain_run = true;
1069 cv_signal(&uvmpd_pool_drain_cv); 1062 cv_signal(&uvmpd_pool_drain_cv);
1070 mutex_exit(&uvmpd_lock); 1063 mutex_exit(&uvmpd_lock);
1071} 1064}

cvs diff -r1.17 -r1.18 src/sys/uvm/uvm_pdaemon.h (expand / switch to unified diff)

--- src/sys/uvm/uvm_pdaemon.h 2011/02/02 15:25:27 1.17
+++ src/sys/uvm/uvm_pdaemon.h 2019/12/30 18:08:38 1.18
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: uvm_pdaemon.h,v 1.17 2011/02/02 15:25:27 chuck Exp $ */ 1/* $NetBSD: uvm_pdaemon.h,v 1.18 2019/12/30 18:08:38 ad Exp $ */
2 2
3/* 3/*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * Copyright (c) 1991, 1993, The Regents of the University of California. 5 * Copyright (c) 1991, 1993, The Regents of the University of California.
6 * 6 *
7 * All rights reserved. 7 * All rights reserved.
8 * 8 *
9 * This code is derived from software contributed to Berkeley by 9 * This code is derived from software contributed to Berkeley by
10 * The Mach Operating System project at Carnegie-Mellon University. 10 * The Mach Operating System project at Carnegie-Mellon University.
11 * 11 *
12 * Redistribution and use in source and binary forms, with or without 12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions 13 * modification, are permitted provided that the following conditions
14 * are met: 14 * are met:
@@ -69,21 +69,21 @@ @@ -69,21 +69,21 @@
69 */ 69 */
70 70
71#ifdef _KERNEL 71#ifdef _KERNEL
72 72
73/* 73/*
74 * prototypes 74 * prototypes
75 */ 75 */
76 76
77void uvm_wait(const char *); 77void uvm_wait(const char *);
78bool uvm_reclaimable(void); 78bool uvm_reclaimable(void);
79 79
80kmutex_t *uvmpd_trylockowner(struct vm_page *); 80kmutex_t *uvmpd_trylockowner(struct vm_page *);
81#ifdef VMSWAP 81#ifdef VMSWAP
82bool uvmpd_trydropswap(struct vm_page *); 82bool uvmpd_dropswap(struct vm_page *);
83#else 83#else
84#define uvmpd_trydropswap(_a_) (/*CONSTCOND*/false) 84#define uvmpd_dropswap(_a_) (/*CONSTCOND*/false)
85#endif 85#endif
86 86
87#endif /* _KERNEL */ 87#endif /* _KERNEL */
88 88
89#endif /* _UVM_UVM_PDAEMON_H_ */ 89#endif /* _UVM_UVM_PDAEMON_H_ */

cvs diff -r1.4 -r1.5 src/sys/uvm/uvm_pdpolicy.h (expand / switch to unified diff)

--- src/sys/uvm/uvm_pdpolicy.h 2019/12/13 20:10:22 1.4
+++ src/sys/uvm/uvm_pdpolicy.h 2019/12/30 18:08:38 1.5
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: uvm_pdpolicy.h,v 1.4 2019/12/13 20:10:22 ad Exp $ */ 1/* $NetBSD: uvm_pdpolicy.h,v 1.5 2019/12/30 18:08:38 ad Exp $ */
2 2
3/*- 3/*-
4 * Copyright (c)2005, 2006 YAMAMOTO Takashi, 4 * Copyright (c)2005, 2006 YAMAMOTO Takashi,
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * Redistribution and use in source and binary forms, with or without 7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions 8 * modification, are permitted provided that the following conditions
9 * are met: 9 * are met:
10 * 1. Redistributions of source code must retain the above copyright 10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer. 11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright 12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the 13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution. 14 * documentation and/or other materials provided with the distribution.
@@ -41,19 +41,20 @@ void uvmpdpol_init(void); @@ -41,19 +41,20 @@ void uvmpdpol_init(void);
41void uvmpdpol_reinit(void); 41void uvmpdpol_reinit(void);
42void uvmpdpol_estimatepageable(int *, int *); 42void uvmpdpol_estimatepageable(int *, int *);
43bool uvmpdpol_needsscan_p(void); 43bool uvmpdpol_needsscan_p(void);
44 44
45void uvmpdpol_pageactivate(struct vm_page *); 45void uvmpdpol_pageactivate(struct vm_page *);
46void uvmpdpol_pagedeactivate(struct vm_page *); 46void uvmpdpol_pagedeactivate(struct vm_page *);
47void uvmpdpol_pagedequeue(struct vm_page *); 47void uvmpdpol_pagedequeue(struct vm_page *);
48void uvmpdpol_pageenqueue(struct vm_page *); 48void uvmpdpol_pageenqueue(struct vm_page *);
49bool uvmpdpol_pageisqueued_p(struct vm_page *); 49bool uvmpdpol_pageisqueued_p(struct vm_page *);
50void uvmpdpol_anfree(struct vm_anon *); 50void uvmpdpol_anfree(struct vm_anon *);
51 51
52void uvmpdpol_tune(void); 52void uvmpdpol_tune(void);
53void uvmpdpol_scaninit(void); 53void uvmpdpol_scaninit(void);
 54void uvmpdpol_scanfini(void);
54struct vm_page *uvmpdpol_selectvictim(kmutex_t **lock); 55struct vm_page *uvmpdpol_selectvictim(kmutex_t **lock);
55void uvmpdpol_balancequeue(int); 56void uvmpdpol_balancequeue(int);
56 57
57void uvmpdpol_sysctlsetup(void); 58void uvmpdpol_sysctlsetup(void);
58 59
59#endif /* !_UVM_PDPOLICY_H_ */ 60#endif /* !_UVM_PDPOLICY_H_ */

cvs diff -r1.23 -r1.24 src/sys/uvm/uvm_pdpolicy_clock.c (expand / switch to unified diff)

--- src/sys/uvm/uvm_pdpolicy_clock.c 2019/12/27 13:13:17 1.23
+++ src/sys/uvm/uvm_pdpolicy_clock.c 2019/12/30 18:08:38 1.24
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: uvm_pdpolicy_clock.c,v 1.23 2019/12/27 13:13:17 ad Exp $ */ 1/* $NetBSD: uvm_pdpolicy_clock.c,v 1.24 2019/12/30 18:08:38 ad Exp $ */
2/* NetBSD: uvm_pdaemon.c,v 1.72 2006/01/05 10:47:33 yamt Exp $ */ 2/* NetBSD: uvm_pdaemon.c,v 1.72 2006/01/05 10:47:33 yamt Exp $ */
3 3
4/* 4/*
5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * Copyright (c) 1997 Charles D. Cranor and Washington University.
6 * Copyright (c) 1991, 1993, The Regents of the University of California. 6 * Copyright (c) 1991, 1993, The Regents of the University of California.
7 * 7 *
8 * All rights reserved. 8 * All rights reserved.
9 * 9 *
10 * This code is derived from software contributed to Berkeley by 10 * This code is derived from software contributed to Berkeley by
11 * The Mach Operating System project at Carnegie-Mellon University. 11 * The Mach Operating System project at Carnegie-Mellon University.
12 * 12 *
13 * Redistribution and use in source and binary forms, with or without 13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions 14 * modification, are permitted provided that the following conditions
@@ -59,27 +59,27 @@ @@ -59,27 +59,27 @@
59 * Pittsburgh PA 15213-3890 59 * Pittsburgh PA 15213-3890
60 * 60 *
61 * any improvements or extensions that they make and grant Carnegie the 61 * any improvements or extensions that they make and grant Carnegie the
62 * rights to redistribute these changes. 62 * rights to redistribute these changes.
63 */ 63 */
64 64
65#if defined(PDSIM) 65#if defined(PDSIM)
66 66
67#include "pdsim.h" 67#include "pdsim.h"
68 68
69#else /* defined(PDSIM) */ 69#else /* defined(PDSIM) */
70 70
71#include <sys/cdefs.h> 71#include <sys/cdefs.h>
72__KERNEL_RCSID(0, "$NetBSD: uvm_pdpolicy_clock.c,v 1.23 2019/12/27 13:13:17 ad Exp $"); 72__KERNEL_RCSID(0, "$NetBSD: uvm_pdpolicy_clock.c,v 1.24 2019/12/30 18:08:38 ad Exp $");
73 73
74#include <sys/param.h> 74#include <sys/param.h>
75#include <sys/proc.h> 75#include <sys/proc.h>
76#include <sys/systm.h> 76#include <sys/systm.h>
77#include <sys/kernel.h> 77#include <sys/kernel.h>
78 78
79#include <uvm/uvm.h> 79#include <uvm/uvm.h>
80#include <uvm/uvm_pdpolicy.h> 80#include <uvm/uvm_pdpolicy.h>
81#include <uvm/uvm_pdpolicy_impl.h> 81#include <uvm/uvm_pdpolicy_impl.h>
82#include <uvm/uvm_stat.h> 82#include <uvm/uvm_stat.h>
83 83
84#endif /* defined(PDSIM) */ 84#endif /* defined(PDSIM) */
85 85
@@ -100,29 +100,28 @@ struct uvmpdpol_globalstate { @@ -100,29 +100,28 @@ struct uvmpdpol_globalstate {
100 int s_active; 100 int s_active;
101 int s_inactive; 101 int s_inactive;
102 int s_inactarg; 102 int s_inactarg;
103 struct uvm_pctparam s_anonmin; 103 struct uvm_pctparam s_anonmin;
104 struct uvm_pctparam s_filemin; 104 struct uvm_pctparam s_filemin;
105 struct uvm_pctparam s_execmin; 105 struct uvm_pctparam s_execmin;
106 struct uvm_pctparam s_anonmax; 106 struct uvm_pctparam s_anonmax;
107 struct uvm_pctparam s_filemax; 107 struct uvm_pctparam s_filemax;
108 struct uvm_pctparam s_execmax; 108 struct uvm_pctparam s_execmax;
109 struct uvm_pctparam s_inactivepct; 109 struct uvm_pctparam s_inactivepct;
110}; 110};
111 111
112struct uvmpdpol_scanstate { 112struct uvmpdpol_scanstate {
113 bool ss_first; 
114 bool ss_anonreact, ss_filereact, ss_execreact; 113 bool ss_anonreact, ss_filereact, ss_execreact;
115 struct vm_page *ss_nextpg; 114 struct vm_page ss_marker;
116}; 115};
117 116
118static void uvmpdpol_pageactivate_locked(struct vm_page *); 117static void uvmpdpol_pageactivate_locked(struct vm_page *);
119static void uvmpdpol_pagedeactivate_locked(struct vm_page *); 118static void uvmpdpol_pagedeactivate_locked(struct vm_page *);
120static void uvmpdpol_pagedequeue_locked(struct vm_page *); 119static void uvmpdpol_pagedequeue_locked(struct vm_page *);
121 120
122static struct uvmpdpol_globalstate pdpol_state __cacheline_aligned; 121static struct uvmpdpol_globalstate pdpol_state __cacheline_aligned;
123static struct uvmpdpol_scanstate pdpol_scanstate; 122static struct uvmpdpol_scanstate pdpol_scanstate;
124 123
125PDPOL_EVCNT_DEFINE(reactexec) 124PDPOL_EVCNT_DEFINE(reactexec)
126PDPOL_EVCNT_DEFINE(reactfile) 125PDPOL_EVCNT_DEFINE(reactfile)
127PDPOL_EVCNT_DEFINE(reactanon) 126PDPOL_EVCNT_DEFINE(reactanon)
128 127
@@ -167,74 +166,86 @@ uvmpdpol_scaninit(void) @@ -167,74 +166,86 @@ uvmpdpol_scaninit(void)
167 execunder = execpg <= UVM_PCTPARAM_APPLY(&s->s_execmin, t); 166 execunder = execpg <= UVM_PCTPARAM_APPLY(&s->s_execmin, t);
168 anonover = anonpg > UVM_PCTPARAM_APPLY(&s->s_anonmax, t); 167 anonover = anonpg > UVM_PCTPARAM_APPLY(&s->s_anonmax, t);
169 fileover = filepg > UVM_PCTPARAM_APPLY(&s->s_filemax, t); 168 fileover = filepg > UVM_PCTPARAM_APPLY(&s->s_filemax, t);
170 execover = execpg > UVM_PCTPARAM_APPLY(&s->s_execmax, t); 169 execover = execpg > UVM_PCTPARAM_APPLY(&s->s_execmax, t);
171 anonreact = anonunder || (!anonover && (fileover || execover)); 170 anonreact = anonunder || (!anonover && (fileover || execover));
172 filereact = fileunder || (!fileover && (anonover || execover)); 171 filereact = fileunder || (!fileover && (anonover || execover));
173 execreact = execunder || (!execover && (anonover || fileover)); 172 execreact = execunder || (!execover && (anonover || fileover));
174 if (filereact && execreact && (anonreact || uvm_swapisfull())) { 173 if (filereact && execreact && (anonreact || uvm_swapisfull())) {
175 anonreact = filereact = execreact = false; 174 anonreact = filereact = execreact = false;
176 } 175 }
177 ss->ss_anonreact = anonreact; 176 ss->ss_anonreact = anonreact;
178 ss->ss_filereact = filereact; 177 ss->ss_filereact = filereact;
179 ss->ss_execreact = execreact; 178 ss->ss_execreact = execreact;
 179 memset(&ss->ss_marker, 0, sizeof(ss->ss_marker));
 180 ss->ss_marker.flags = PG_MARKER;
 181 TAILQ_INSERT_HEAD(&pdpol_state.s_inactiveq, &ss->ss_marker, pdqueue);
 182 mutex_exit(&s->lock);
 183}
 184
 185void
 186uvmpdpol_scanfini(void)
 187{
 188 struct uvmpdpol_globalstate *s = &pdpol_state;
 189 struct uvmpdpol_scanstate *ss = &pdpol_scanstate;
180 190
181 ss->ss_first = true; 191 mutex_enter(&s->lock);
 192 TAILQ_REMOVE(&pdpol_state.s_inactiveq, &ss->ss_marker, pdqueue);
182 mutex_exit(&s->lock); 193 mutex_exit(&s->lock);
183} 194}
184 195
185struct vm_page * 196struct vm_page *
186uvmpdpol_selectvictim(kmutex_t **plock) 197uvmpdpol_selectvictim(kmutex_t **plock)
187{ 198{
188 struct uvmpdpol_globalstate *s = &pdpol_state; 199 struct uvmpdpol_globalstate *s = &pdpol_state;
189 struct uvmpdpol_scanstate *ss = &pdpol_scanstate; 200 struct uvmpdpol_scanstate *ss = &pdpol_scanstate;
190 struct vm_page *pg; 201 struct vm_page *pg;
191 kmutex_t *lock; 202 kmutex_t *lock;
192 203
193 mutex_enter(&s->lock); 204 mutex_enter(&s->lock);
194 while (/* CONSTCOND */ 1) { 205 while (/* CONSTCOND */ 1) {
195 struct vm_anon *anon; 206 struct vm_anon *anon;
196 struct uvm_object *uobj; 207 struct uvm_object *uobj;
197 208
198 if (ss->ss_first) { 209 pg = TAILQ_NEXT(&ss->ss_marker, pdqueue);
199 pg = TAILQ_FIRST(&pdpol_state.s_inactiveq); 
200 ss->ss_first = false; 
201 } else { 
202 pg = ss->ss_nextpg; 
203 if (pg != NULL && (pg->pqflags & PQ_INACTIVE) == 0) { 
204 pg = TAILQ_FIRST(&pdpol_state.s_inactiveq); 
205 } 
206 } 
207 if (pg == NULL) { 210 if (pg == NULL) {
208 break; 211 break;
209 } 212 }
210 ss->ss_nextpg = TAILQ_NEXT(pg, pdqueue); 213 KASSERT((pg->flags & PG_MARKER) == 0);
211 uvmexp.pdscans++; 214 uvmexp.pdscans++;
212 215
213 /* 216 /*
214 * acquire interlock to stablize page identity. 217 * acquire interlock to stablize page identity.
215 * if we have caught the page in a state of flux 218 * if we have caught the page in a state of flux
216 * and it should be dequeued, do it now and then 219 * and it should be dequeued, do it now and then
217 * move on to the next. 220 * move on to the next.
218 */ 221 */
219 mutex_enter(&pg->interlock); 222 mutex_enter(&pg->interlock);
220 if ((pg->uobject == NULL && pg->uanon == NULL) || 223 if ((pg->uobject == NULL && pg->uanon == NULL) ||
221 pg->wire_count > 0) { 224 pg->wire_count > 0) {
222 mutex_exit(&pg->interlock); 225 mutex_exit(&pg->interlock);
223 uvmpdpol_pagedequeue_locked(pg); 226 uvmpdpol_pagedequeue_locked(pg);
224 continue; 227 continue;
225 } 228 }
226 229
227 /* 230 /*
 231 * now prepare to move on to the next page.
 232 */
 233 TAILQ_REMOVE(&pdpol_state.s_inactiveq, &ss->ss_marker,
 234 pdqueue);
 235 TAILQ_INSERT_AFTER(&pdpol_state.s_inactiveq, pg,
 236 &ss->ss_marker, pdqueue);
 237
 238 /*
228 * enforce the minimum thresholds on different 239 * enforce the minimum thresholds on different
229 * types of memory usage. if reusing the current 240 * types of memory usage. if reusing the current
230 * page would reduce that type of usage below its 241 * page would reduce that type of usage below its
231 * minimum, reactivate the page instead and move 242 * minimum, reactivate the page instead and move
232 * on to the next page. 243 * on to the next page.
233 */ 244 */
234 anon = pg->uanon; 245 anon = pg->uanon;
235 uobj = pg->uobject; 246 uobj = pg->uobject;
236 if (uobj && UVM_OBJ_IS_VTEXT(uobj) && ss->ss_execreact) { 247 if (uobj && UVM_OBJ_IS_VTEXT(uobj) && ss->ss_execreact) {
237 mutex_exit(&pg->interlock); 248 mutex_exit(&pg->interlock);
238 uvmpdpol_pageactivate_locked(pg); 249 uvmpdpol_pageactivate_locked(pg);
239 PDPOL_EVCNT_INCR(reactexec); 250 PDPOL_EVCNT_INCR(reactexec);
240 continue; 251 continue;
@@ -290,87 +301,108 @@ uvmpdpol_selectvictim(kmutex_t **plock) @@ -290,87 +301,108 @@ uvmpdpol_selectvictim(kmutex_t **plock)
290 /* we have a potential victim. */ 301 /* we have a potential victim. */
291 *plock = lock; 302 *plock = lock;
292 break; 303 break;
293 } 304 }
294 mutex_exit(&s->lock); 305 mutex_exit(&s->lock);
295 return pg; 306 return pg;
296} 307}
297 308
298void 309void
299uvmpdpol_balancequeue(int swap_shortage) 310uvmpdpol_balancequeue(int swap_shortage)
300{ 311{
301 struct uvmpdpol_globalstate *s = &pdpol_state; 312 struct uvmpdpol_globalstate *s = &pdpol_state;
302 int inactive_shortage; 313 int inactive_shortage;
303 struct vm_page *p, *nextpg; 314 struct vm_page *p, marker;
304 kmutex_t *lock; 315 kmutex_t *lock;
305 316
306 /* 317 /*
307 * we have done the scan to get free pages. now we work on meeting 318 * we have done the scan to get free pages. now we work on meeting
308 * our inactive target. 319 * our inactive target.
309 */ 320 */
310 321
311 mutex_enter(&s->lock); 322 memset(&marker, 0, sizeof(marker));
312 inactive_shortage = pdpol_state.s_inactarg - pdpol_state.s_inactive; 323 marker.flags = PG_MARKER;
313 for (p = TAILQ_FIRST(&pdpol_state.s_activeq); 
314 p != NULL && (inactive_shortage > 0 || swap_shortage > 0); 
315 p = nextpg) { 
316 nextpg = TAILQ_NEXT(p, pdqueue); 
317 324
318 /* 325 mutex_enter(&s->lock);
319 * if there's a shortage of swap slots, try to free it. 326 TAILQ_INSERT_HEAD(&pdpol_state.s_activeq, &marker, pdqueue);
320 */ 327 for (;;) {
321 328 inactive_shortage =
322 if (swap_shortage > 0 && (p->flags & PG_SWAPBACKED) != 0) { 329 pdpol_state.s_inactarg - pdpol_state.s_inactive;
323 mutex_enter(&p->interlock); 330 if (inactive_shortage <= 0 && swap_shortage <= 0) {
324 mutex_exit(&s->lock); 331 break;
325 if (uvmpd_trydropswap(p)) { 
326 swap_shortage--; 
327 } 
328 /* p->interlock now released */ 
329 mutex_enter(&s->lock); 
330 } 332 }
331 333 p = TAILQ_NEXT(&marker, pdqueue);
332 /* 334 if (p == NULL) {
333 * if there's a shortage of inactive pages, deactivate. 335 break;
334 */ 
335 
336 if (inactive_shortage <= 0) { 
337 continue; 
338 } 336 }
 337 KASSERT((p->flags & PG_MARKER) == 0);
339 338
340 /* 339 /*
341 * acquire interlock to stablize page identity. 340 * acquire interlock to stablize page identity.
342 * if we have caught the page in a state of flux 341 * if we have caught the page in a state of flux
343 * and it should be dequeued, do it now and then 342 * and it should be dequeued, do it now and then
344 * move on to the next. 343 * move on to the next.
345 */ 344 */
346 mutex_enter(&p->interlock); 345 mutex_enter(&p->interlock);
347 if ((p->uobject == NULL && p->uanon == NULL) || 346 if ((p->uobject == NULL && p->uanon == NULL) ||
348 p->wire_count > 0) { 347 p->wire_count > 0) {
349 mutex_exit(&p->interlock); 348 mutex_exit(&p->interlock);
350 uvmpdpol_pagedequeue_locked(p); 349 uvmpdpol_pagedequeue_locked(p);
351 continue; 350 continue;
352 } 351 }
353 mutex_exit(&s->lock); 352
354 lock = uvmpd_trylockowner(p); 353 /*
355 /* p->interlock now released */ 354 * now prepare to move on to the next page.
356 mutex_enter(&s->lock); 355 */
357 if (lock != NULL) { 356
 357 TAILQ_REMOVE(&pdpol_state.s_activeq, &marker, pdqueue);
 358 TAILQ_INSERT_AFTER(&pdpol_state.s_activeq, p, &marker,
 359 pdqueue);
 360
 361 /*
 362 * try to lock the object that owns the page. see comments
 363 * in uvmpdol_selectvictim().
 364 */
 365 mutex_exit(&s->lock);
 366 lock = uvmpd_trylockowner(p);
 367 /* p->interlock now released */
 368 mutex_enter(&s->lock);
 369 if (lock == NULL) {
 370 /* didn't get it - try the next page. */
 371 continue;
 372 }
 373
 374 /*
 375 * if there's a shortage of swap slots, try to free it.
 376 */
 377 if (swap_shortage > 0 && (p->flags & PG_SWAPBACKED) != 0 &&
 378 (p->flags & PG_BUSY) == 0) {
 379 if (uvmpd_dropswap(p)) {
 380 swap_shortage--;
 381 }
 382 }
 383
 384 /*
 385 * if there's a shortage of inactive pages, deactivate.
 386 */
 387
 388 if (inactive_shortage > 0) {
358 uvmpdpol_pagedeactivate_locked(p); 389 uvmpdpol_pagedeactivate_locked(p);
359 uvmexp.pddeact++; 390 uvmexp.pddeact++;
360 inactive_shortage--; 391 inactive_shortage--;
361 mutex_exit(lock); 
362 } 392 }
 393 mutex_exit(lock);
363 } 394 }
 395 TAILQ_REMOVE(&pdpol_state.s_activeq, &marker, pdqueue);
364 mutex_exit(&s->lock); 396 mutex_exit(&s->lock);
365} 397}
366 398
367static void 399static void
368uvmpdpol_pagedeactivate_locked(struct vm_page *pg) 400uvmpdpol_pagedeactivate_locked(struct vm_page *pg)
369{ 401{
370 402
371 KASSERT(uvm_page_locked_p(pg)); 403 KASSERT(uvm_page_locked_p(pg));
372 404
373 if (pg->pqflags & PQ_ACTIVE) { 405 if (pg->pqflags & PQ_ACTIVE) {
374 TAILQ_REMOVE(&pdpol_state.s_activeq, pg, pdqueue); 406 TAILQ_REMOVE(&pdpol_state.s_activeq, pg, pdqueue);
375 pg->pqflags &= ~(PQ_ACTIVE | PQ_TIME); 407 pg->pqflags &= ~(PQ_ACTIVE | PQ_TIME);
376 KASSERT(pdpol_state.s_active > 0); 408 KASSERT(pdpol_state.s_active > 0);

cvs diff -r1.19 -r1.20 src/sys/uvm/uvm_pdpolicy_clockpro.c (expand / switch to unified diff)

--- src/sys/uvm/uvm_pdpolicy_clockpro.c 2019/12/27 13:13:17 1.19
+++ src/sys/uvm/uvm_pdpolicy_clockpro.c 2019/12/30 18:08:38 1.20
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: uvm_pdpolicy_clockpro.c,v 1.19 2019/12/27 13:13:17 ad Exp $ */ 1/* $NetBSD: uvm_pdpolicy_clockpro.c,v 1.20 2019/12/30 18:08:38 ad Exp $ */
2 2
3/*- 3/*-
4 * Copyright (c)2005, 2006 YAMAMOTO Takashi, 4 * Copyright (c)2005, 2006 YAMAMOTO Takashi,
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * Redistribution and use in source and binary forms, with or without 7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions 8 * modification, are permitted provided that the following conditions
9 * are met: 9 * are met:
10 * 1. Redistributions of source code must retain the above copyright 10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer. 11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright 12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the 13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution. 14 * documentation and/or other materials provided with the distribution.
@@ -33,27 +33,27 @@ @@ -33,27 +33,27 @@
33 * approximation of the list of non-resident pages using hash: 33 * approximation of the list of non-resident pages using hash:
34 * http://linux-mm.org/ClockProApproximation 34 * http://linux-mm.org/ClockProApproximation
35 */ 35 */
36 36
37/* #define CLOCKPRO_DEBUG */ 37/* #define CLOCKPRO_DEBUG */
38 38
39#if defined(PDSIM) 39#if defined(PDSIM)
40 40
41#include "pdsim.h" 41#include "pdsim.h"
42 42
43#else /* defined(PDSIM) */ 43#else /* defined(PDSIM) */
44 44
45#include <sys/cdefs.h> 45#include <sys/cdefs.h>
46__KERNEL_RCSID(0, "$NetBSD: uvm_pdpolicy_clockpro.c,v 1.19 2019/12/27 13:13:17 ad Exp $"); 46__KERNEL_RCSID(0, "$NetBSD: uvm_pdpolicy_clockpro.c,v 1.20 2019/12/30 18:08:38 ad Exp $");
47 47
48#include "opt_ddb.h" 48#include "opt_ddb.h"
49 49
50#include <sys/param.h> 50#include <sys/param.h>
51#include <sys/proc.h> 51#include <sys/proc.h>
52#include <sys/systm.h> 52#include <sys/systm.h>
53#include <sys/kernel.h> 53#include <sys/kernel.h>
54#include <sys/hash.h> 54#include <sys/hash.h>
55 55
56#include <uvm/uvm.h> 56#include <uvm/uvm.h>
57#include <uvm/uvm_pdaemon.h> /* for uvmpd_trylockowner */ 57#include <uvm/uvm_pdaemon.h> /* for uvmpd_trylockowner */
58#include <uvm/uvm_pdpolicy.h> 58#include <uvm/uvm_pdpolicy.h>
59#include <uvm/uvm_pdpolicy_impl.h> 59#include <uvm/uvm_pdpolicy_impl.h>
@@ -1248,26 +1248,32 @@ uvmpdpol_pageisqueued_p(struct vm_page * @@ -1248,26 +1248,32 @@ uvmpdpol_pageisqueued_p(struct vm_page *
1248} 1248}
1249 1249
1250void 1250void
1251uvmpdpol_scaninit(void) 1251uvmpdpol_scaninit(void)
1252{ 1252{
1253 struct clockpro_state * const s = &clockpro; 1253 struct clockpro_state * const s = &clockpro;
1254 struct clockpro_scanstate * const ss = &scanstate; 1254 struct clockpro_scanstate * const ss = &scanstate;
1255 1255
1256 mutex_enter(&s->lock); 1256 mutex_enter(&s->lock);
1257 ss->ss_nscanned = 0; 1257 ss->ss_nscanned = 0;
1258 mutex_exit(&s->lock); 1258 mutex_exit(&s->lock);
1259} 1259}
1260 1260
 1261void
 1262uvmpdpol_scanfini(void)
 1263{
 1264
 1265}
 1266
1261struct vm_page * 1267struct vm_page *
1262uvmpdpol_selectvictim(kmutex_t **plock) 1268uvmpdpol_selectvictim(kmutex_t **plock)
1263{ 1269{
1264 struct clockpro_state * const s = &clockpro; 1270 struct clockpro_state * const s = &clockpro;
1265 struct clockpro_scanstate * const ss = &scanstate; 1271 struct clockpro_scanstate * const ss = &scanstate;
1266 struct vm_page *pg; 1272 struct vm_page *pg;
1267 kmutex_t *lock = NULL; 1273 kmutex_t *lock = NULL;
1268 1274
1269 do { 1275 do {
1270 mutex_enter(&s->lock); 1276 mutex_enter(&s->lock);
1271 if (ss->ss_nscanned > s->s_npages) { 1277 if (ss->ss_nscanned > s->s_npages) {
1272 DPRINTF("scan too much\n"); 1278 DPRINTF("scan too much\n");
1273 mutex_exit(&s->lock); 1279 mutex_exit(&s->lock);
@@ -1295,45 +1301,66 @@ uvmpdpol_selectvictim(kmutex_t **plock) @@ -1295,45 +1301,66 @@ uvmpdpol_selectvictim(kmutex_t **plock)
1295 } 1301 }
1296 mutex_exit(&s->lock); 1302 mutex_exit(&s->lock);
1297 lock = uvmpd_trylockowner(pg); 1303 lock = uvmpd_trylockowner(pg);
1298 /* pg->interlock now dropped */ 1304 /* pg->interlock now dropped */
1299 } while (lock == NULL); 1305 } while (lock == NULL);
1300 *plock = lock; 1306 *plock = lock;
1301 return pg; 1307 return pg;
1302} 1308}
1303 1309
1304static void 1310static void
1305clockpro_dropswap(pageq_t *q, int *todo) 1311clockpro_dropswap(pageq_t *q, int *todo)
1306{ 1312{
1307 struct vm_page *pg; 1313 struct vm_page *pg;
 1314 kmutex_t *lock;
1308 1315
1309 KASSERT(mutex_owned(&clockpro.lock)); 1316 KASSERT(mutex_owned(&clockpro.lock));
1310 1317
1311 TAILQ_FOREACH_REVERSE(pg, &q->q_q, pglist, pdqueue) { 1318 TAILQ_FOREACH_REVERSE(pg, &q->q_q, pglist, pdqueue) {
1312 if (*todo <= 0) { 1319 if (*todo <= 0) {
1313 break; 1320 break;
1314 } 1321 }
1315 if ((pg->pqflags & PQ_HOT) == 0) { 1322 if ((pg->pqflags & PQ_HOT) == 0) {
1316 continue; 1323 continue;
1317 } 1324 }
1318 mutex_enter(&pg->interlock); 1325 mutex_enter(&pg->interlock);
1319 if ((pg->flags & PG_SWAPBACKED) == 0) { 1326 if ((pg->flags & PG_SWAPBACKED) == 0) {
1320 mutex_exit(&pg->interlock); 1327 mutex_exit(&pg->interlock);
1321 continue; 1328 continue;
1322 } 1329 }
1323 if (uvmpd_trydropswap(pg)) { 1330
1324 (*todo)--; 1331 /*
 1332 * try to lock the object that owns the page.
 1333 */
 1334 mutex_exit(&clockpro.lock);
 1335 lock = uvmpd_trylockowner(pg);
 1336 /* pg->interlock now released */
 1337 mutex_enter(&clockpro.lock);
 1338 if (lock == NULL) {
 1339 /* didn't get it - try the next page. */
 1340 /* XXXAD lost position in queue */
 1341 continue;
1325 } 1342 }
1326 /* pg->interlock now dropped */ 1343
 1344 /*
 1345 * if there's a shortage of swap slots, try to free it.
 1346 */
 1347 if ((pg->flags & PG_SWAPBACKED) != 0 &&
 1348 (pg->flags & PG_BUSY) == 0) {
 1349 if (uvmpd_dropswap(pg)) {
 1350 (*todo)--;
 1351 }
 1352 }
 1353 mutex_exit(lock);
1327 } 1354 }
1328} 1355}
1329 1356
1330void 1357void
1331uvmpdpol_balancequeue(int swap_shortage) 1358uvmpdpol_balancequeue(int swap_shortage)
1332{ 1359{
1333 struct clockpro_state * const s = &clockpro; 1360 struct clockpro_state * const s = &clockpro;
1334 int todo = swap_shortage; 1361 int todo = swap_shortage;
1335 1362
1336 if (todo == 0) { 1363 if (todo == 0) {
1337 return; 1364 return;
1338 } 1365 }
1339 1366