pagedaemon: - Use marker pages to keep place in the queue when scanning, rather than relying on assumptions. - In uvmpdpol_balancequeue(), lock the object once instead of twice. - When draining pools, the situation is getting desperate, but try to avoid saturating the system with xcall, lock and interrupt activity by sleeping for 1 clock tick if being continually awoken and all pools have been cycled through at least once. - Pause & resume the freelist cache during pool draining. PR kern/54209: NetBSD 8 large memory performance extremely low PR kern/54210: NetBSD-8 processes presumably not exiting PR kern/54727: writing a large file causes unreasonable system behaviourdiff -r1.118 -r1.119 src/sys/uvm/uvm_pdaemon.c
(ad)
--- src/sys/uvm/uvm_pdaemon.c 2019/12/21 16:10:20 1.118
+++ src/sys/uvm/uvm_pdaemon.c 2019/12/30 18:08:37 1.119
@@ -1,14 +1,14 @@ | @@ -1,14 +1,14 @@ | |||
1 | /* $NetBSD: uvm_pdaemon.c,v 1.118 2019/12/21 16:10:20 ad Exp $ */ | 1 | /* $NetBSD: uvm_pdaemon.c,v 1.119 2019/12/30 18:08:37 ad Exp $ */ | |
2 | 2 | |||
3 | /* | 3 | /* | |
4 | * Copyright (c) 1997 Charles D. Cranor and Washington University. | 4 | * Copyright (c) 1997 Charles D. Cranor and Washington University. | |
5 | * Copyright (c) 1991, 1993, The Regents of the University of California. | 5 | * Copyright (c) 1991, 1993, The Regents of the University of California. | |
6 | * | 6 | * | |
7 | * All rights reserved. | 7 | * All rights reserved. | |
8 | * | 8 | * | |
9 | * This code is derived from software contributed to Berkeley by | 9 | * This code is derived from software contributed to Berkeley by | |
10 | * The Mach Operating System project at Carnegie-Mellon University. | 10 | * The Mach Operating System project at Carnegie-Mellon University. | |
11 | * | 11 | * | |
12 | * Redistribution and use in source and binary forms, with or without | 12 | * Redistribution and use in source and binary forms, with or without | |
13 | * modification, are permitted provided that the following conditions | 13 | * modification, are permitted provided that the following conditions | |
14 | * are met: | 14 | * are met: | |
@@ -56,43 +56,44 @@ | @@ -56,43 +56,44 @@ | |||
56 | * School of Computer Science | 56 | * School of Computer Science | |
57 | * Carnegie Mellon University | 57 | * Carnegie Mellon University | |
58 | * Pittsburgh PA 15213-3890 | 58 | * Pittsburgh PA 15213-3890 | |
59 | * | 59 | * | |
60 | * any improvements or extensions that they make and grant Carnegie the | 60 | * any improvements or extensions that they make and grant Carnegie the | |
61 | * rights to redistribute these changes. | 61 | * rights to redistribute these changes. | |
62 | */ | 62 | */ | |
63 | 63 | |||
64 | /* | 64 | /* | |
65 | * uvm_pdaemon.c: the page daemon | 65 | * uvm_pdaemon.c: the page daemon | |
66 | */ | 66 | */ | |
67 | 67 | |||
68 | #include <sys/cdefs.h> | 68 | #include <sys/cdefs.h> | |
69 | __KERNEL_RCSID(0, "$NetBSD: uvm_pdaemon.c,v 1.118 2019/12/21 16:10:20 ad Exp $"); | 69 | __KERNEL_RCSID(0, "$NetBSD: uvm_pdaemon.c,v 1.119 2019/12/30 18:08:37 ad Exp $"); | |
70 | 70 | |||
71 | #include "opt_uvmhist.h" | 71 | #include "opt_uvmhist.h" | |
72 | #include "opt_readahead.h" | 72 | #include "opt_readahead.h" | |
73 | 73 | |||
74 | #include <sys/param.h> | 74 | #include <sys/param.h> | |
75 | #include <sys/proc.h> | 75 | #include <sys/proc.h> | |
76 | #include <sys/systm.h> | 76 | #include <sys/systm.h> | |
77 | #include <sys/kernel.h> | 77 | #include <sys/kernel.h> | |
78 | #include <sys/pool.h> | 78 | #include <sys/pool.h> | |
79 | #include <sys/buf.h> | 79 | #include <sys/buf.h> | |
80 | #include <sys/module.h> | 80 | #include <sys/module.h> | |
81 | #include <sys/atomic.h> | 81 | #include <sys/atomic.h> | |
82 | #include <sys/kthread.h> | 82 | #include <sys/kthread.h> | |
83 | 83 | |||
84 | #include <uvm/uvm.h> | 84 | #include <uvm/uvm.h> | |
85 | #include <uvm/uvm_pdpolicy.h> | 85 | #include <uvm/uvm_pdpolicy.h> | |
86 | #include <uvm/uvm_pgflcache.h> | |||
86 | 87 | |||
87 | #ifdef UVMHIST | 88 | #ifdef UVMHIST | |
88 | UVMHIST_DEFINE(pdhist); | 89 | UVMHIST_DEFINE(pdhist); | |
89 | #endif | 90 | #endif | |
90 | 91 | |||
91 | /* | 92 | /* | |
92 | * UVMPD_NUMDIRTYREACTS is how many dirty pages the pagedaemon will reactivate | 93 | * UVMPD_NUMDIRTYREACTS is how many dirty pages the pagedaemon will reactivate | |
93 | * in a pass thru the inactive list when swap is full. the value should be | 94 | * in a pass thru the inactive list when swap is full. the value should be | |
94 | * "small"... if it's too large we'll cycle the active pages thru the inactive | 95 | * "small"... if it's too large we'll cycle the active pages thru the inactive | |
95 | * queue too quickly to for them to be referenced and avoid being freed. | 96 | * queue too quickly to for them to be referenced and avoid being freed. | |
96 | */ | 97 | */ | |
97 | 98 | |||
98 | #define UVMPD_NUMDIRTYREACTS 16 | 99 | #define UVMPD_NUMDIRTYREACTS 16 | |
@@ -588,94 +589,50 @@ static int | @@ -588,94 +589,50 @@ static int | |||
588 | swapcluster_nused(struct swapcluster *swc) | 589 | swapcluster_nused(struct swapcluster *swc) | |
589 | { | 590 | { | |
590 | 591 | |||
591 | return swc->swc_nused; | 592 | return swc->swc_nused; | |
592 | } | 593 | } | |
593 | 594 | |||
594 | /* | 595 | /* | |
595 | * uvmpd_dropswap: free any swap allocated to this page. | 596 | * uvmpd_dropswap: free any swap allocated to this page. | |
596 | * | 597 | * | |
597 | * => called with owner locked. | 598 | * => called with owner locked. | |
598 | * => return true if a page had an associated slot. | 599 | * => return true if a page had an associated slot. | |
599 | */ | 600 | */ | |
600 | 601 | |||
601 | static bool | 602 | bool | |
602 | uvmpd_dropswap(struct vm_page *pg) | 603 | uvmpd_dropswap(struct vm_page *pg) | |
603 | { | 604 | { | |
604 | bool result = false; | 605 | bool result = false; | |
605 | struct vm_anon *anon = pg->uanon; | 606 | struct vm_anon *anon = pg->uanon; | |
606 | 607 | |||
607 | if ((pg->flags & PG_ANON) && anon->an_swslot) { | 608 | if ((pg->flags & PG_ANON) && anon->an_swslot) { | |
608 | uvm_swap_free(anon->an_swslot, 1); | 609 | uvm_swap_free(anon->an_swslot, 1); | |
609 | anon->an_swslot = 0; | 610 | anon->an_swslot = 0; | |
610 | pg->flags &= ~PG_CLEAN; | 611 | pg->flags &= ~PG_CLEAN; | |
611 | result = true; | 612 | result = true; | |
612 | } else if (pg->flags & PG_AOBJ) { | 613 | } else if (pg->flags & PG_AOBJ) { | |
613 | int slot = uao_set_swslot(pg->uobject, | 614 | int slot = uao_set_swslot(pg->uobject, | |
614 | pg->offset >> PAGE_SHIFT, 0); | 615 | pg->offset >> PAGE_SHIFT, 0); | |
615 | if (slot) { | 616 | if (slot) { | |
616 | uvm_swap_free(slot, 1); | 617 | uvm_swap_free(slot, 1); | |
617 | pg->flags &= ~PG_CLEAN; | 618 | pg->flags &= ~PG_CLEAN; | |
618 | result = true; | 619 | result = true; | |
619 | } | 620 | } | |
620 | } | 621 | } | |
621 | 622 | |||
622 | return result; | 623 | return result; | |
623 | } | 624 | } | |
624 | 625 | |||
625 | /* | |||
626 | * uvmpd_trydropswap: try to free any swap allocated to this page. | |||
627 | * | |||
628 | * => return true if a slot is successfully freed. | |||
629 | * => page interlock must be held, and will be dropped. | |||
630 | */ | |||
631 | ||||
632 | bool | |||
633 | uvmpd_trydropswap(struct vm_page *pg) | |||
634 | { | |||
635 | kmutex_t *slock; | |||
636 | bool result; | |||
637 | ||||
638 | if ((pg->flags & PG_BUSY) != 0) { | |||
639 | mutex_exit(&pg->interlock); | |||
640 | return false; | |||
641 | } | |||
642 | ||||
643 | /* | |||
644 | * lock the page's owner. | |||
645 | * this will drop pg->interlock. | |||
646 | */ | |||
647 | ||||
648 | slock = uvmpd_trylockowner(pg); | |||
649 | if (slock == NULL) { | |||
650 | return false; | |||
651 | } | |||
652 | ||||
653 | /* | |||
654 | * skip this page if it's busy. | |||
655 | */ | |||
656 | ||||
657 | if ((pg->flags & PG_BUSY) != 0) { | |||
658 | mutex_exit(slock); | |||
659 | return false; | |||
660 | } | |||
661 | ||||
662 | result = uvmpd_dropswap(pg); | |||
663 | ||||
664 | mutex_exit(slock); | |||
665 | ||||
666 | return result; | |||
667 | } | |||
668 | ||||
669 | #endif /* defined(VMSWAP) */ | 626 | #endif /* defined(VMSWAP) */ | |
670 | 627 | |||
671 | /* | 628 | /* | |
672 | * uvmpd_scan_queue: scan an replace candidate list for pages | 629 | * uvmpd_scan_queue: scan an replace candidate list for pages | |
673 | * to clean or free. | 630 | * to clean or free. | |
674 | * | 631 | * | |
675 | * => we work on meeting our free target by converting inactive pages | 632 | * => we work on meeting our free target by converting inactive pages | |
676 | * into free pages. | 633 | * into free pages. | |
677 | * => we handle the building of swap-backed clusters | 634 | * => we handle the building of swap-backed clusters | |
678 | */ | 635 | */ | |
679 | 636 | |||
680 | static void | 637 | static void | |
681 | uvmpd_scan_queue(void) | 638 | uvmpd_scan_queue(void) | |
@@ -899,26 +856,28 @@ uvmpd_scan_queue(void) | @@ -899,26 +856,28 @@ uvmpd_scan_queue(void) | |||
899 | /* | 856 | /* | |
900 | * the pageout is in progress. bump counters and set up | 857 | * the pageout is in progress. bump counters and set up | |
901 | * for the next loop. | 858 | * for the next loop. | |
902 | */ | 859 | */ | |
903 | 860 | |||
904 | atomic_inc_uint(&uvmexp.pdpending); | 861 | atomic_inc_uint(&uvmexp.pdpending); | |
905 | 862 | |||
906 | #else /* defined(VMSWAP) */ | 863 | #else /* defined(VMSWAP) */ | |
907 | uvm_pageactivate(p); | 864 | uvm_pageactivate(p); | |
908 | mutex_exit(slock); | 865 | mutex_exit(slock); | |
909 | #endif /* defined(VMSWAP) */ | 866 | #endif /* defined(VMSWAP) */ | |
910 | } | 867 | } | |
911 | 868 | |||
869 | uvmpdpol_scanfini(); | |||
870 | ||||
912 | #if defined(VMSWAP) | 871 | #if defined(VMSWAP) | |
913 | swapcluster_flush(&swc, true); | 872 | swapcluster_flush(&swc, true); | |
914 | #endif /* defined(VMSWAP) */ | 873 | #endif /* defined(VMSWAP) */ | |
915 | } | 874 | } | |
916 | 875 | |||
917 | /* | 876 | /* | |
918 | * uvmpd_scan: scan the page queues and attempt to meet our targets. | 877 | * uvmpd_scan: scan the page queues and attempt to meet our targets. | |
919 | */ | 878 | */ | |
920 | 879 | |||
921 | static void | 880 | static void | |
922 | uvmpd_scan(void) | 881 | uvmpd_scan(void) | |
923 | { | 882 | { | |
924 | int swap_shortage, pages_freed, fpages; | 883 | int swap_shortage, pages_freed, fpages; | |
@@ -1021,51 +980,85 @@ uvm_estimatepageable(int *active, int *i | @@ -1021,51 +980,85 @@ uvm_estimatepageable(int *active, int *i | |||
1021 | uvmpdpol_estimatepageable(active, inactive); | 980 | uvmpdpol_estimatepageable(active, inactive); | |
1022 | } | 981 | } | |
1023 | 982 | |||
1024 | 983 | |||
1025 | /* | 984 | /* | |
1026 | * Use a separate thread for draining pools. | 985 | * Use a separate thread for draining pools. | |
1027 | * This work can't done from the main pagedaemon thread because | 986 | * This work can't done from the main pagedaemon thread because | |
1028 | * some pool allocators need to take vm_map locks. | 987 | * some pool allocators need to take vm_map locks. | |
1029 | */ | 988 | */ | |
1030 | 989 | |||
1031 | static void | 990 | static void | |
1032 | uvmpd_pool_drain_thread(void *arg) | 991 | uvmpd_pool_drain_thread(void *arg) | |
1033 | { | 992 | { | |
1034 | int bufcnt; | 993 | struct pool *firstpool, *curpool; | |
994 | int bufcnt, lastslept; | |||
995 | bool cycled; | |||
1035 | 996 | |||
997 | firstpool = NULL; | |||
998 | cycled = true; | |||
1036 | for (;;) { | 999 | for (;;) { | |
1000 | /* | |||
1001 | * sleep until awoken by the pagedaemon. | |||
1002 | */ | |||
1037 | mutex_enter(&uvmpd_lock); | 1003 | mutex_enter(&uvmpd_lock); | |
1038 | if (!uvmpd_pool_drain_run) { | 1004 | if (!uvmpd_pool_drain_run) { | |
1005 | lastslept = hardclock_ticks; | |||
1039 | cv_wait(&uvmpd_pool_drain_cv, &uvmpd_lock); | 1006 | cv_wait(&uvmpd_pool_drain_cv, &uvmpd_lock); | |
1007 | if (hardclock_ticks != lastslept) { | |||
1008 | cycled = false; | |||
1009 | firstpool = NULL; | |||
1010 | } | |||
1040 | } | 1011 | } | |
1041 | uvmpd_pool_drain_run = false; | 1012 | uvmpd_pool_drain_run = false; | |
1042 | mutex_exit(&uvmpd_lock); | 1013 | mutex_exit(&uvmpd_lock); | |
1043 | 1014 | |||
1044 | /* | 1015 | /* | |
1016 | * rate limit draining, otherwise in desperate circumstances | |||
1017 | * this can totally saturate the system with xcall activity. | |||
1018 | */ | |||
1019 | if (cycled) { | |||
1020 | kpause("uvmpdlmt", false, 1, NULL); | |||
1021 | cycled = false; | |||
1022 | firstpool = NULL; | |||
1023 | } | |||
1024 | ||||
1025 | /* | |||
1026 | * drain and temporarily disable the freelist cache. | |||
1027 | */ | |||
1028 | uvm_pgflcache_pause(); | |||
1029 | ||||
1030 | /* | |||
1045 | * kill unused metadata buffers. | 1031 | * kill unused metadata buffers. | |
1046 | */ | 1032 | */ | |
1047 | bufcnt = uvmexp.freetarg - uvm_free(); | 1033 | bufcnt = uvmexp.freetarg - uvm_free(); | |
1048 | if (bufcnt < 0) | 1034 | if (bufcnt < 0) | |
1049 | bufcnt = 0; | 1035 | bufcnt = 0; | |
1050 | 1036 | |||
1051 | mutex_enter(&bufcache_lock); | 1037 | mutex_enter(&bufcache_lock); | |
1052 | buf_drain(bufcnt << PAGE_SHIFT); | 1038 | buf_drain(bufcnt << PAGE_SHIFT); | |
1053 | mutex_exit(&bufcache_lock); | 1039 | mutex_exit(&bufcache_lock); | |
1054 | 1040 | |||
1055 | /* | 1041 | /* | |
1056 | * drain a pool. | 1042 | * drain a pool, and then re-enable the freelist cache. | |
1057 | */ | 1043 | */ | |
1058 | pool_drain(NULL); | 1044 | (void)pool_drain(&curpool); | |
1045 | KASSERT(curpool != NULL); | |||
1046 | if (firstpool == NULL) { | |||
1047 | firstpool = curpool; | |||
1048 | } else if (firstpool == curpool) { | |||
1049 | cycled = true; | |||
1050 | } | |||
1051 | uvm_pgflcache_resume(); | |||
1059 | } | 1052 | } | |
1060 | /*NOTREACHED*/ | 1053 | /*NOTREACHED*/ | |
1061 | } | 1054 | } | |
1062 | 1055 | |||
1063 | static void | 1056 | static void | |
1064 | uvmpd_pool_drain_wakeup(void) | 1057 | uvmpd_pool_drain_wakeup(void) | |
1065 | { | 1058 | { | |
1066 | 1059 | |||
1067 | mutex_enter(&uvmpd_lock); | 1060 | mutex_enter(&uvmpd_lock); | |
1068 | uvmpd_pool_drain_run = true; | 1061 | uvmpd_pool_drain_run = true; | |
1069 | cv_signal(&uvmpd_pool_drain_cv); | 1062 | cv_signal(&uvmpd_pool_drain_cv); | |
1070 | mutex_exit(&uvmpd_lock); | 1063 | mutex_exit(&uvmpd_lock); | |
1071 | } | 1064 | } |
--- src/sys/uvm/uvm_pdaemon.h 2011/02/02 15:25:27 1.17
+++ src/sys/uvm/uvm_pdaemon.h 2019/12/30 18:08:38 1.18
@@ -1,14 +1,14 @@ | @@ -1,14 +1,14 @@ | |||
1 | /* $NetBSD: uvm_pdaemon.h,v 1.17 2011/02/02 15:25:27 chuck Exp $ */ | 1 | /* $NetBSD: uvm_pdaemon.h,v 1.18 2019/12/30 18:08:38 ad Exp $ */ | |
2 | 2 | |||
3 | /* | 3 | /* | |
4 | * Copyright (c) 1997 Charles D. Cranor and Washington University. | 4 | * Copyright (c) 1997 Charles D. Cranor and Washington University. | |
5 | * Copyright (c) 1991, 1993, The Regents of the University of California. | 5 | * Copyright (c) 1991, 1993, The Regents of the University of California. | |
6 | * | 6 | * | |
7 | * All rights reserved. | 7 | * All rights reserved. | |
8 | * | 8 | * | |
9 | * This code is derived from software contributed to Berkeley by | 9 | * This code is derived from software contributed to Berkeley by | |
10 | * The Mach Operating System project at Carnegie-Mellon University. | 10 | * The Mach Operating System project at Carnegie-Mellon University. | |
11 | * | 11 | * | |
12 | * Redistribution and use in source and binary forms, with or without | 12 | * Redistribution and use in source and binary forms, with or without | |
13 | * modification, are permitted provided that the following conditions | 13 | * modification, are permitted provided that the following conditions | |
14 | * are met: | 14 | * are met: | |
@@ -69,21 +69,21 @@ | @@ -69,21 +69,21 @@ | |||
69 | */ | 69 | */ | |
70 | 70 | |||
71 | #ifdef _KERNEL | 71 | #ifdef _KERNEL | |
72 | 72 | |||
73 | /* | 73 | /* | |
74 | * prototypes | 74 | * prototypes | |
75 | */ | 75 | */ | |
76 | 76 | |||
77 | void uvm_wait(const char *); | 77 | void uvm_wait(const char *); | |
78 | bool uvm_reclaimable(void); | 78 | bool uvm_reclaimable(void); | |
79 | 79 | |||
80 | kmutex_t *uvmpd_trylockowner(struct vm_page *); | 80 | kmutex_t *uvmpd_trylockowner(struct vm_page *); | |
81 | #ifdef VMSWAP | 81 | #ifdef VMSWAP | |
82 | bool uvmpd_trydropswap(struct vm_page *); | 82 | bool uvmpd_dropswap(struct vm_page *); | |
83 | #else | 83 | #else | |
84 | #define uvmpd_trydropswap(_a_) (/*CONSTCOND*/false) | 84 | #define uvmpd_dropswap(_a_) (/*CONSTCOND*/false) | |
85 | #endif | 85 | #endif | |
86 | 86 | |||
87 | #endif /* _KERNEL */ | 87 | #endif /* _KERNEL */ | |
88 | 88 | |||
89 | #endif /* _UVM_UVM_PDAEMON_H_ */ | 89 | #endif /* _UVM_UVM_PDAEMON_H_ */ |
--- src/sys/uvm/uvm_pdpolicy.h 2019/12/13 20:10:22 1.4
+++ src/sys/uvm/uvm_pdpolicy.h 2019/12/30 18:08:38 1.5
@@ -1,14 +1,14 @@ | @@ -1,14 +1,14 @@ | |||
1 | /* $NetBSD: uvm_pdpolicy.h,v 1.4 2019/12/13 20:10:22 ad Exp $ */ | 1 | /* $NetBSD: uvm_pdpolicy.h,v 1.5 2019/12/30 18:08:38 ad Exp $ */ | |
2 | 2 | |||
3 | /*- | 3 | /*- | |
4 | * Copyright (c)2005, 2006 YAMAMOTO Takashi, | 4 | * Copyright (c)2005, 2006 YAMAMOTO Takashi, | |
5 | * All rights reserved. | 5 | * All rights reserved. | |
6 | * | 6 | * | |
7 | * Redistribution and use in source and binary forms, with or without | 7 | * Redistribution and use in source and binary forms, with or without | |
8 | * modification, are permitted provided that the following conditions | 8 | * modification, are permitted provided that the following conditions | |
9 | * are met: | 9 | * are met: | |
10 | * 1. Redistributions of source code must retain the above copyright | 10 | * 1. Redistributions of source code must retain the above copyright | |
11 | * notice, this list of conditions and the following disclaimer. | 11 | * notice, this list of conditions and the following disclaimer. | |
12 | * 2. Redistributions in binary form must reproduce the above copyright | 12 | * 2. Redistributions in binary form must reproduce the above copyright | |
13 | * notice, this list of conditions and the following disclaimer in the | 13 | * notice, this list of conditions and the following disclaimer in the | |
14 | * documentation and/or other materials provided with the distribution. | 14 | * documentation and/or other materials provided with the distribution. | |
@@ -41,19 +41,20 @@ void uvmpdpol_init(void); | @@ -41,19 +41,20 @@ void uvmpdpol_init(void); | |||
41 | void uvmpdpol_reinit(void); | 41 | void uvmpdpol_reinit(void); | |
42 | void uvmpdpol_estimatepageable(int *, int *); | 42 | void uvmpdpol_estimatepageable(int *, int *); | |
43 | bool uvmpdpol_needsscan_p(void); | 43 | bool uvmpdpol_needsscan_p(void); | |
44 | 44 | |||
45 | void uvmpdpol_pageactivate(struct vm_page *); | 45 | void uvmpdpol_pageactivate(struct vm_page *); | |
46 | void uvmpdpol_pagedeactivate(struct vm_page *); | 46 | void uvmpdpol_pagedeactivate(struct vm_page *); | |
47 | void uvmpdpol_pagedequeue(struct vm_page *); | 47 | void uvmpdpol_pagedequeue(struct vm_page *); | |
48 | void uvmpdpol_pageenqueue(struct vm_page *); | 48 | void uvmpdpol_pageenqueue(struct vm_page *); | |
49 | bool uvmpdpol_pageisqueued_p(struct vm_page *); | 49 | bool uvmpdpol_pageisqueued_p(struct vm_page *); | |
50 | void uvmpdpol_anfree(struct vm_anon *); | 50 | void uvmpdpol_anfree(struct vm_anon *); | |
51 | 51 | |||
52 | void uvmpdpol_tune(void); | 52 | void uvmpdpol_tune(void); | |
53 | void uvmpdpol_scaninit(void); | 53 | void uvmpdpol_scaninit(void); | |
54 | void uvmpdpol_scanfini(void); | |||
54 | struct vm_page *uvmpdpol_selectvictim(kmutex_t **lock); | 55 | struct vm_page *uvmpdpol_selectvictim(kmutex_t **lock); | |
55 | void uvmpdpol_balancequeue(int); | 56 | void uvmpdpol_balancequeue(int); | |
56 | 57 | |||
57 | void uvmpdpol_sysctlsetup(void); | 58 | void uvmpdpol_sysctlsetup(void); | |
58 | 59 | |||
59 | #endif /* !_UVM_PDPOLICY_H_ */ | 60 | #endif /* !_UVM_PDPOLICY_H_ */ |
--- src/sys/uvm/uvm_pdpolicy_clock.c 2019/12/27 13:13:17 1.23
+++ src/sys/uvm/uvm_pdpolicy_clock.c 2019/12/30 18:08:38 1.24
@@ -1,14 +1,14 @@ | @@ -1,14 +1,14 @@ | |||
1 | /* $NetBSD: uvm_pdpolicy_clock.c,v 1.23 2019/12/27 13:13:17 ad Exp $ */ | 1 | /* $NetBSD: uvm_pdpolicy_clock.c,v 1.24 2019/12/30 18:08:38 ad Exp $ */ | |
2 | /* NetBSD: uvm_pdaemon.c,v 1.72 2006/01/05 10:47:33 yamt Exp $ */ | 2 | /* NetBSD: uvm_pdaemon.c,v 1.72 2006/01/05 10:47:33 yamt Exp $ */ | |
3 | 3 | |||
4 | /* | 4 | /* | |
5 | * Copyright (c) 1997 Charles D. Cranor and Washington University. | 5 | * Copyright (c) 1997 Charles D. Cranor and Washington University. | |
6 | * Copyright (c) 1991, 1993, The Regents of the University of California. | 6 | * Copyright (c) 1991, 1993, The Regents of the University of California. | |
7 | * | 7 | * | |
8 | * All rights reserved. | 8 | * All rights reserved. | |
9 | * | 9 | * | |
10 | * This code is derived from software contributed to Berkeley by | 10 | * This code is derived from software contributed to Berkeley by | |
11 | * The Mach Operating System project at Carnegie-Mellon University. | 11 | * The Mach Operating System project at Carnegie-Mellon University. | |
12 | * | 12 | * | |
13 | * Redistribution and use in source and binary forms, with or without | 13 | * Redistribution and use in source and binary forms, with or without | |
14 | * modification, are permitted provided that the following conditions | 14 | * modification, are permitted provided that the following conditions | |
@@ -59,27 +59,27 @@ | @@ -59,27 +59,27 @@ | |||
59 | * Pittsburgh PA 15213-3890 | 59 | * Pittsburgh PA 15213-3890 | |
60 | * | 60 | * | |
61 | * any improvements or extensions that they make and grant Carnegie the | 61 | * any improvements or extensions that they make and grant Carnegie the | |
62 | * rights to redistribute these changes. | 62 | * rights to redistribute these changes. | |
63 | */ | 63 | */ | |
64 | 64 | |||
65 | #if defined(PDSIM) | 65 | #if defined(PDSIM) | |
66 | 66 | |||
67 | #include "pdsim.h" | 67 | #include "pdsim.h" | |
68 | 68 | |||
69 | #else /* defined(PDSIM) */ | 69 | #else /* defined(PDSIM) */ | |
70 | 70 | |||
71 | #include <sys/cdefs.h> | 71 | #include <sys/cdefs.h> | |
72 | __KERNEL_RCSID(0, "$NetBSD: uvm_pdpolicy_clock.c,v 1.23 2019/12/27 13:13:17 ad Exp $"); | 72 | __KERNEL_RCSID(0, "$NetBSD: uvm_pdpolicy_clock.c,v 1.24 2019/12/30 18:08:38 ad Exp $"); | |
73 | 73 | |||
74 | #include <sys/param.h> | 74 | #include <sys/param.h> | |
75 | #include <sys/proc.h> | 75 | #include <sys/proc.h> | |
76 | #include <sys/systm.h> | 76 | #include <sys/systm.h> | |
77 | #include <sys/kernel.h> | 77 | #include <sys/kernel.h> | |
78 | 78 | |||
79 | #include <uvm/uvm.h> | 79 | #include <uvm/uvm.h> | |
80 | #include <uvm/uvm_pdpolicy.h> | 80 | #include <uvm/uvm_pdpolicy.h> | |
81 | #include <uvm/uvm_pdpolicy_impl.h> | 81 | #include <uvm/uvm_pdpolicy_impl.h> | |
82 | #include <uvm/uvm_stat.h> | 82 | #include <uvm/uvm_stat.h> | |
83 | 83 | |||
84 | #endif /* defined(PDSIM) */ | 84 | #endif /* defined(PDSIM) */ | |
85 | 85 | |||
@@ -100,29 +100,28 @@ struct uvmpdpol_globalstate { | @@ -100,29 +100,28 @@ struct uvmpdpol_globalstate { | |||
100 | int s_active; | 100 | int s_active; | |
101 | int s_inactive; | 101 | int s_inactive; | |
102 | int s_inactarg; | 102 | int s_inactarg; | |
103 | struct uvm_pctparam s_anonmin; | 103 | struct uvm_pctparam s_anonmin; | |
104 | struct uvm_pctparam s_filemin; | 104 | struct uvm_pctparam s_filemin; | |
105 | struct uvm_pctparam s_execmin; | 105 | struct uvm_pctparam s_execmin; | |
106 | struct uvm_pctparam s_anonmax; | 106 | struct uvm_pctparam s_anonmax; | |
107 | struct uvm_pctparam s_filemax; | 107 | struct uvm_pctparam s_filemax; | |
108 | struct uvm_pctparam s_execmax; | 108 | struct uvm_pctparam s_execmax; | |
109 | struct uvm_pctparam s_inactivepct; | 109 | struct uvm_pctparam s_inactivepct; | |
110 | }; | 110 | }; | |
111 | 111 | |||
112 | struct uvmpdpol_scanstate { | 112 | struct uvmpdpol_scanstate { | |
113 | bool ss_first; | |||
114 | bool ss_anonreact, ss_filereact, ss_execreact; | 113 | bool ss_anonreact, ss_filereact, ss_execreact; | |
115 | struct vm_page *ss_nextpg; | 114 | struct vm_page ss_marker; | |
116 | }; | 115 | }; | |
117 | 116 | |||
118 | static void uvmpdpol_pageactivate_locked(struct vm_page *); | 117 | static void uvmpdpol_pageactivate_locked(struct vm_page *); | |
119 | static void uvmpdpol_pagedeactivate_locked(struct vm_page *); | 118 | static void uvmpdpol_pagedeactivate_locked(struct vm_page *); | |
120 | static void uvmpdpol_pagedequeue_locked(struct vm_page *); | 119 | static void uvmpdpol_pagedequeue_locked(struct vm_page *); | |
121 | 120 | |||
122 | static struct uvmpdpol_globalstate pdpol_state __cacheline_aligned; | 121 | static struct uvmpdpol_globalstate pdpol_state __cacheline_aligned; | |
123 | static struct uvmpdpol_scanstate pdpol_scanstate; | 122 | static struct uvmpdpol_scanstate pdpol_scanstate; | |
124 | 123 | |||
125 | PDPOL_EVCNT_DEFINE(reactexec) | 124 | PDPOL_EVCNT_DEFINE(reactexec) | |
126 | PDPOL_EVCNT_DEFINE(reactfile) | 125 | PDPOL_EVCNT_DEFINE(reactfile) | |
127 | PDPOL_EVCNT_DEFINE(reactanon) | 126 | PDPOL_EVCNT_DEFINE(reactanon) | |
128 | 127 | |||
@@ -167,74 +166,86 @@ uvmpdpol_scaninit(void) | @@ -167,74 +166,86 @@ uvmpdpol_scaninit(void) | |||
167 | execunder = execpg <= UVM_PCTPARAM_APPLY(&s->s_execmin, t); | 166 | execunder = execpg <= UVM_PCTPARAM_APPLY(&s->s_execmin, t); | |
168 | anonover = anonpg > UVM_PCTPARAM_APPLY(&s->s_anonmax, t); | 167 | anonover = anonpg > UVM_PCTPARAM_APPLY(&s->s_anonmax, t); | |
169 | fileover = filepg > UVM_PCTPARAM_APPLY(&s->s_filemax, t); | 168 | fileover = filepg > UVM_PCTPARAM_APPLY(&s->s_filemax, t); | |
170 | execover = execpg > UVM_PCTPARAM_APPLY(&s->s_execmax, t); | 169 | execover = execpg > UVM_PCTPARAM_APPLY(&s->s_execmax, t); | |
171 | anonreact = anonunder || (!anonover && (fileover || execover)); | 170 | anonreact = anonunder || (!anonover && (fileover || execover)); | |
172 | filereact = fileunder || (!fileover && (anonover || execover)); | 171 | filereact = fileunder || (!fileover && (anonover || execover)); | |
173 | execreact = execunder || (!execover && (anonover || fileover)); | 172 | execreact = execunder || (!execover && (anonover || fileover)); | |
174 | if (filereact && execreact && (anonreact || uvm_swapisfull())) { | 173 | if (filereact && execreact && (anonreact || uvm_swapisfull())) { | |
175 | anonreact = filereact = execreact = false; | 174 | anonreact = filereact = execreact = false; | |
176 | } | 175 | } | |
177 | ss->ss_anonreact = anonreact; | 176 | ss->ss_anonreact = anonreact; | |
178 | ss->ss_filereact = filereact; | 177 | ss->ss_filereact = filereact; | |
179 | ss->ss_execreact = execreact; | 178 | ss->ss_execreact = execreact; | |
179 | memset(&ss->ss_marker, 0, sizeof(ss->ss_marker)); | |||
180 | ss->ss_marker.flags = PG_MARKER; | |||
181 | TAILQ_INSERT_HEAD(&pdpol_state.s_inactiveq, &ss->ss_marker, pdqueue); | |||
182 | mutex_exit(&s->lock); | |||
183 | } | |||
184 | ||||
185 | void | |||
186 | uvmpdpol_scanfini(void) | |||
187 | { | |||
188 | struct uvmpdpol_globalstate *s = &pdpol_state; | |||
189 | struct uvmpdpol_scanstate *ss = &pdpol_scanstate; | |||
180 | 190 | |||
181 | ss->ss_first = true; | 191 | mutex_enter(&s->lock); | |
192 | TAILQ_REMOVE(&pdpol_state.s_inactiveq, &ss->ss_marker, pdqueue); | |||
182 | mutex_exit(&s->lock); | 193 | mutex_exit(&s->lock); | |
183 | } | 194 | } | |
184 | 195 | |||
185 | struct vm_page * | 196 | struct vm_page * | |
186 | uvmpdpol_selectvictim(kmutex_t **plock) | 197 | uvmpdpol_selectvictim(kmutex_t **plock) | |
187 | { | 198 | { | |
188 | struct uvmpdpol_globalstate *s = &pdpol_state; | 199 | struct uvmpdpol_globalstate *s = &pdpol_state; | |
189 | struct uvmpdpol_scanstate *ss = &pdpol_scanstate; | 200 | struct uvmpdpol_scanstate *ss = &pdpol_scanstate; | |
190 | struct vm_page *pg; | 201 | struct vm_page *pg; | |
191 | kmutex_t *lock; | 202 | kmutex_t *lock; | |
192 | 203 | |||
193 | mutex_enter(&s->lock); | 204 | mutex_enter(&s->lock); | |
194 | while (/* CONSTCOND */ 1) { | 205 | while (/* CONSTCOND */ 1) { | |
195 | struct vm_anon *anon; | 206 | struct vm_anon *anon; | |
196 | struct uvm_object *uobj; | 207 | struct uvm_object *uobj; | |
197 | 208 | |||
198 | if (ss->ss_first) { | 209 | pg = TAILQ_NEXT(&ss->ss_marker, pdqueue); | |
199 | pg = TAILQ_FIRST(&pdpol_state.s_inactiveq); | |||
200 | ss->ss_first = false; | |||
201 | } else { | |||
202 | pg = ss->ss_nextpg; | |||
203 | if (pg != NULL && (pg->pqflags & PQ_INACTIVE) == 0) { | |||
204 | pg = TAILQ_FIRST(&pdpol_state.s_inactiveq); | |||
205 | } | |||
206 | } | |||
207 | if (pg == NULL) { | 210 | if (pg == NULL) { | |
208 | break; | 211 | break; | |
209 | } | 212 | } | |
210 | ss->ss_nextpg = TAILQ_NEXT(pg, pdqueue); | 213 | KASSERT((pg->flags & PG_MARKER) == 0); | |
211 | uvmexp.pdscans++; | 214 | uvmexp.pdscans++; | |
212 | 215 | |||
213 | /* | 216 | /* | |
214 | * acquire interlock to stablize page identity. | 217 | * acquire interlock to stablize page identity. | |
215 | * if we have caught the page in a state of flux | 218 | * if we have caught the page in a state of flux | |
216 | * and it should be dequeued, do it now and then | 219 | * and it should be dequeued, do it now and then | |
217 | * move on to the next. | 220 | * move on to the next. | |
218 | */ | 221 | */ | |
219 | mutex_enter(&pg->interlock); | 222 | mutex_enter(&pg->interlock); | |
220 | if ((pg->uobject == NULL && pg->uanon == NULL) || | 223 | if ((pg->uobject == NULL && pg->uanon == NULL) || | |
221 | pg->wire_count > 0) { | 224 | pg->wire_count > 0) { | |
222 | mutex_exit(&pg->interlock); | 225 | mutex_exit(&pg->interlock); | |
223 | uvmpdpol_pagedequeue_locked(pg); | 226 | uvmpdpol_pagedequeue_locked(pg); | |
224 | continue; | 227 | continue; | |
225 | } | 228 | } | |
226 | 229 | |||
227 | /* | 230 | /* | |
231 | * now prepare to move on to the next page. | |||
232 | */ | |||
233 | TAILQ_REMOVE(&pdpol_state.s_inactiveq, &ss->ss_marker, | |||
234 | pdqueue); | |||
235 | TAILQ_INSERT_AFTER(&pdpol_state.s_inactiveq, pg, | |||
236 | &ss->ss_marker, pdqueue); | |||
237 | ||||
238 | /* | |||
228 | * enforce the minimum thresholds on different | 239 | * enforce the minimum thresholds on different | |
229 | * types of memory usage. if reusing the current | 240 | * types of memory usage. if reusing the current | |
230 | * page would reduce that type of usage below its | 241 | * page would reduce that type of usage below its | |
231 | * minimum, reactivate the page instead and move | 242 | * minimum, reactivate the page instead and move | |
232 | * on to the next page. | 243 | * on to the next page. | |
233 | */ | 244 | */ | |
234 | anon = pg->uanon; | 245 | anon = pg->uanon; | |
235 | uobj = pg->uobject; | 246 | uobj = pg->uobject; | |
236 | if (uobj && UVM_OBJ_IS_VTEXT(uobj) && ss->ss_execreact) { | 247 | if (uobj && UVM_OBJ_IS_VTEXT(uobj) && ss->ss_execreact) { | |
237 | mutex_exit(&pg->interlock); | 248 | mutex_exit(&pg->interlock); | |
238 | uvmpdpol_pageactivate_locked(pg); | 249 | uvmpdpol_pageactivate_locked(pg); | |
239 | PDPOL_EVCNT_INCR(reactexec); | 250 | PDPOL_EVCNT_INCR(reactexec); | |
240 | continue; | 251 | continue; | |
@@ -290,87 +301,108 @@ uvmpdpol_selectvictim(kmutex_t **plock) | @@ -290,87 +301,108 @@ uvmpdpol_selectvictim(kmutex_t **plock) | |||
290 | /* we have a potential victim. */ | 301 | /* we have a potential victim. */ | |
291 | *plock = lock; | 302 | *plock = lock; | |
292 | break; | 303 | break; | |
293 | } | 304 | } | |
294 | mutex_exit(&s->lock); | 305 | mutex_exit(&s->lock); | |
295 | return pg; | 306 | return pg; | |
296 | } | 307 | } | |
297 | 308 | |||
298 | void | 309 | void | |
299 | uvmpdpol_balancequeue(int swap_shortage) | 310 | uvmpdpol_balancequeue(int swap_shortage) | |
300 | { | 311 | { | |
301 | struct uvmpdpol_globalstate *s = &pdpol_state; | 312 | struct uvmpdpol_globalstate *s = &pdpol_state; | |
302 | int inactive_shortage; | 313 | int inactive_shortage; | |
303 | struct vm_page *p, *nextpg; | 314 | struct vm_page *p, marker; | |
304 | kmutex_t *lock; | 315 | kmutex_t *lock; | |
305 | 316 | |||
306 | /* | 317 | /* | |
307 | * we have done the scan to get free pages. now we work on meeting | 318 | * we have done the scan to get free pages. now we work on meeting | |
308 | * our inactive target. | 319 | * our inactive target. | |
309 | */ | 320 | */ | |
310 | 321 | |||
311 | mutex_enter(&s->lock); | 322 | memset(&marker, 0, sizeof(marker)); | |
312 | inactive_shortage = pdpol_state.s_inactarg - pdpol_state.s_inactive; | 323 | marker.flags = PG_MARKER; | |
313 | for (p = TAILQ_FIRST(&pdpol_state.s_activeq); | |||
314 | p != NULL && (inactive_shortage > 0 || swap_shortage > 0); | |||
315 | p = nextpg) { | |||
316 | nextpg = TAILQ_NEXT(p, pdqueue); | |||
317 | 324 | |||
318 | /* | 325 | mutex_enter(&s->lock); | |
319 | * if there's a shortage of swap slots, try to free it. | 326 | TAILQ_INSERT_HEAD(&pdpol_state.s_activeq, &marker, pdqueue); | |
320 | */ | 327 | for (;;) { | |
321 | 328 | inactive_shortage = | ||
322 | if (swap_shortage > 0 && (p->flags & PG_SWAPBACKED) != 0) { | 329 | pdpol_state.s_inactarg - pdpol_state.s_inactive; | |
323 | mutex_enter(&p->interlock); | 330 | if (inactive_shortage <= 0 && swap_shortage <= 0) { | |
324 | mutex_exit(&s->lock); | 331 | break; | |
325 | if (uvmpd_trydropswap(p)) { | |||
326 | swap_shortage--; | |||
327 | } | |||
328 | /* p->interlock now released */ | |||
329 | mutex_enter(&s->lock); | |||
330 | } | 332 | } | |
331 | 333 | p = TAILQ_NEXT(&marker, pdqueue); | ||
332 | /* | 334 | if (p == NULL) { | |
333 | * if there's a shortage of inactive pages, deactivate. | 335 | break; | |
334 | */ | |||
335 | ||||
336 | if (inactive_shortage <= 0) { | |||
337 | continue; | |||
338 | } | 336 | } | |
337 | KASSERT((p->flags & PG_MARKER) == 0); | |||
339 | 338 | |||
340 | /* | 339 | /* | |
341 | * acquire interlock to stablize page identity. | 340 | * acquire interlock to stablize page identity. | |
342 | * if we have caught the page in a state of flux | 341 | * if we have caught the page in a state of flux | |
343 | * and it should be dequeued, do it now and then | 342 | * and it should be dequeued, do it now and then | |
344 | * move on to the next. | 343 | * move on to the next. | |
345 | */ | 344 | */ | |
346 | mutex_enter(&p->interlock); | 345 | mutex_enter(&p->interlock); | |
347 | if ((p->uobject == NULL && p->uanon == NULL) || | 346 | if ((p->uobject == NULL && p->uanon == NULL) || | |
348 | p->wire_count > 0) { | 347 | p->wire_count > 0) { | |
349 | mutex_exit(&p->interlock); | 348 | mutex_exit(&p->interlock); | |
350 | uvmpdpol_pagedequeue_locked(p); | 349 | uvmpdpol_pagedequeue_locked(p); | |
351 | continue; | 350 | continue; | |
352 | } | 351 | } | |
353 | mutex_exit(&s->lock); | 352 | ||
354 | lock = uvmpd_trylockowner(p); | 353 | /* | |
355 | /* p->interlock now released */ | 354 | * now prepare to move on to the next page. | |
356 | mutex_enter(&s->lock); | 355 | */ | |
357 | if (lock != NULL) { | 356 | ||
357 | TAILQ_REMOVE(&pdpol_state.s_activeq, &marker, pdqueue); | |||
358 | TAILQ_INSERT_AFTER(&pdpol_state.s_activeq, p, &marker, | |||
359 | pdqueue); | |||
360 | ||||
361 | /* | |||
362 | * try to lock the object that owns the page. see comments | |||
363 | * in uvmpdol_selectvictim(). | |||
364 | */ | |||
365 | mutex_exit(&s->lock); | |||
366 | lock = uvmpd_trylockowner(p); | |||
367 | /* p->interlock now released */ | |||
368 | mutex_enter(&s->lock); | |||
369 | if (lock == NULL) { | |||
370 | /* didn't get it - try the next page. */ | |||
371 | continue; | |||
372 | } | |||
373 | ||||
374 | /* | |||
375 | * if there's a shortage of swap slots, try to free it. | |||
376 | */ | |||
377 | if (swap_shortage > 0 && (p->flags & PG_SWAPBACKED) != 0 && | |||
378 | (p->flags & PG_BUSY) == 0) { | |||
379 | if (uvmpd_dropswap(p)) { | |||
380 | swap_shortage--; | |||
381 | } | |||
382 | } | |||
383 | ||||
384 | /* | |||
385 | * if there's a shortage of inactive pages, deactivate. | |||
386 | */ | |||
387 | ||||
388 | if (inactive_shortage > 0) { | |||
358 | uvmpdpol_pagedeactivate_locked(p); | 389 | uvmpdpol_pagedeactivate_locked(p); | |
359 | uvmexp.pddeact++; | 390 | uvmexp.pddeact++; | |
360 | inactive_shortage--; | 391 | inactive_shortage--; | |
361 | mutex_exit(lock); | |||
362 | } | 392 | } | |
393 | mutex_exit(lock); | |||
363 | } | 394 | } | |
395 | TAILQ_REMOVE(&pdpol_state.s_activeq, &marker, pdqueue); | |||
364 | mutex_exit(&s->lock); | 396 | mutex_exit(&s->lock); | |
365 | } | 397 | } | |
366 | 398 | |||
367 | static void | 399 | static void | |
368 | uvmpdpol_pagedeactivate_locked(struct vm_page *pg) | 400 | uvmpdpol_pagedeactivate_locked(struct vm_page *pg) | |
369 | { | 401 | { | |
370 | 402 | |||
371 | KASSERT(uvm_page_locked_p(pg)); | 403 | KASSERT(uvm_page_locked_p(pg)); | |
372 | 404 | |||
373 | if (pg->pqflags & PQ_ACTIVE) { | 405 | if (pg->pqflags & PQ_ACTIVE) { | |
374 | TAILQ_REMOVE(&pdpol_state.s_activeq, pg, pdqueue); | 406 | TAILQ_REMOVE(&pdpol_state.s_activeq, pg, pdqueue); | |
375 | pg->pqflags &= ~(PQ_ACTIVE | PQ_TIME); | 407 | pg->pqflags &= ~(PQ_ACTIVE | PQ_TIME); | |
376 | KASSERT(pdpol_state.s_active > 0); | 408 | KASSERT(pdpol_state.s_active > 0); |
--- src/sys/uvm/uvm_pdpolicy_clockpro.c 2019/12/27 13:13:17 1.19
+++ src/sys/uvm/uvm_pdpolicy_clockpro.c 2019/12/30 18:08:38 1.20
@@ -1,14 +1,14 @@ | @@ -1,14 +1,14 @@ | |||
1 | /* $NetBSD: uvm_pdpolicy_clockpro.c,v 1.19 2019/12/27 13:13:17 ad Exp $ */ | 1 | /* $NetBSD: uvm_pdpolicy_clockpro.c,v 1.20 2019/12/30 18:08:38 ad Exp $ */ | |
2 | 2 | |||
3 | /*- | 3 | /*- | |
4 | * Copyright (c)2005, 2006 YAMAMOTO Takashi, | 4 | * Copyright (c)2005, 2006 YAMAMOTO Takashi, | |
5 | * All rights reserved. | 5 | * All rights reserved. | |
6 | * | 6 | * | |
7 | * Redistribution and use in source and binary forms, with or without | 7 | * Redistribution and use in source and binary forms, with or without | |
8 | * modification, are permitted provided that the following conditions | 8 | * modification, are permitted provided that the following conditions | |
9 | * are met: | 9 | * are met: | |
10 | * 1. Redistributions of source code must retain the above copyright | 10 | * 1. Redistributions of source code must retain the above copyright | |
11 | * notice, this list of conditions and the following disclaimer. | 11 | * notice, this list of conditions and the following disclaimer. | |
12 | * 2. Redistributions in binary form must reproduce the above copyright | 12 | * 2. Redistributions in binary form must reproduce the above copyright | |
13 | * notice, this list of conditions and the following disclaimer in the | 13 | * notice, this list of conditions and the following disclaimer in the | |
14 | * documentation and/or other materials provided with the distribution. | 14 | * documentation and/or other materials provided with the distribution. | |
@@ -33,27 +33,27 @@ | @@ -33,27 +33,27 @@ | |||
33 | * approximation of the list of non-resident pages using hash: | 33 | * approximation of the list of non-resident pages using hash: | |
34 | * http://linux-mm.org/ClockProApproximation | 34 | * http://linux-mm.org/ClockProApproximation | |
35 | */ | 35 | */ | |
36 | 36 | |||
37 | /* #define CLOCKPRO_DEBUG */ | 37 | /* #define CLOCKPRO_DEBUG */ | |
38 | 38 | |||
39 | #if defined(PDSIM) | 39 | #if defined(PDSIM) | |
40 | 40 | |||
41 | #include "pdsim.h" | 41 | #include "pdsim.h" | |
42 | 42 | |||
43 | #else /* defined(PDSIM) */ | 43 | #else /* defined(PDSIM) */ | |
44 | 44 | |||
45 | #include <sys/cdefs.h> | 45 | #include <sys/cdefs.h> | |
46 | __KERNEL_RCSID(0, "$NetBSD: uvm_pdpolicy_clockpro.c,v 1.19 2019/12/27 13:13:17 ad Exp $"); | 46 | __KERNEL_RCSID(0, "$NetBSD: uvm_pdpolicy_clockpro.c,v 1.20 2019/12/30 18:08:38 ad Exp $"); | |
47 | 47 | |||
48 | #include "opt_ddb.h" | 48 | #include "opt_ddb.h" | |
49 | 49 | |||
50 | #include <sys/param.h> | 50 | #include <sys/param.h> | |
51 | #include <sys/proc.h> | 51 | #include <sys/proc.h> | |
52 | #include <sys/systm.h> | 52 | #include <sys/systm.h> | |
53 | #include <sys/kernel.h> | 53 | #include <sys/kernel.h> | |
54 | #include <sys/hash.h> | 54 | #include <sys/hash.h> | |
55 | 55 | |||
56 | #include <uvm/uvm.h> | 56 | #include <uvm/uvm.h> | |
57 | #include <uvm/uvm_pdaemon.h> /* for uvmpd_trylockowner */ | 57 | #include <uvm/uvm_pdaemon.h> /* for uvmpd_trylockowner */ | |
58 | #include <uvm/uvm_pdpolicy.h> | 58 | #include <uvm/uvm_pdpolicy.h> | |
59 | #include <uvm/uvm_pdpolicy_impl.h> | 59 | #include <uvm/uvm_pdpolicy_impl.h> | |
@@ -1248,26 +1248,32 @@ uvmpdpol_pageisqueued_p(struct vm_page * | @@ -1248,26 +1248,32 @@ uvmpdpol_pageisqueued_p(struct vm_page * | |||
1248 | } | 1248 | } | |
1249 | 1249 | |||
1250 | void | 1250 | void | |
1251 | uvmpdpol_scaninit(void) | 1251 | uvmpdpol_scaninit(void) | |
1252 | { | 1252 | { | |
1253 | struct clockpro_state * const s = &clockpro; | 1253 | struct clockpro_state * const s = &clockpro; | |
1254 | struct clockpro_scanstate * const ss = &scanstate; | 1254 | struct clockpro_scanstate * const ss = &scanstate; | |
1255 | 1255 | |||
1256 | mutex_enter(&s->lock); | 1256 | mutex_enter(&s->lock); | |
1257 | ss->ss_nscanned = 0; | 1257 | ss->ss_nscanned = 0; | |
1258 | mutex_exit(&s->lock); | 1258 | mutex_exit(&s->lock); | |
1259 | } | 1259 | } | |
1260 | 1260 | |||
1261 | void | |||
1262 | uvmpdpol_scanfini(void) | |||
1263 | { | |||
1264 | ||||
1265 | } | |||
1266 | ||||
1261 | struct vm_page * | 1267 | struct vm_page * | |
1262 | uvmpdpol_selectvictim(kmutex_t **plock) | 1268 | uvmpdpol_selectvictim(kmutex_t **plock) | |
1263 | { | 1269 | { | |
1264 | struct clockpro_state * const s = &clockpro; | 1270 | struct clockpro_state * const s = &clockpro; | |
1265 | struct clockpro_scanstate * const ss = &scanstate; | 1271 | struct clockpro_scanstate * const ss = &scanstate; | |
1266 | struct vm_page *pg; | 1272 | struct vm_page *pg; | |
1267 | kmutex_t *lock = NULL; | 1273 | kmutex_t *lock = NULL; | |
1268 | 1274 | |||
1269 | do { | 1275 | do { | |
1270 | mutex_enter(&s->lock); | 1276 | mutex_enter(&s->lock); | |
1271 | if (ss->ss_nscanned > s->s_npages) { | 1277 | if (ss->ss_nscanned > s->s_npages) { | |
1272 | DPRINTF("scan too much\n"); | 1278 | DPRINTF("scan too much\n"); | |
1273 | mutex_exit(&s->lock); | 1279 | mutex_exit(&s->lock); | |
@@ -1295,45 +1301,66 @@ uvmpdpol_selectvictim(kmutex_t **plock) | @@ -1295,45 +1301,66 @@ uvmpdpol_selectvictim(kmutex_t **plock) | |||
1295 | } | 1301 | } | |
1296 | mutex_exit(&s->lock); | 1302 | mutex_exit(&s->lock); | |
1297 | lock = uvmpd_trylockowner(pg); | 1303 | lock = uvmpd_trylockowner(pg); | |
1298 | /* pg->interlock now dropped */ | 1304 | /* pg->interlock now dropped */ | |
1299 | } while (lock == NULL); | 1305 | } while (lock == NULL); | |
1300 | *plock = lock; | 1306 | *plock = lock; | |
1301 | return pg; | 1307 | return pg; | |
1302 | } | 1308 | } | |
1303 | 1309 | |||
1304 | static void | 1310 | static void | |
1305 | clockpro_dropswap(pageq_t *q, int *todo) | 1311 | clockpro_dropswap(pageq_t *q, int *todo) | |
1306 | { | 1312 | { | |
1307 | struct vm_page *pg; | 1313 | struct vm_page *pg; | |
1314 | kmutex_t *lock; | |||
1308 | 1315 | |||
1309 | KASSERT(mutex_owned(&clockpro.lock)); | 1316 | KASSERT(mutex_owned(&clockpro.lock)); | |
1310 | 1317 | |||
1311 | TAILQ_FOREACH_REVERSE(pg, &q->q_q, pglist, pdqueue) { | 1318 | TAILQ_FOREACH_REVERSE(pg, &q->q_q, pglist, pdqueue) { | |
1312 | if (*todo <= 0) { | 1319 | if (*todo <= 0) { | |
1313 | break; | 1320 | break; | |
1314 | } | 1321 | } | |
1315 | if ((pg->pqflags & PQ_HOT) == 0) { | 1322 | if ((pg->pqflags & PQ_HOT) == 0) { | |
1316 | continue; | 1323 | continue; | |
1317 | } | 1324 | } | |
1318 | mutex_enter(&pg->interlock); | 1325 | mutex_enter(&pg->interlock); | |
1319 | if ((pg->flags & PG_SWAPBACKED) == 0) { | 1326 | if ((pg->flags & PG_SWAPBACKED) == 0) { | |
1320 | mutex_exit(&pg->interlock); | 1327 | mutex_exit(&pg->interlock); | |
1321 | continue; | 1328 | continue; | |
1322 | } | 1329 | } | |
1323 | if (uvmpd_trydropswap(pg)) { | 1330 | ||
1324 | (*todo)--; | 1331 | /* | |
1332 | * try to lock the object that owns the page. | |||
1333 | */ | |||
1334 | mutex_exit(&clockpro.lock); | |||
1335 | lock = uvmpd_trylockowner(pg); | |||
1336 | /* pg->interlock now released */ | |||
1337 | mutex_enter(&clockpro.lock); | |||
1338 | if (lock == NULL) { | |||
1339 | /* didn't get it - try the next page. */ | |||
1340 | /* XXXAD lost position in queue */ | |||
1341 | continue; | |||
1325 | } | 1342 | } | |
1326 | /* pg->interlock now dropped */ | 1343 | ||
1344 | /* | |||
1345 | * if there's a shortage of swap slots, try to free it. | |||
1346 | */ | |||
1347 | if ((pg->flags & PG_SWAPBACKED) != 0 && | |||
1348 | (pg->flags & PG_BUSY) == 0) { | |||
1349 | if (uvmpd_dropswap(pg)) { | |||
1350 | (*todo)--; | |||
1351 | } | |||
1352 | } | |||
1353 | mutex_exit(lock); | |||
1327 | } | 1354 | } | |
1328 | } | 1355 | } | |
1329 | 1356 | |||
1330 | void | 1357 | void | |
1331 | uvmpdpol_balancequeue(int swap_shortage) | 1358 | uvmpdpol_balancequeue(int swap_shortage) | |
1332 | { | 1359 | { | |
1333 | struct clockpro_state * const s = &clockpro; | 1360 | struct clockpro_state * const s = &clockpro; | |
1334 | int todo = swap_shortage; | 1361 | int todo = swap_shortage; | |
1335 | 1362 | |||
1336 | if (todo == 0) { | 1363 | if (todo == 0) { | |
1337 | return; | 1364 | return; | |
1338 | } | 1365 | } | |
1339 | 1366 |