| @@ -1,14 +1,14 @@ | | | @@ -1,14 +1,14 @@ |
1 | /* $NetBSD: vfs_bio.c,v 1.297 2020/07/31 04:07:30 chs Exp $ */ | | 1 | /* $NetBSD: vfs_bio.c,v 1.298 2021/04/01 06:25:59 simonb Exp $ */ |
2 | | | 2 | |
3 | /*- | | 3 | /*- |
4 | * Copyright (c) 2007, 2008, 2009, 2019, 2020 The NetBSD Foundation, Inc. | | 4 | * Copyright (c) 2007, 2008, 2009, 2019, 2020 The NetBSD Foundation, Inc. |
5 | * All rights reserved. | | 5 | * All rights reserved. |
6 | * | | 6 | * |
7 | * This code is derived from software contributed to The NetBSD Foundation | | 7 | * This code is derived from software contributed to The NetBSD Foundation |
8 | * by Andrew Doran, and by Wasabi Systems, Inc. | | 8 | * by Andrew Doran, and by Wasabi Systems, Inc. |
9 | * | | 9 | * |
10 | * Redistribution and use in source and binary forms, with or without | | 10 | * Redistribution and use in source and binary forms, with or without |
11 | * modification, are permitted provided that the following conditions | | 11 | * modification, are permitted provided that the following conditions |
12 | * are met: | | 12 | * are met: |
13 | * 1. Redistributions of source code must retain the above copyright | | 13 | * 1. Redistributions of source code must retain the above copyright |
14 | * notice, this list of conditions and the following disclaimer. | | 14 | * notice, this list of conditions and the following disclaimer. |
| @@ -113,27 +113,27 @@ | | | @@ -113,27 +113,27 @@ |
113 | * - bufcache_lock: protects global buffer cache state. | | 113 | * - bufcache_lock: protects global buffer cache state. |
114 | * - BC_BUSY: a long term per-buffer lock. | | 114 | * - BC_BUSY: a long term per-buffer lock. |
115 | * - buf_t::b_objlock: lock on completion (biowait vs biodone). | | 115 | * - buf_t::b_objlock: lock on completion (biowait vs biodone). |
116 | * | | 116 | * |
117 | * For buffers associated with vnodes (a most common case) b_objlock points | | 117 | * For buffers associated with vnodes (a most common case) b_objlock points |
118 | * to the vnode_t::v_interlock. Otherwise, it points to generic buffer_lock. | | 118 | * to the vnode_t::v_interlock. Otherwise, it points to generic buffer_lock. |
119 | * | | 119 | * |
120 | * Lock order: | | 120 | * Lock order: |
121 | * bufcache_lock -> | | 121 | * bufcache_lock -> |
122 | * buf_t::b_objlock | | 122 | * buf_t::b_objlock |
123 | */ | | 123 | */ |
124 | | | 124 | |
125 | #include <sys/cdefs.h> | | 125 | #include <sys/cdefs.h> |
126 | __KERNEL_RCSID(0, "$NetBSD: vfs_bio.c,v 1.297 2020/07/31 04:07:30 chs Exp $"); | | 126 | __KERNEL_RCSID(0, "$NetBSD: vfs_bio.c,v 1.298 2021/04/01 06:25:59 simonb Exp $"); |
127 | | | 127 | |
128 | #ifdef _KERNEL_OPT | | 128 | #ifdef _KERNEL_OPT |
129 | #include "opt_bufcache.h" | | 129 | #include "opt_bufcache.h" |
130 | #include "opt_dtrace.h" | | 130 | #include "opt_dtrace.h" |
131 | #include "opt_biohist.h" | | 131 | #include "opt_biohist.h" |
132 | #endif | | 132 | #endif |
133 | | | 133 | |
134 | #include <sys/param.h> | | 134 | #include <sys/param.h> |
135 | #include <sys/systm.h> | | 135 | #include <sys/systm.h> |
136 | #include <sys/kernel.h> | | 136 | #include <sys/kernel.h> |
137 | #include <sys/proc.h> | | 137 | #include <sys/proc.h> |
138 | #include <sys/buf.h> | | 138 | #include <sys/buf.h> |
139 | #include <sys/vnode.h> | | 139 | #include <sys/vnode.h> |
| @@ -242,26 +242,28 @@ biohist_init(void) | | | @@ -242,26 +242,28 @@ biohist_init(void) |
242 | { | | 242 | { |
243 | | | 243 | |
244 | BIOHIST_INIT(biohist, BIOHIST_SIZE); | | 244 | BIOHIST_INIT(biohist, BIOHIST_SIZE); |
245 | } | | 245 | } |
246 | | | 246 | |
247 | /* | | 247 | /* |
248 | * Definitions for the buffer hash lists. | | 248 | * Definitions for the buffer hash lists. |
249 | */ | | 249 | */ |
250 | #define BUFHASH(dvp, lbn) \ | | 250 | #define BUFHASH(dvp, lbn) \ |
251 | (&bufhashtbl[(((long)(dvp) >> 8) + (int)(lbn)) & bufhash]) | | 251 | (&bufhashtbl[(((long)(dvp) >> 8) + (int)(lbn)) & bufhash]) |
252 | LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash; | | 252 | LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash; |
253 | u_long bufhash; | | 253 | u_long bufhash; |
254 | | | 254 | |
| | | 255 | static int bufhash_stats(struct hashstat_sysctl *, bool); |
| | | 256 | |
255 | static kcondvar_t needbuffer_cv; | | 257 | static kcondvar_t needbuffer_cv; |
256 | | | 258 | |
257 | /* | | 259 | /* |
258 | * Buffer queue lock. | | 260 | * Buffer queue lock. |
259 | */ | | 261 | */ |
260 | kmutex_t bufcache_lock __cacheline_aligned; | | 262 | kmutex_t bufcache_lock __cacheline_aligned; |
261 | kmutex_t buffer_lock __cacheline_aligned; | | 263 | kmutex_t buffer_lock __cacheline_aligned; |
262 | | | 264 | |
263 | /* Software ISR for completed transfers. */ | | 265 | /* Software ISR for completed transfers. */ |
264 | static void *biodone_sih; | | 266 | static void *biodone_sih; |
265 | | | 267 | |
266 | /* Buffer pool for I/O buffers. */ | | 268 | /* Buffer pool for I/O buffers. */ |
267 | static pool_cache_t buf_cache; | | 269 | static pool_cache_t buf_cache; |
| @@ -526,26 +528,27 @@ bufinit(void) | | | @@ -526,26 +528,27 @@ bufinit(void) |
526 | | | 528 | |
527 | /* | | 529 | /* |
528 | * Estimate hash table size based on the amount of memory we | | 530 | * Estimate hash table size based on the amount of memory we |
529 | * intend to use for the buffer cache. The average buffer | | 531 | * intend to use for the buffer cache. The average buffer |
530 | * size is dependent on our clients (i.e. filesystems). | | 532 | * size is dependent on our clients (i.e. filesystems). |
531 | * | | 533 | * |
532 | * For now, use an empirical 3K per buffer. | | 534 | * For now, use an empirical 3K per buffer. |
533 | */ | | 535 | */ |
534 | nbuf = (bufmem_hiwater / 1024) / 3; | | 536 | nbuf = (bufmem_hiwater / 1024) / 3; |
535 | bufhashtbl = hashinit(nbuf, HASH_LIST, true, &bufhash); | | 537 | bufhashtbl = hashinit(nbuf, HASH_LIST, true, &bufhash); |
536 | | | 538 | |
537 | sysctl_kern_buf_setup(); | | 539 | sysctl_kern_buf_setup(); |
538 | sysctl_vm_buf_setup(); | | 540 | sysctl_vm_buf_setup(); |
| | | 541 | hashstat_register("bufhash", bufhash_stats); |
539 | } | | 542 | } |
540 | | | 543 | |
541 | void | | 544 | void |
542 | bufinit2(void) | | 545 | bufinit2(void) |
543 | { | | 546 | { |
544 | | | 547 | |
545 | biodone_sih = softint_establish(SOFTINT_BIO | SOFTINT_MPSAFE, biointr, | | 548 | biodone_sih = softint_establish(SOFTINT_BIO | SOFTINT_MPSAFE, biointr, |
546 | NULL); | | 549 | NULL); |
547 | if (biodone_sih == NULL) | | 550 | if (biodone_sih == NULL) |
548 | panic("bufinit2: can't establish soft interrupt"); | | 551 | panic("bufinit2: can't establish soft interrupt"); |
549 | } | | 552 | } |
550 | | | 553 | |
551 | static int | | 554 | static int |
| @@ -1943,26 +1946,60 @@ sysctl_vm_buf_setup(void) | | | @@ -1943,26 +1946,60 @@ sysctl_vm_buf_setup(void) |
1943 | SYSCTL_DESCR("Minimum amount of kernel memory to " | | 1946 | SYSCTL_DESCR("Minimum amount of kernel memory to " |
1944 | "reserve for buffer cache"), | | 1947 | "reserve for buffer cache"), |
1945 | sysctl_bufvm_update, 0, &bufmem_lowater, 0, | | 1948 | sysctl_bufvm_update, 0, &bufmem_lowater, 0, |
1946 | CTL_VM, CTL_CREATE, CTL_EOL); | | 1949 | CTL_VM, CTL_CREATE, CTL_EOL); |
1947 | sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL, | | 1950 | sysctl_createv(&vfsbio_sysctllog, 0, NULL, NULL, |
1948 | CTLFLAG_PERMANENT|CTLFLAG_READWRITE, | | 1951 | CTLFLAG_PERMANENT|CTLFLAG_READWRITE, |
1949 | CTLTYPE_LONG, "bufmem_hiwater", | | 1952 | CTLTYPE_LONG, "bufmem_hiwater", |
1950 | SYSCTL_DESCR("Maximum amount of kernel memory to use " | | 1953 | SYSCTL_DESCR("Maximum amount of kernel memory to use " |
1951 | "for buffer cache"), | | 1954 | "for buffer cache"), |
1952 | sysctl_bufvm_update, 0, &bufmem_hiwater, 0, | | 1955 | sysctl_bufvm_update, 0, &bufmem_hiwater, 0, |
1953 | CTL_VM, CTL_CREATE, CTL_EOL); | | 1956 | CTL_VM, CTL_CREATE, CTL_EOL); |
1954 | } | | 1957 | } |
1955 | | | 1958 | |
| | | 1959 | static int |
| | | 1960 | bufhash_stats(struct hashstat_sysctl *hs, bool fill) |
| | | 1961 | { |
| | | 1962 | buf_t *bp; |
| | | 1963 | uint64_t chain; |
| | | 1964 | |
| | | 1965 | strlcpy(hs->hash_name, "bufhash", sizeof(hs->hash_name)); |
| | | 1966 | strlcpy(hs->hash_desc, "buffer hash", sizeof(hs->hash_desc)); |
| | | 1967 | if (!fill) |
| | | 1968 | return 0; |
| | | 1969 | |
| | | 1970 | hs->hash_size = bufhash + 1; |
| | | 1971 | |
| | | 1972 | for (size_t i = 0; i < hs->hash_size; i++) { |
| | | 1973 | chain = 0; |
| | | 1974 | |
| | | 1975 | mutex_enter(&bufcache_lock); |
| | | 1976 | LIST_FOREACH(bp, &bufhashtbl[i], b_hash) { |
| | | 1977 | chain++; |
| | | 1978 | } |
| | | 1979 | mutex_exit(&bufcache_lock); |
| | | 1980 | |
| | | 1981 | if (chain > 0) { |
| | | 1982 | hs->hash_used++; |
| | | 1983 | hs->hash_items += chain; |
| | | 1984 | if (chain > hs->hash_maxchain) |
| | | 1985 | hs->hash_maxchain = chain; |
| | | 1986 | } |
| | | 1987 | preempt_point(); |
| | | 1988 | } |
| | | 1989 | |
| | | 1990 | return 0; |
| | | 1991 | } |
| | | 1992 | |
1956 | #ifdef DEBUG | | 1993 | #ifdef DEBUG |
1957 | /* | | 1994 | /* |
1958 | * Print out statistics on the current allocation of the buffer pool. | | 1995 | * Print out statistics on the current allocation of the buffer pool. |
1959 | * Can be enabled to print out on every ``sync'' by setting "syncprt" | | 1996 | * Can be enabled to print out on every ``sync'' by setting "syncprt" |
1960 | * in vfs_syscalls.c using sysctl. | | 1997 | * in vfs_syscalls.c using sysctl. |
1961 | */ | | 1998 | */ |
1962 | void | | 1999 | void |
1963 | vfs_bufstats(void) | | 2000 | vfs_bufstats(void) |
1964 | { | | 2001 | { |
1965 | int i, j, count; | | 2002 | int i, j, count; |
1966 | buf_t *bp; | | 2003 | buf_t *bp; |
1967 | struct bqueue *dp; | | 2004 | struct bqueue *dp; |
1968 | int counts[MAXBSIZE / MIN_PAGE_SIZE + 1]; | | 2005 | int counts[MAXBSIZE / MIN_PAGE_SIZE + 1]; |