Mon Mar 6 10:07:52 2017 UTC ()
Change vrecycle() and vgone() to lock with LK_RETRY.  If this node is
a layerfs node the lower node(s) may already be reclaimed.


(hannken)
diff -r1.75 -r1.76 src/sys/kern/vfs_vnode.c
cvs diff -r1.75 -r1.76 src/sys/kern/vfs_vnode.c (switch to unified diff)

--- src/sys/kern/vfs_vnode.c 2017/02/17 08:30:00 1.75
+++ src/sys/kern/vfs_vnode.c 2017/03/06 10:07:52 1.76
@@ -1,1672 +1,1671 @@ @@ -1,1672 +1,1671 @@
1/* $NetBSD: vfs_vnode.c,v 1.75 2017/02/17 08:30:00 hannken Exp $ */ 1/* $NetBSD: vfs_vnode.c,v 1.76 2017/03/06 10:07:52 hannken Exp $ */
2 2
3/*- 3/*-
4 * Copyright (c) 1997-2011 The NetBSD Foundation, Inc. 4 * Copyright (c) 1997-2011 The NetBSD Foundation, Inc.
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * This code is derived from software contributed to The NetBSD Foundation 7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 * NASA Ames Research Center, by Charles M. Hannum, and by Andrew Doran. 9 * NASA Ames Research Center, by Charles M. Hannum, and by Andrew Doran.
10 * 10 *
11 * Redistribution and use in source and binary forms, with or without 11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions 12 * modification, are permitted provided that the following conditions
13 * are met: 13 * are met:
14 * 1. Redistributions of source code must retain the above copyright 14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer. 15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright 16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the 17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution. 18 * documentation and/or other materials provided with the distribution.
19 * 19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE. 30 * POSSIBILITY OF SUCH DAMAGE.
31 */ 31 */
32 32
33/* 33/*
34 * Copyright (c) 1989, 1993 34 * Copyright (c) 1989, 1993
35 * The Regents of the University of California. All rights reserved. 35 * The Regents of the University of California. All rights reserved.
36 * (c) UNIX System Laboratories, Inc. 36 * (c) UNIX System Laboratories, Inc.
37 * All or some portions of this file are derived from material licensed 37 * All or some portions of this file are derived from material licensed
38 * to the University of California by American Telephone and Telegraph 38 * to the University of California by American Telephone and Telegraph
39 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 39 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
40 * the permission of UNIX System Laboratories, Inc. 40 * the permission of UNIX System Laboratories, Inc.
41 * 41 *
42 * Redistribution and use in source and binary forms, with or without 42 * Redistribution and use in source and binary forms, with or without
43 * modification, are permitted provided that the following conditions 43 * modification, are permitted provided that the following conditions
44 * are met: 44 * are met:
45 * 1. Redistributions of source code must retain the above copyright 45 * 1. Redistributions of source code must retain the above copyright
46 * notice, this list of conditions and the following disclaimer. 46 * notice, this list of conditions and the following disclaimer.
47 * 2. Redistributions in binary form must reproduce the above copyright 47 * 2. Redistributions in binary form must reproduce the above copyright
48 * notice, this list of conditions and the following disclaimer in the 48 * notice, this list of conditions and the following disclaimer in the
49 * documentation and/or other materials provided with the distribution. 49 * documentation and/or other materials provided with the distribution.
50 * 3. Neither the name of the University nor the names of its contributors 50 * 3. Neither the name of the University nor the names of its contributors
51 * may be used to endorse or promote products derived from this software 51 * may be used to endorse or promote products derived from this software
52 * without specific prior written permission. 52 * without specific prior written permission.
53 * 53 *
54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 * SUCH DAMAGE. 64 * SUCH DAMAGE.
65 * 65 *
66 * @(#)vfs_subr.c 8.13 (Berkeley) 4/18/94 66 * @(#)vfs_subr.c 8.13 (Berkeley) 4/18/94
67 */ 67 */
68 68
69/* 69/*
70 * The vnode cache subsystem. 70 * The vnode cache subsystem.
71 * 71 *
72 * Life-cycle 72 * Life-cycle
73 * 73 *
74 * Normally, there are two points where new vnodes are created: 74 * Normally, there are two points where new vnodes are created:
75 * VOP_CREATE(9) and VOP_LOOKUP(9). The life-cycle of a vnode 75 * VOP_CREATE(9) and VOP_LOOKUP(9). The life-cycle of a vnode
76 * starts in one of the following ways: 76 * starts in one of the following ways:
77 * 77 *
78 * - Allocation, via vcache_get(9) or vcache_new(9). 78 * - Allocation, via vcache_get(9) or vcache_new(9).
79 * - Reclamation of inactive vnode, via vcache_vget(9). 79 * - Reclamation of inactive vnode, via vcache_vget(9).
80 * 80 *
81 * Recycle from a free list, via getnewvnode(9) -> getcleanvnode(9) 81 * Recycle from a free list, via getnewvnode(9) -> getcleanvnode(9)
82 * was another, traditional way. Currently, only the draining thread 82 * was another, traditional way. Currently, only the draining thread
83 * recycles the vnodes. This behaviour might be revisited. 83 * recycles the vnodes. This behaviour might be revisited.
84 * 84 *
85 * The life-cycle ends when the last reference is dropped, usually 85 * The life-cycle ends when the last reference is dropped, usually
86 * in VOP_REMOVE(9). In such case, VOP_INACTIVE(9) is called to inform 86 * in VOP_REMOVE(9). In such case, VOP_INACTIVE(9) is called to inform
87 * the file system that vnode is inactive. Via this call, file system 87 * the file system that vnode is inactive. Via this call, file system
88 * indicates whether vnode can be recycled (usually, it checks its own 88 * indicates whether vnode can be recycled (usually, it checks its own
89 * references, e.g. count of links, whether the file was removed). 89 * references, e.g. count of links, whether the file was removed).
90 * 90 *
91 * Depending on indication, vnode can be put into a free list (cache), 91 * Depending on indication, vnode can be put into a free list (cache),
92 * or cleaned via vcache_reclaim, which calls VOP_RECLAIM(9) to 92 * or cleaned via vcache_reclaim, which calls VOP_RECLAIM(9) to
93 * disassociate underlying file system from the vnode, and finally 93 * disassociate underlying file system from the vnode, and finally
94 * destroyed. 94 * destroyed.
95 * 95 *
96 * Vnode state 96 * Vnode state
97 * 97 *
98 * Vnode is always in one of six states: 98 * Vnode is always in one of six states:
99 * - MARKER This is a marker vnode to help list traversal. It 99 * - MARKER This is a marker vnode to help list traversal. It
100 * will never change its state. 100 * will never change its state.
101 * - LOADING Vnode is associating underlying file system and not 101 * - LOADING Vnode is associating underlying file system and not
102 * yet ready to use. 102 * yet ready to use.
103 * - ACTIVE Vnode has associated underlying file system and is 103 * - ACTIVE Vnode has associated underlying file system and is
104 * ready to use. 104 * ready to use.
105 * - BLOCKED Vnode is active but cannot get new references. 105 * - BLOCKED Vnode is active but cannot get new references.
106 * - RECLAIMING Vnode is disassociating from the underlying file 106 * - RECLAIMING Vnode is disassociating from the underlying file
107 * system. 107 * system.
108 * - RECLAIMED Vnode has disassociated from underlying file system 108 * - RECLAIMED Vnode has disassociated from underlying file system
109 * and is dead. 109 * and is dead.
110 * 110 *
111 * Valid state changes are: 111 * Valid state changes are:
112 * LOADING -> ACTIVE 112 * LOADING -> ACTIVE
113 * Vnode has been initialised in vcache_get() or 113 * Vnode has been initialised in vcache_get() or
114 * vcache_new() and is ready to use. 114 * vcache_new() and is ready to use.
115 * ACTIVE -> RECLAIMING 115 * ACTIVE -> RECLAIMING
116 * Vnode starts disassociation from underlying file 116 * Vnode starts disassociation from underlying file
117 * system in vcache_reclaim(). 117 * system in vcache_reclaim().
118 * RECLAIMING -> RECLAIMED 118 * RECLAIMING -> RECLAIMED
119 * Vnode finished disassociation from underlying file 119 * Vnode finished disassociation from underlying file
120 * system in vcache_reclaim(). 120 * system in vcache_reclaim().
121 * ACTIVE -> BLOCKED 121 * ACTIVE -> BLOCKED
122 * Either vcache_rekey*() is changing the vnode key or 122 * Either vcache_rekey*() is changing the vnode key or
123 * vrelel() is about to call VOP_INACTIVE(). 123 * vrelel() is about to call VOP_INACTIVE().
124 * BLOCKED -> ACTIVE 124 * BLOCKED -> ACTIVE
125 * The block condition is over. 125 * The block condition is over.
126 * LOADING -> RECLAIMED 126 * LOADING -> RECLAIMED
127 * Either vcache_get() or vcache_new() failed to 127 * Either vcache_get() or vcache_new() failed to
128 * associate the underlying file system or vcache_rekey*() 128 * associate the underlying file system or vcache_rekey*()
129 * drops a vnode used as placeholder. 129 * drops a vnode used as placeholder.
130 * 130 *
131 * Of these states LOADING, BLOCKED and RECLAIMING are intermediate 131 * Of these states LOADING, BLOCKED and RECLAIMING are intermediate
132 * and it is possible to wait for state change. 132 * and it is possible to wait for state change.
133 * 133 *
134 * State is protected with v_interlock with one exception: 134 * State is protected with v_interlock with one exception:
135 * to change from LOADING both v_interlock and vcache_lock must be held 135 * to change from LOADING both v_interlock and vcache_lock must be held
136 * so it is possible to check "state == LOADING" without holding 136 * so it is possible to check "state == LOADING" without holding
137 * v_interlock. See vcache_get() for details. 137 * v_interlock. See vcache_get() for details.
138 * 138 *
139 * Reference counting 139 * Reference counting
140 * 140 *
141 * Vnode is considered active, if reference count (vnode_t::v_usecount) 141 * Vnode is considered active, if reference count (vnode_t::v_usecount)
142 * is non-zero. It is maintained using: vref(9) and vrele(9), as well 142 * is non-zero. It is maintained using: vref(9) and vrele(9), as well
143 * as vput(9), routines. Common points holding references are e.g. 143 * as vput(9), routines. Common points holding references are e.g.
144 * file openings, current working directory, mount points, etc.  144 * file openings, current working directory, mount points, etc.
145 * 145 *
146 * Note on v_usecount and its locking 146 * Note on v_usecount and its locking
147 * 147 *
148 * At nearly all points it is known that v_usecount could be zero, 148 * At nearly all points it is known that v_usecount could be zero,
149 * the vnode_t::v_interlock will be held. To change v_usecount away 149 * the vnode_t::v_interlock will be held. To change v_usecount away
150 * from zero, the interlock must be held. To change from a non-zero 150 * from zero, the interlock must be held. To change from a non-zero
151 * value to zero, again the interlock must be held. 151 * value to zero, again the interlock must be held.
152 * 152 *
153 * Changing the usecount from a non-zero value to a non-zero value can 153 * Changing the usecount from a non-zero value to a non-zero value can
154 * safely be done using atomic operations, without the interlock held. 154 * safely be done using atomic operations, without the interlock held.
155 * 155 *
156 */ 156 */
157 157
158#include <sys/cdefs.h> 158#include <sys/cdefs.h>
159__KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.75 2017/02/17 08:30:00 hannken Exp $"); 159__KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.76 2017/03/06 10:07:52 hannken Exp $");
160 160
161#include <sys/param.h> 161#include <sys/param.h>
162#include <sys/kernel.h> 162#include <sys/kernel.h>
163 163
164#include <sys/atomic.h> 164#include <sys/atomic.h>
165#include <sys/buf.h> 165#include <sys/buf.h>
166#include <sys/conf.h> 166#include <sys/conf.h>
167#include <sys/device.h> 167#include <sys/device.h>
168#include <sys/hash.h> 168#include <sys/hash.h>
169#include <sys/kauth.h> 169#include <sys/kauth.h>
170#include <sys/kmem.h> 170#include <sys/kmem.h>
171#include <sys/kthread.h> 171#include <sys/kthread.h>
172#include <sys/module.h> 172#include <sys/module.h>
173#include <sys/mount.h> 173#include <sys/mount.h>
174#include <sys/namei.h> 174#include <sys/namei.h>
175#include <sys/syscallargs.h> 175#include <sys/syscallargs.h>
176#include <sys/sysctl.h> 176#include <sys/sysctl.h>
177#include <sys/systm.h> 177#include <sys/systm.h>
178#include <sys/vnode_impl.h> 178#include <sys/vnode_impl.h>
179#include <sys/wapbl.h> 179#include <sys/wapbl.h>
180#include <sys/fstrans.h> 180#include <sys/fstrans.h>
181 181
182#include <uvm/uvm.h> 182#include <uvm/uvm.h>
183#include <uvm/uvm_readahead.h> 183#include <uvm/uvm_readahead.h>
184 184
185/* Flags to vrelel. */ 185/* Flags to vrelel. */
186#define VRELEL_ASYNC_RELE 0x0001 /* Always defer to vrele thread. */ 186#define VRELEL_ASYNC_RELE 0x0001 /* Always defer to vrele thread. */
187 187
188u_int numvnodes __cacheline_aligned; 188u_int numvnodes __cacheline_aligned;
189 189
190/* 190/*
191 * There are three lru lists: one holds vnodes waiting for async release, 191 * There are three lru lists: one holds vnodes waiting for async release,
192 * one is for vnodes which have no buffer/page references and 192 * one is for vnodes which have no buffer/page references and
193 * one for those which do (i.e. v_holdcnt is non-zero). 193 * one for those which do (i.e. v_holdcnt is non-zero).
194 */ 194 */
195static vnodelst_t lru_vrele_list __cacheline_aligned; 195static vnodelst_t lru_vrele_list __cacheline_aligned;
196static vnodelst_t lru_free_list __cacheline_aligned; 196static vnodelst_t lru_free_list __cacheline_aligned;
197static vnodelst_t lru_hold_list __cacheline_aligned; 197static vnodelst_t lru_hold_list __cacheline_aligned;
198static kmutex_t vdrain_lock __cacheline_aligned; 198static kmutex_t vdrain_lock __cacheline_aligned;
199static kcondvar_t vdrain_cv __cacheline_aligned; 199static kcondvar_t vdrain_cv __cacheline_aligned;
200static int vdrain_gen; 200static int vdrain_gen;
201static kcondvar_t vdrain_gen_cv; 201static kcondvar_t vdrain_gen_cv;
202static bool vdrain_retry; 202static bool vdrain_retry;
203static lwp_t * vdrain_lwp; 203static lwp_t * vdrain_lwp;
204SLIST_HEAD(hashhead, vnode_impl); 204SLIST_HEAD(hashhead, vnode_impl);
205static kmutex_t vcache_lock __cacheline_aligned; 205static kmutex_t vcache_lock __cacheline_aligned;
206static kcondvar_t vcache_cv __cacheline_aligned; 206static kcondvar_t vcache_cv __cacheline_aligned;
207static u_int vcache_hashsize; 207static u_int vcache_hashsize;
208static u_long vcache_hashmask; 208static u_long vcache_hashmask;
209static struct hashhead *vcache_hashtab __cacheline_aligned; 209static struct hashhead *vcache_hashtab __cacheline_aligned;
210static pool_cache_t vcache_pool; 210static pool_cache_t vcache_pool;
211static void lru_requeue(vnode_t *, vnodelst_t *); 211static void lru_requeue(vnode_t *, vnodelst_t *);
212static vnodelst_t * lru_which(vnode_t *); 212static vnodelst_t * lru_which(vnode_t *);
213static vnode_impl_t * vcache_alloc(void); 213static vnode_impl_t * vcache_alloc(void);
214static void vcache_free(vnode_impl_t *); 214static void vcache_free(vnode_impl_t *);
215static void vcache_init(void); 215static void vcache_init(void);
216static void vcache_reinit(void); 216static void vcache_reinit(void);
217static void vcache_reclaim(vnode_t *); 217static void vcache_reclaim(vnode_t *);
218static void vrelel(vnode_t *, int); 218static void vrelel(vnode_t *, int);
219static void vdrain_thread(void *); 219static void vdrain_thread(void *);
220static void vnpanic(vnode_t *, const char *, ...) 220static void vnpanic(vnode_t *, const char *, ...)
221 __printflike(2, 3); 221 __printflike(2, 3);
222 222
223/* Routines having to do with the management of the vnode table. */ 223/* Routines having to do with the management of the vnode table. */
224extern struct mount *dead_rootmount; 224extern struct mount *dead_rootmount;
225extern int (**dead_vnodeop_p)(void *); 225extern int (**dead_vnodeop_p)(void *);
226extern struct vfsops dead_vfsops; 226extern struct vfsops dead_vfsops;
227 227
228/* Vnode state operations and diagnostics. */ 228/* Vnode state operations and diagnostics. */
229 229
230#if defined(DIAGNOSTIC) 230#if defined(DIAGNOSTIC)
231 231
232#define VSTATE_GET(vp) \ 232#define VSTATE_GET(vp) \
233 vstate_assert_get((vp), __func__, __LINE__) 233 vstate_assert_get((vp), __func__, __LINE__)
234#define VSTATE_CHANGE(vp, from, to) \ 234#define VSTATE_CHANGE(vp, from, to) \
235 vstate_assert_change((vp), (from), (to), __func__, __LINE__) 235 vstate_assert_change((vp), (from), (to), __func__, __LINE__)
236#define VSTATE_WAIT_STABLE(vp) \ 236#define VSTATE_WAIT_STABLE(vp) \
237 vstate_assert_wait_stable((vp), __func__, __LINE__) 237 vstate_assert_wait_stable((vp), __func__, __LINE__)
238#define VSTATE_ASSERT(vp, state) \ 238#define VSTATE_ASSERT(vp, state) \
239 vstate_assert((vp), (state), __func__, __LINE__) 239 vstate_assert((vp), (state), __func__, __LINE__)
240 240
241static void 241static void
242vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line) 242vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line)
243{ 243{
244 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 244 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
245 245
246 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line); 246 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
247 247
248 if (__predict_true(vip->vi_state == state)) 248 if (__predict_true(vip->vi_state == state))
249 return; 249 return;
250 vnpanic(vp, "state is %s, expected %s at %s:%d", 250 vnpanic(vp, "state is %s, expected %s at %s:%d",
251 vstate_name(vip->vi_state), vstate_name(state), func, line); 251 vstate_name(vip->vi_state), vstate_name(state), func, line);
252} 252}
253 253
254static enum vnode_state 254static enum vnode_state
255vstate_assert_get(vnode_t *vp, const char *func, int line) 255vstate_assert_get(vnode_t *vp, const char *func, int line)
256{ 256{
257 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 257 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
258 258
259 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line); 259 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
260 if (vip->vi_state == VS_MARKER) 260 if (vip->vi_state == VS_MARKER)
261 vnpanic(vp, "state is %s at %s:%d", 261 vnpanic(vp, "state is %s at %s:%d",
262 vstate_name(vip->vi_state), func, line); 262 vstate_name(vip->vi_state), func, line);
263 263
264 return vip->vi_state; 264 return vip->vi_state;
265} 265}
266 266
267static void 267static void
268vstate_assert_wait_stable(vnode_t *vp, const char *func, int line) 268vstate_assert_wait_stable(vnode_t *vp, const char *func, int line)
269{ 269{
270 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 270 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
271 271
272 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line); 272 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
273 if (vip->vi_state == VS_MARKER) 273 if (vip->vi_state == VS_MARKER)
274 vnpanic(vp, "state is %s at %s:%d", 274 vnpanic(vp, "state is %s at %s:%d",
275 vstate_name(vip->vi_state), func, line); 275 vstate_name(vip->vi_state), func, line);
276 276
277 while (vip->vi_state != VS_ACTIVE && vip->vi_state != VS_RECLAIMED) 277 while (vip->vi_state != VS_ACTIVE && vip->vi_state != VS_RECLAIMED)
278 cv_wait(&vp->v_cv, vp->v_interlock); 278 cv_wait(&vp->v_cv, vp->v_interlock);
279 279
280 if (vip->vi_state == VS_MARKER) 280 if (vip->vi_state == VS_MARKER)
281 vnpanic(vp, "state is %s at %s:%d", 281 vnpanic(vp, "state is %s at %s:%d",
282 vstate_name(vip->vi_state), func, line); 282 vstate_name(vip->vi_state), func, line);
283} 283}
284 284
285static void 285static void
286vstate_assert_change(vnode_t *vp, enum vnode_state from, enum vnode_state to, 286vstate_assert_change(vnode_t *vp, enum vnode_state from, enum vnode_state to,
287 const char *func, int line) 287 const char *func, int line)
288{ 288{
289 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 289 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
290 290
291 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line); 291 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
292 if (from == VS_LOADING) 292 if (from == VS_LOADING)
293 KASSERTMSG(mutex_owned(&vcache_lock), "at %s:%d", func, line); 293 KASSERTMSG(mutex_owned(&vcache_lock), "at %s:%d", func, line);
294 294
295 if (from == VS_MARKER) 295 if (from == VS_MARKER)
296 vnpanic(vp, "from is %s at %s:%d", 296 vnpanic(vp, "from is %s at %s:%d",
297 vstate_name(from), func, line); 297 vstate_name(from), func, line);
298 if (to == VS_MARKER) 298 if (to == VS_MARKER)
299 vnpanic(vp, "to is %s at %s:%d", 299 vnpanic(vp, "to is %s at %s:%d",
300 vstate_name(to), func, line); 300 vstate_name(to), func, line);
301 if (vip->vi_state != from) 301 if (vip->vi_state != from)
302 vnpanic(vp, "from is %s, expected %s at %s:%d\n", 302 vnpanic(vp, "from is %s, expected %s at %s:%d\n",
303 vstate_name(vip->vi_state), vstate_name(from), func, line); 303 vstate_name(vip->vi_state), vstate_name(from), func, line);
304 if ((from == VS_BLOCKED || to == VS_BLOCKED) && vp->v_usecount != 1) 304 if ((from == VS_BLOCKED || to == VS_BLOCKED) && vp->v_usecount != 1)
305 vnpanic(vp, "%s to %s with usecount %d at %s:%d", 305 vnpanic(vp, "%s to %s with usecount %d at %s:%d",
306 vstate_name(from), vstate_name(to), vp->v_usecount, 306 vstate_name(from), vstate_name(to), vp->v_usecount,
307 func, line); 307 func, line);
308 308
309 vip->vi_state = to; 309 vip->vi_state = to;
310 if (from == VS_LOADING) 310 if (from == VS_LOADING)
311 cv_broadcast(&vcache_cv); 311 cv_broadcast(&vcache_cv);
312 if (to == VS_ACTIVE || to == VS_RECLAIMED) 312 if (to == VS_ACTIVE || to == VS_RECLAIMED)
313 cv_broadcast(&vp->v_cv); 313 cv_broadcast(&vp->v_cv);
314} 314}
315 315
316#else /* defined(DIAGNOSTIC) */ 316#else /* defined(DIAGNOSTIC) */
317 317
318#define VSTATE_GET(vp) \ 318#define VSTATE_GET(vp) \
319 (VNODE_TO_VIMPL((vp))->vi_state) 319 (VNODE_TO_VIMPL((vp))->vi_state)
320#define VSTATE_CHANGE(vp, from, to) \ 320#define VSTATE_CHANGE(vp, from, to) \
321 vstate_change((vp), (from), (to)) 321 vstate_change((vp), (from), (to))
322#define VSTATE_WAIT_STABLE(vp) \ 322#define VSTATE_WAIT_STABLE(vp) \
323 vstate_wait_stable((vp)) 323 vstate_wait_stable((vp))
324#define VSTATE_ASSERT(vp, state) 324#define VSTATE_ASSERT(vp, state)
325 325
326static void 326static void
327vstate_wait_stable(vnode_t *vp) 327vstate_wait_stable(vnode_t *vp)
328{ 328{
329 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 329 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
330 330
331 while (vip->vi_state != VS_ACTIVE && vip->vi_state != VS_RECLAIMED) 331 while (vip->vi_state != VS_ACTIVE && vip->vi_state != VS_RECLAIMED)
332 cv_wait(&vp->v_cv, vp->v_interlock); 332 cv_wait(&vp->v_cv, vp->v_interlock);
333} 333}
334 334
335static void 335static void
336vstate_change(vnode_t *vp, enum vnode_state from, enum vnode_state to) 336vstate_change(vnode_t *vp, enum vnode_state from, enum vnode_state to)
337{ 337{
338 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 338 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
339 339
340 vip->vi_state = to; 340 vip->vi_state = to;
341 if (from == VS_LOADING) 341 if (from == VS_LOADING)
342 cv_broadcast(&vcache_cv); 342 cv_broadcast(&vcache_cv);
343 if (to == VS_ACTIVE || to == VS_RECLAIMED) 343 if (to == VS_ACTIVE || to == VS_RECLAIMED)
344 cv_broadcast(&vp->v_cv); 344 cv_broadcast(&vp->v_cv);
345} 345}
346 346
347#endif /* defined(DIAGNOSTIC) */ 347#endif /* defined(DIAGNOSTIC) */
348 348
349void 349void
350vfs_vnode_sysinit(void) 350vfs_vnode_sysinit(void)
351{ 351{
352 int error __diagused; 352 int error __diagused;
353 353
354 dead_rootmount = vfs_mountalloc(&dead_vfsops, NULL); 354 dead_rootmount = vfs_mountalloc(&dead_vfsops, NULL);
355 KASSERT(dead_rootmount != NULL); 355 KASSERT(dead_rootmount != NULL);
356 dead_rootmount->mnt_iflag = IMNT_MPSAFE; 356 dead_rootmount->mnt_iflag = IMNT_MPSAFE;
357 357
358 mutex_init(&vdrain_lock, MUTEX_DEFAULT, IPL_NONE); 358 mutex_init(&vdrain_lock, MUTEX_DEFAULT, IPL_NONE);
359 TAILQ_INIT(&lru_free_list); 359 TAILQ_INIT(&lru_free_list);
360 TAILQ_INIT(&lru_hold_list); 360 TAILQ_INIT(&lru_hold_list);
361 TAILQ_INIT(&lru_vrele_list); 361 TAILQ_INIT(&lru_vrele_list);
362 362
363 vcache_init(); 363 vcache_init();
364 364
365 cv_init(&vdrain_cv, "vdrain"); 365 cv_init(&vdrain_cv, "vdrain");
366 cv_init(&vdrain_gen_cv, "vdrainwt"); 366 cv_init(&vdrain_gen_cv, "vdrainwt");
367 error = kthread_create(PRI_VM, KTHREAD_MPSAFE, NULL, vdrain_thread, 367 error = kthread_create(PRI_VM, KTHREAD_MPSAFE, NULL, vdrain_thread,
368 NULL, &vdrain_lwp, "vdrain"); 368 NULL, &vdrain_lwp, "vdrain");
369 KASSERTMSG((error == 0), "kthread_create(vdrain) failed: %d", error); 369 KASSERTMSG((error == 0), "kthread_create(vdrain) failed: %d", error);
370} 370}
371 371
372/* 372/*
373 * Allocate a new marker vnode. 373 * Allocate a new marker vnode.
374 */ 374 */
375vnode_t * 375vnode_t *
376vnalloc_marker(struct mount *mp) 376vnalloc_marker(struct mount *mp)
377{ 377{
378 vnode_impl_t *vip; 378 vnode_impl_t *vip;
379 vnode_t *vp; 379 vnode_t *vp;
380 380
381 vip = pool_cache_get(vcache_pool, PR_WAITOK); 381 vip = pool_cache_get(vcache_pool, PR_WAITOK);
382 memset(vip, 0, sizeof(*vip)); 382 memset(vip, 0, sizeof(*vip));
383 vp = VIMPL_TO_VNODE(vip); 383 vp = VIMPL_TO_VNODE(vip);
384 uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 0); 384 uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 0);
385 vp->v_mount = mp; 385 vp->v_mount = mp;
386 vp->v_type = VBAD; 386 vp->v_type = VBAD;
387 vip->vi_state = VS_MARKER; 387 vip->vi_state = VS_MARKER;
388 388
389 return vp; 389 return vp;
390} 390}
391 391
392/* 392/*
393 * Free a marker vnode. 393 * Free a marker vnode.
394 */ 394 */
395void 395void
396vnfree_marker(vnode_t *vp) 396vnfree_marker(vnode_t *vp)
397{ 397{
398 vnode_impl_t *vip; 398 vnode_impl_t *vip;
399 399
400 vip = VNODE_TO_VIMPL(vp); 400 vip = VNODE_TO_VIMPL(vp);
401 KASSERT(vip->vi_state == VS_MARKER); 401 KASSERT(vip->vi_state == VS_MARKER);
402 uvm_obj_destroy(&vp->v_uobj, true); 402 uvm_obj_destroy(&vp->v_uobj, true);
403 pool_cache_put(vcache_pool, vip); 403 pool_cache_put(vcache_pool, vip);
404} 404}
405 405
406/* 406/*
407 * Test a vnode for being a marker vnode. 407 * Test a vnode for being a marker vnode.
408 */ 408 */
409bool 409bool
410vnis_marker(vnode_t *vp) 410vnis_marker(vnode_t *vp)
411{ 411{
412 412
413 return (VNODE_TO_VIMPL(vp)->vi_state == VS_MARKER); 413 return (VNODE_TO_VIMPL(vp)->vi_state == VS_MARKER);
414} 414}
415 415
416/* 416/*
417 * Return the lru list this node should be on. 417 * Return the lru list this node should be on.
418 */ 418 */
419static vnodelst_t * 419static vnodelst_t *
420lru_which(vnode_t *vp) 420lru_which(vnode_t *vp)
421{ 421{
422 422
423 KASSERT(mutex_owned(vp->v_interlock)); 423 KASSERT(mutex_owned(vp->v_interlock));
424 424
425 if (vp->v_holdcnt > 0) 425 if (vp->v_holdcnt > 0)
426 return &lru_hold_list; 426 return &lru_hold_list;
427 else 427 else
428 return &lru_free_list; 428 return &lru_free_list;
429} 429}
430 430
431/* 431/*
432 * Put vnode to end of given list. 432 * Put vnode to end of given list.
433 * Both the current and the new list may be NULL, used on vnode alloc/free. 433 * Both the current and the new list may be NULL, used on vnode alloc/free.
434 * Adjust numvnodes and signal vdrain thread if there is work. 434 * Adjust numvnodes and signal vdrain thread if there is work.
435 */ 435 */
436static void 436static void
437lru_requeue(vnode_t *vp, vnodelst_t *listhd) 437lru_requeue(vnode_t *vp, vnodelst_t *listhd)
438{ 438{
439 vnode_impl_t *vip; 439 vnode_impl_t *vip;
440 440
441 mutex_enter(&vdrain_lock); 441 mutex_enter(&vdrain_lock);
442 vip = VNODE_TO_VIMPL(vp); 442 vip = VNODE_TO_VIMPL(vp);
443 if (vip->vi_lrulisthd != NULL) 443 if (vip->vi_lrulisthd != NULL)
444 TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist); 444 TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
445 else 445 else
446 numvnodes++; 446 numvnodes++;
447 vip->vi_lrulisthd = listhd; 447 vip->vi_lrulisthd = listhd;
448 if (vip->vi_lrulisthd != NULL) 448 if (vip->vi_lrulisthd != NULL)
449 TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist); 449 TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
450 else 450 else
451 numvnodes--; 451 numvnodes--;
452 if (numvnodes > desiredvnodes || listhd == &lru_vrele_list) 452 if (numvnodes > desiredvnodes || listhd == &lru_vrele_list)
453 cv_broadcast(&vdrain_cv); 453 cv_broadcast(&vdrain_cv);
454 mutex_exit(&vdrain_lock); 454 mutex_exit(&vdrain_lock);
455} 455}
456 456
457/* 457/*
458 * Release deferred vrele vnodes for this mount. 458 * Release deferred vrele vnodes for this mount.
459 * Called with file system suspended. 459 * Called with file system suspended.
460 */ 460 */
461void 461void
462vrele_flush(struct mount *mp) 462vrele_flush(struct mount *mp)
463{ 463{
464 vnode_impl_t *vip, *marker; 464 vnode_impl_t *vip, *marker;
465 465
466 KASSERT(fstrans_is_owner(mp)); 466 KASSERT(fstrans_is_owner(mp));
467 467
468 marker = VNODE_TO_VIMPL(vnalloc_marker(NULL)); 468 marker = VNODE_TO_VIMPL(vnalloc_marker(NULL));
469 469
470 mutex_enter(&vdrain_lock); 470 mutex_enter(&vdrain_lock);
471 TAILQ_INSERT_HEAD(&lru_vrele_list, marker, vi_lrulist); 471 TAILQ_INSERT_HEAD(&lru_vrele_list, marker, vi_lrulist);
472 472
473 while ((vip = TAILQ_NEXT(marker, vi_lrulist))) { 473 while ((vip = TAILQ_NEXT(marker, vi_lrulist))) {
474 TAILQ_REMOVE(&lru_vrele_list, marker, vi_lrulist); 474 TAILQ_REMOVE(&lru_vrele_list, marker, vi_lrulist);
475 TAILQ_INSERT_AFTER(&lru_vrele_list, vip, marker, vi_lrulist); 475 TAILQ_INSERT_AFTER(&lru_vrele_list, vip, marker, vi_lrulist);
476 if (vnis_marker(VIMPL_TO_VNODE(vip))) 476 if (vnis_marker(VIMPL_TO_VNODE(vip)))
477 continue; 477 continue;
478 478
479 KASSERT(vip->vi_lrulisthd == &lru_vrele_list); 479 KASSERT(vip->vi_lrulisthd == &lru_vrele_list);
480 TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist); 480 TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
481 vip->vi_lrulisthd = &lru_hold_list; 481 vip->vi_lrulisthd = &lru_hold_list;
482 TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist); 482 TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
483 mutex_exit(&vdrain_lock); 483 mutex_exit(&vdrain_lock);
484 484
485 vrele(VIMPL_TO_VNODE(vip)); 485 vrele(VIMPL_TO_VNODE(vip));
486 486
487 mutex_enter(&vdrain_lock); 487 mutex_enter(&vdrain_lock);
488 } 488 }
489 489
490 TAILQ_REMOVE(&lru_vrele_list, marker, vi_lrulist); 490 TAILQ_REMOVE(&lru_vrele_list, marker, vi_lrulist);
491 mutex_exit(&vdrain_lock); 491 mutex_exit(&vdrain_lock);
492 492
493 vnfree_marker(VIMPL_TO_VNODE(marker)); 493 vnfree_marker(VIMPL_TO_VNODE(marker));
494} 494}
495 495
496/* 496/*
497 * Reclaim a cached vnode. Used from vdrain_thread only. 497 * Reclaim a cached vnode. Used from vdrain_thread only.
498 */ 498 */
499static __inline void 499static __inline void
500vdrain_remove(vnode_t *vp) 500vdrain_remove(vnode_t *vp)
501{ 501{
502 struct mount *mp; 502 struct mount *mp;
503 503
504 KASSERT(mutex_owned(&vdrain_lock)); 504 KASSERT(mutex_owned(&vdrain_lock));
505 505
506 /* Probe usecount (unlocked). */ 506 /* Probe usecount (unlocked). */
507 if (vp->v_usecount > 0) 507 if (vp->v_usecount > 0)
508 return; 508 return;
509 /* Try v_interlock -- we lock the wrong direction! */ 509 /* Try v_interlock -- we lock the wrong direction! */
510 if (!mutex_tryenter(vp->v_interlock)) 510 if (!mutex_tryenter(vp->v_interlock))
511 return; 511 return;
512 /* Probe usecount and state. */ 512 /* Probe usecount and state. */
513 if (vp->v_usecount > 0 || VSTATE_GET(vp) != VS_ACTIVE) { 513 if (vp->v_usecount > 0 || VSTATE_GET(vp) != VS_ACTIVE) {
514 mutex_exit(vp->v_interlock); 514 mutex_exit(vp->v_interlock);
515 return; 515 return;
516 } 516 }
517 mp = vp->v_mount; 517 mp = vp->v_mount;
518 if (fstrans_start_nowait(mp, FSTRANS_SHARED) != 0) { 518 if (fstrans_start_nowait(mp, FSTRANS_SHARED) != 0) {
519 mutex_exit(vp->v_interlock); 519 mutex_exit(vp->v_interlock);
520 return; 520 return;
521 } 521 }
522 vdrain_retry = true; 522 vdrain_retry = true;
523 mutex_exit(&vdrain_lock); 523 mutex_exit(&vdrain_lock);
524 524
525 if (vcache_vget(vp) == 0) { 525 if (vcache_vget(vp) == 0) {
526 if (!vrecycle(vp)) 526 if (!vrecycle(vp))
527 vrele(vp); 527 vrele(vp);
528 } 528 }
529 fstrans_done(mp); 529 fstrans_done(mp);
530 530
531 mutex_enter(&vdrain_lock); 531 mutex_enter(&vdrain_lock);
532} 532}
533 533
534/* 534/*
535 * Release a cached vnode. Used from vdrain_thread only. 535 * Release a cached vnode. Used from vdrain_thread only.
536 */ 536 */
537static __inline void 537static __inline void
538vdrain_vrele(vnode_t *vp) 538vdrain_vrele(vnode_t *vp)
539{ 539{
540 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 540 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
541 struct mount *mp; 541 struct mount *mp;
542 542
543 KASSERT(mutex_owned(&vdrain_lock)); 543 KASSERT(mutex_owned(&vdrain_lock));
544 544
545 mp = vp->v_mount; 545 mp = vp->v_mount;
546 if (fstrans_start_nowait(mp, FSTRANS_LAZY) != 0) 546 if (fstrans_start_nowait(mp, FSTRANS_LAZY) != 0)
547 return; 547 return;
548 548
549 /* 549 /*
550 * First remove the vnode from the vrele list. 550 * First remove the vnode from the vrele list.
551 * Put it on the last lru list, the last vrele() 551 * Put it on the last lru list, the last vrele()
552 * will put it back onto the right list before 552 * will put it back onto the right list before
553 * its v_usecount reaches zero. 553 * its v_usecount reaches zero.
554 */ 554 */
555 KASSERT(vip->vi_lrulisthd == &lru_vrele_list); 555 KASSERT(vip->vi_lrulisthd == &lru_vrele_list);
556 TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist); 556 TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
557 vip->vi_lrulisthd = &lru_hold_list; 557 vip->vi_lrulisthd = &lru_hold_list;
558 TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist); 558 TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
559 559
560 vdrain_retry = true; 560 vdrain_retry = true;
561 mutex_exit(&vdrain_lock); 561 mutex_exit(&vdrain_lock);
562 562
563 mutex_enter(vp->v_interlock); 563 mutex_enter(vp->v_interlock);
564 vrelel(vp, 0); 564 vrelel(vp, 0);
565 fstrans_done(mp); 565 fstrans_done(mp);
566 566
567 mutex_enter(&vdrain_lock); 567 mutex_enter(&vdrain_lock);
568} 568}
569 569
570/* 570/*
571 * Helper thread to keep the number of vnodes below desiredvnodes 571 * Helper thread to keep the number of vnodes below desiredvnodes
572 * and release vnodes from asynchronous vrele. 572 * and release vnodes from asynchronous vrele.
573 */ 573 */
574static void 574static void
575vdrain_thread(void *cookie) 575vdrain_thread(void *cookie)
576{ 576{
577 vnodelst_t *listhd[] = { 577 vnodelst_t *listhd[] = {
578 &lru_vrele_list, &lru_free_list, &lru_hold_list 578 &lru_vrele_list, &lru_free_list, &lru_hold_list
579 }; 579 };
580 int i; 580 int i;
581 u_int target; 581 u_int target;
582 vnode_impl_t *vip, *marker; 582 vnode_impl_t *vip, *marker;
583 583
584 marker = VNODE_TO_VIMPL(vnalloc_marker(NULL)); 584 marker = VNODE_TO_VIMPL(vnalloc_marker(NULL));
585 585
586 mutex_enter(&vdrain_lock); 586 mutex_enter(&vdrain_lock);
587 587
588 for (;;) { 588 for (;;) {
589 vdrain_retry = false; 589 vdrain_retry = false;
590 target = desiredvnodes - desiredvnodes/10; 590 target = desiredvnodes - desiredvnodes/10;
591 591
592 for (i = 0; i < __arraycount(listhd); i++) { 592 for (i = 0; i < __arraycount(listhd); i++) {
593 TAILQ_INSERT_HEAD(listhd[i], marker, vi_lrulist); 593 TAILQ_INSERT_HEAD(listhd[i], marker, vi_lrulist);
594 while ((vip = TAILQ_NEXT(marker, vi_lrulist))) { 594 while ((vip = TAILQ_NEXT(marker, vi_lrulist))) {
595 TAILQ_REMOVE(listhd[i], marker, vi_lrulist); 595 TAILQ_REMOVE(listhd[i], marker, vi_lrulist);
596 TAILQ_INSERT_AFTER(listhd[i], vip, marker, 596 TAILQ_INSERT_AFTER(listhd[i], vip, marker,
597 vi_lrulist); 597 vi_lrulist);
598 if (vnis_marker(VIMPL_TO_VNODE(vip))) 598 if (vnis_marker(VIMPL_TO_VNODE(vip)))
599 continue; 599 continue;
600 if (listhd[i] == &lru_vrele_list) 600 if (listhd[i] == &lru_vrele_list)
601 vdrain_vrele(VIMPL_TO_VNODE(vip)); 601 vdrain_vrele(VIMPL_TO_VNODE(vip));
602 else if (numvnodes < target) 602 else if (numvnodes < target)
603 break; 603 break;
604 else 604 else
605 vdrain_remove(VIMPL_TO_VNODE(vip)); 605 vdrain_remove(VIMPL_TO_VNODE(vip));
606 } 606 }
607 TAILQ_REMOVE(listhd[i], marker, vi_lrulist); 607 TAILQ_REMOVE(listhd[i], marker, vi_lrulist);
608 } 608 }
609 609
610 if (vdrain_retry) { 610 if (vdrain_retry) {
611 mutex_exit(&vdrain_lock); 611 mutex_exit(&vdrain_lock);
612 yield(); 612 yield();
613 mutex_enter(&vdrain_lock); 613 mutex_enter(&vdrain_lock);
614 } else { 614 } else {
615 vdrain_gen++; 615 vdrain_gen++;
616 cv_broadcast(&vdrain_gen_cv); 616 cv_broadcast(&vdrain_gen_cv);
617 cv_wait(&vdrain_cv, &vdrain_lock); 617 cv_wait(&vdrain_cv, &vdrain_lock);
618 } 618 }
619 } 619 }
620} 620}
621 621
622/* 622/*
623 * vput: unlock and release the reference. 623 * vput: unlock and release the reference.
624 */ 624 */
625void 625void
626vput(vnode_t *vp) 626vput(vnode_t *vp)
627{ 627{
628 628
629 VOP_UNLOCK(vp); 629 VOP_UNLOCK(vp);
630 vrele(vp); 630 vrele(vp);
631} 631}
632 632
633/* 633/*
634 * Try to drop reference on a vnode. Abort if we are releasing the 634 * Try to drop reference on a vnode. Abort if we are releasing the
635 * last reference. Note: this _must_ succeed if not the last reference. 635 * last reference. Note: this _must_ succeed if not the last reference.
636 */ 636 */
637static inline bool 637static inline bool
638vtryrele(vnode_t *vp) 638vtryrele(vnode_t *vp)
639{ 639{
640 u_int use, next; 640 u_int use, next;
641 641
642 for (use = vp->v_usecount;; use = next) { 642 for (use = vp->v_usecount;; use = next) {
643 if (use == 1) { 643 if (use == 1) {
644 return false; 644 return false;
645 } 645 }
646 KASSERT(use > 1); 646 KASSERT(use > 1);
647 next = atomic_cas_uint(&vp->v_usecount, use, use - 1); 647 next = atomic_cas_uint(&vp->v_usecount, use, use - 1);
648 if (__predict_true(next == use)) { 648 if (__predict_true(next == use)) {
649 return true; 649 return true;
650 } 650 }
651 } 651 }
652} 652}
653 653
654/* 654/*
655 * Vnode release. If reference count drops to zero, call inactive 655 * Vnode release. If reference count drops to zero, call inactive
656 * routine and either return to freelist or free to the pool. 656 * routine and either return to freelist or free to the pool.
657 */ 657 */
658static void 658static void
659vrelel(vnode_t *vp, int flags) 659vrelel(vnode_t *vp, int flags)
660{ 660{
661 bool recycle, defer; 661 bool recycle, defer;
662 int error; 662 int error;
663 663
664 KASSERT(mutex_owned(vp->v_interlock)); 664 KASSERT(mutex_owned(vp->v_interlock));
665 665
666 if (__predict_false(vp->v_op == dead_vnodeop_p && 666 if (__predict_false(vp->v_op == dead_vnodeop_p &&
667 VSTATE_GET(vp) != VS_RECLAIMED)) { 667 VSTATE_GET(vp) != VS_RECLAIMED)) {
668 vnpanic(vp, "dead but not clean"); 668 vnpanic(vp, "dead but not clean");
669 } 669 }
670 670
671 /* 671 /*
672 * If not the last reference, just drop the reference count 672 * If not the last reference, just drop the reference count
673 * and unlock. 673 * and unlock.
674 */ 674 */
675 if (vtryrele(vp)) { 675 if (vtryrele(vp)) {
676 mutex_exit(vp->v_interlock); 676 mutex_exit(vp->v_interlock);
677 return; 677 return;
678 } 678 }
679 if (vp->v_usecount <= 0 || vp->v_writecount != 0) { 679 if (vp->v_usecount <= 0 || vp->v_writecount != 0) {
680 vnpanic(vp, "%s: bad ref count", __func__); 680 vnpanic(vp, "%s: bad ref count", __func__);
681 } 681 }
682 682
683#ifdef DIAGNOSTIC 683#ifdef DIAGNOSTIC
684 if ((vp->v_type == VBLK || vp->v_type == VCHR) && 684 if ((vp->v_type == VBLK || vp->v_type == VCHR) &&
685 vp->v_specnode != NULL && vp->v_specnode->sn_opencnt != 0) { 685 vp->v_specnode != NULL && vp->v_specnode->sn_opencnt != 0) {
686 vprint("vrelel: missing VOP_CLOSE()", vp); 686 vprint("vrelel: missing VOP_CLOSE()", vp);
687 } 687 }
688#endif 688#endif
689 689
690 /* 690 /*
691 * If not clean, deactivate the vnode, but preserve 691 * If not clean, deactivate the vnode, but preserve
692 * our reference across the call to VOP_INACTIVE(). 692 * our reference across the call to VOP_INACTIVE().
693 */ 693 */
694 if (VSTATE_GET(vp) != VS_RECLAIMED) { 694 if (VSTATE_GET(vp) != VS_RECLAIMED) {
695 recycle = false; 695 recycle = false;
696 696
697 /* 697 /*
698 * XXX This ugly block can be largely eliminated if 698 * XXX This ugly block can be largely eliminated if
699 * locking is pushed down into the file systems. 699 * locking is pushed down into the file systems.
700 * 700 *
701 * Defer vnode release to vdrain_thread if caller 701 * Defer vnode release to vdrain_thread if caller
702 * requests it explicitly or is the pagedaemon. 702 * requests it explicitly or is the pagedaemon.
703 */ 703 */
704 if ((curlwp == uvm.pagedaemon_lwp) || 704 if ((curlwp == uvm.pagedaemon_lwp) ||
705 (flags & VRELEL_ASYNC_RELE) != 0) { 705 (flags & VRELEL_ASYNC_RELE) != 0) {
706 defer = true; 706 defer = true;
707 } else if (curlwp == vdrain_lwp) { 707 } else if (curlwp == vdrain_lwp) {
708 /* 708 /*
709 * We have to try harder. 709 * We have to try harder.
710 */ 710 */
711 mutex_exit(vp->v_interlock); 711 mutex_exit(vp->v_interlock);
712 error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 712 error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
713 KASSERTMSG((error == 0), "vn_lock failed: %d", error); 713 KASSERTMSG((error == 0), "vn_lock failed: %d", error);
714 mutex_enter(vp->v_interlock); 714 mutex_enter(vp->v_interlock);
715 defer = false; 715 defer = false;
716 } else { 716 } else {
717 /* If we can't acquire the lock, then defer. */ 717 /* If we can't acquire the lock, then defer. */
718 mutex_exit(vp->v_interlock); 718 mutex_exit(vp->v_interlock);
719 error = vn_lock(vp, 719 error = vn_lock(vp,
720 LK_EXCLUSIVE | LK_RETRY | LK_NOWAIT); 720 LK_EXCLUSIVE | LK_RETRY | LK_NOWAIT);
721 defer = (error != 0); 721 defer = (error != 0);
722 mutex_enter(vp->v_interlock); 722 mutex_enter(vp->v_interlock);
723 } 723 }
724 724
725 KASSERT(mutex_owned(vp->v_interlock)); 725 KASSERT(mutex_owned(vp->v_interlock));
726 KASSERT(! (curlwp == vdrain_lwp && defer)); 726 KASSERT(! (curlwp == vdrain_lwp && defer));
727 727
728 if (defer) { 728 if (defer) {
729 /* 729 /*
730 * Defer reclaim to the kthread; it's not safe to 730 * Defer reclaim to the kthread; it's not safe to
731 * clean it here. We donate it our last reference. 731 * clean it here. We donate it our last reference.
732 */ 732 */
733 lru_requeue(vp, &lru_vrele_list); 733 lru_requeue(vp, &lru_vrele_list);
734 mutex_exit(vp->v_interlock); 734 mutex_exit(vp->v_interlock);
735 return; 735 return;
736 } 736 }
737 737
738 /* 738 /*
739 * If the node got another reference while we 739 * If the node got another reference while we
740 * released the interlock, don't try to inactivate it yet. 740 * released the interlock, don't try to inactivate it yet.
741 */ 741 */
742 if (__predict_false(vtryrele(vp))) { 742 if (__predict_false(vtryrele(vp))) {
743 VOP_UNLOCK(vp); 743 VOP_UNLOCK(vp);
744 mutex_exit(vp->v_interlock); 744 mutex_exit(vp->v_interlock);
745 return; 745 return;
746 } 746 }
747 VSTATE_CHANGE(vp, VS_ACTIVE, VS_BLOCKED); 747 VSTATE_CHANGE(vp, VS_ACTIVE, VS_BLOCKED);
748 mutex_exit(vp->v_interlock); 748 mutex_exit(vp->v_interlock);
749 749
750 /* 750 /*
751 * The vnode must not gain another reference while being 751 * The vnode must not gain another reference while being
752 * deactivated. If VOP_INACTIVE() indicates that 752 * deactivated. If VOP_INACTIVE() indicates that
753 * the described file has been deleted, then recycle 753 * the described file has been deleted, then recycle
754 * the vnode. 754 * the vnode.
755 * 755 *
756 * Note that VOP_INACTIVE() will drop the vnode lock. 756 * Note that VOP_INACTIVE() will drop the vnode lock.
757 */ 757 */
758 VOP_INACTIVE(vp, &recycle); 758 VOP_INACTIVE(vp, &recycle);
759 if (recycle) { 759 if (recycle) {
760 /* vcache_reclaim() below will drop the lock. */ 760 /* vcache_reclaim() below will drop the lock. */
761 if (vn_lock(vp, LK_EXCLUSIVE) != 0) 761 if (vn_lock(vp, LK_EXCLUSIVE) != 0)
762 recycle = false; 762 recycle = false;
763 } 763 }
764 mutex_enter(vp->v_interlock); 764 mutex_enter(vp->v_interlock);
765 VSTATE_CHANGE(vp, VS_BLOCKED, VS_ACTIVE); 765 VSTATE_CHANGE(vp, VS_BLOCKED, VS_ACTIVE);
766 if (!recycle) { 766 if (!recycle) {
767 if (vtryrele(vp)) { 767 if (vtryrele(vp)) {
768 mutex_exit(vp->v_interlock); 768 mutex_exit(vp->v_interlock);
769 return; 769 return;
770 } 770 }
771 } 771 }
772 772
773 /* Take care of space accounting. */ 773 /* Take care of space accounting. */
774 if (vp->v_iflag & VI_EXECMAP) { 774 if (vp->v_iflag & VI_EXECMAP) {
775 atomic_add_int(&uvmexp.execpages, 775 atomic_add_int(&uvmexp.execpages,
776 -vp->v_uobj.uo_npages); 776 -vp->v_uobj.uo_npages);
777 atomic_add_int(&uvmexp.filepages, 777 atomic_add_int(&uvmexp.filepages,
778 vp->v_uobj.uo_npages); 778 vp->v_uobj.uo_npages);
779 } 779 }
780 vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP|VI_WRMAP); 780 vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP|VI_WRMAP);
781 vp->v_vflag &= ~VV_MAPPED; 781 vp->v_vflag &= ~VV_MAPPED;
782 782
783 /* 783 /*
784 * Recycle the vnode if the file is now unused (unlinked), 784 * Recycle the vnode if the file is now unused (unlinked),
785 * otherwise just free it. 785 * otherwise just free it.
786 */ 786 */
787 if (recycle) { 787 if (recycle) {
788 VSTATE_ASSERT(vp, VS_ACTIVE); 788 VSTATE_ASSERT(vp, VS_ACTIVE);
789 vcache_reclaim(vp); 789 vcache_reclaim(vp);
790 } 790 }
791 KASSERT(vp->v_usecount > 0); 791 KASSERT(vp->v_usecount > 0);
792 } 792 }
793 793
794 if (atomic_dec_uint_nv(&vp->v_usecount) != 0) { 794 if (atomic_dec_uint_nv(&vp->v_usecount) != 0) {
795 /* Gained another reference while being reclaimed. */ 795 /* Gained another reference while being reclaimed. */
796 mutex_exit(vp->v_interlock); 796 mutex_exit(vp->v_interlock);
797 return; 797 return;
798 } 798 }
799 799
800 if (VSTATE_GET(vp) == VS_RECLAIMED && vp->v_holdcnt == 0) { 800 if (VSTATE_GET(vp) == VS_RECLAIMED && vp->v_holdcnt == 0) {
801 /* 801 /*
802 * It's clean so destroy it. It isn't referenced 802 * It's clean so destroy it. It isn't referenced
803 * anywhere since it has been reclaimed. 803 * anywhere since it has been reclaimed.
804 */ 804 */
805 vcache_free(VNODE_TO_VIMPL(vp)); 805 vcache_free(VNODE_TO_VIMPL(vp));
806 } else { 806 } else {
807 /* 807 /*
808 * Otherwise, put it back onto the freelist. It 808 * Otherwise, put it back onto the freelist. It
809 * can't be destroyed while still associated with 809 * can't be destroyed while still associated with
810 * a file system. 810 * a file system.
811 */ 811 */
812 lru_requeue(vp, lru_which(vp)); 812 lru_requeue(vp, lru_which(vp));
813 mutex_exit(vp->v_interlock); 813 mutex_exit(vp->v_interlock);
814 } 814 }
815} 815}
816 816
817void 817void
818vrele(vnode_t *vp) 818vrele(vnode_t *vp)
819{ 819{
820 820
821 if (vtryrele(vp)) { 821 if (vtryrele(vp)) {
822 return; 822 return;
823 } 823 }
824 mutex_enter(vp->v_interlock); 824 mutex_enter(vp->v_interlock);
825 vrelel(vp, 0); 825 vrelel(vp, 0);
826} 826}
827 827
828/* 828/*
829 * Asynchronous vnode release, vnode is released in different context. 829 * Asynchronous vnode release, vnode is released in different context.
830 */ 830 */
831void 831void
832vrele_async(vnode_t *vp) 832vrele_async(vnode_t *vp)
833{ 833{
834 834
835 if (vtryrele(vp)) { 835 if (vtryrele(vp)) {
836 return; 836 return;
837 } 837 }
838 mutex_enter(vp->v_interlock); 838 mutex_enter(vp->v_interlock);
839 vrelel(vp, VRELEL_ASYNC_RELE); 839 vrelel(vp, VRELEL_ASYNC_RELE);
840} 840}
841 841
842/* 842/*
843 * Vnode reference, where a reference is already held by some other 843 * Vnode reference, where a reference is already held by some other
844 * object (for example, a file structure). 844 * object (for example, a file structure).
845 */ 845 */
846void 846void
847vref(vnode_t *vp) 847vref(vnode_t *vp)
848{ 848{
849 849
850 KASSERT(vp->v_usecount != 0); 850 KASSERT(vp->v_usecount != 0);
851 851
852 atomic_inc_uint(&vp->v_usecount); 852 atomic_inc_uint(&vp->v_usecount);
853} 853}
854 854
855/* 855/*
856 * Page or buffer structure gets a reference. 856 * Page or buffer structure gets a reference.
857 * Called with v_interlock held. 857 * Called with v_interlock held.
858 */ 858 */
859void 859void
860vholdl(vnode_t *vp) 860vholdl(vnode_t *vp)
861{ 861{
862 862
863 KASSERT(mutex_owned(vp->v_interlock)); 863 KASSERT(mutex_owned(vp->v_interlock));
864 864
865 if (vp->v_holdcnt++ == 0 && vp->v_usecount == 0) 865 if (vp->v_holdcnt++ == 0 && vp->v_usecount == 0)
866 lru_requeue(vp, lru_which(vp)); 866 lru_requeue(vp, lru_which(vp));
867} 867}
868 868
869/* 869/*
870 * Page or buffer structure frees a reference. 870 * Page or buffer structure frees a reference.
871 * Called with v_interlock held. 871 * Called with v_interlock held.
872 */ 872 */
873void 873void
874holdrelel(vnode_t *vp) 874holdrelel(vnode_t *vp)
875{ 875{
876 876
877 KASSERT(mutex_owned(vp->v_interlock)); 877 KASSERT(mutex_owned(vp->v_interlock));
878 878
879 if (vp->v_holdcnt <= 0) { 879 if (vp->v_holdcnt <= 0) {
880 vnpanic(vp, "%s: holdcnt vp %p", __func__, vp); 880 vnpanic(vp, "%s: holdcnt vp %p", __func__, vp);
881 } 881 }
882 882
883 vp->v_holdcnt--; 883 vp->v_holdcnt--;
884 if (vp->v_holdcnt == 0 && vp->v_usecount == 0) 884 if (vp->v_holdcnt == 0 && vp->v_usecount == 0)
885 lru_requeue(vp, lru_which(vp)); 885 lru_requeue(vp, lru_which(vp));
886} 886}
887 887
888/* 888/*
889 * Recycle an unused vnode if caller holds the last reference. 889 * Recycle an unused vnode if caller holds the last reference.
890 */ 890 */
891bool 891bool
892vrecycle(vnode_t *vp) 892vrecycle(vnode_t *vp)
893{ 893{
894 int error __diagused; 894 int error __diagused;
895 895
896 mutex_enter(vp->v_interlock); 896 mutex_enter(vp->v_interlock);
897 897
898 /* Make sure we hold the last reference. */ 898 /* Make sure we hold the last reference. */
899 VSTATE_WAIT_STABLE(vp); 899 VSTATE_WAIT_STABLE(vp);
900 if (vp->v_usecount != 1) { 900 if (vp->v_usecount != 1) {
901 mutex_exit(vp->v_interlock); 901 mutex_exit(vp->v_interlock);
902 return false; 902 return false;
903 } 903 }
904 904
905 /* If the vnode is already clean we're done. */ 905 /* If the vnode is already clean we're done. */
906 if (VSTATE_GET(vp) != VS_ACTIVE) { 906 if (VSTATE_GET(vp) != VS_ACTIVE) {
907 VSTATE_ASSERT(vp, VS_RECLAIMED); 907 VSTATE_ASSERT(vp, VS_RECLAIMED);
908 vrelel(vp, 0); 908 vrelel(vp, 0);
909 return true; 909 return true;
910 } 910 }
911 911
912 /* Prevent further references until the vnode is locked. */ 912 /* Prevent further references until the vnode is locked. */
913 VSTATE_CHANGE(vp, VS_ACTIVE, VS_BLOCKED); 913 VSTATE_CHANGE(vp, VS_ACTIVE, VS_BLOCKED);
914 mutex_exit(vp->v_interlock); 914 mutex_exit(vp->v_interlock);
915 915
916 /* 916 /*
917 * On a leaf file system this lock will always succeed as we hold 917 * On a leaf file system this lock will always succeed as we hold
918 * the last reference and prevent further references. 918 * the last reference and prevent further references.
919 * On layered file systems waiting for the lock would open a can of 919 * On layered file systems waiting for the lock would open a can of
920 * deadlocks as the lower vnodes may have other active references. 920 * deadlocks as the lower vnodes may have other active references.
921 */ 921 */
922 error = vn_lock(vp, LK_EXCLUSIVE | LK_NOWAIT); 922 error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY | LK_NOWAIT);
923 923
924 mutex_enter(vp->v_interlock); 924 mutex_enter(vp->v_interlock);
925 VSTATE_CHANGE(vp, VS_BLOCKED, VS_ACTIVE); 925 VSTATE_CHANGE(vp, VS_BLOCKED, VS_ACTIVE);
926 926
927 if (error) { 927 if (error) {
928 mutex_exit(vp->v_interlock); 928 mutex_exit(vp->v_interlock);
929 return false; 929 return false;
930 } 930 }
931 931
932 KASSERT(vp->v_usecount == 1); 932 KASSERT(vp->v_usecount == 1);
933 vcache_reclaim(vp); 933 vcache_reclaim(vp);
934 vrelel(vp, 0); 934 vrelel(vp, 0);
935 935
936 return true; 936 return true;
937} 937}
938 938
939/* 939/*
940 * Eliminate all activity associated with the requested vnode 940 * Eliminate all activity associated with the requested vnode
941 * and with all vnodes aliased to the requested vnode. 941 * and with all vnodes aliased to the requested vnode.
942 */ 942 */
943void 943void
944vrevoke(vnode_t *vp) 944vrevoke(vnode_t *vp)
945{ 945{
946 vnode_t *vq; 946 vnode_t *vq;
947 enum vtype type; 947 enum vtype type;
948 dev_t dev; 948 dev_t dev;
949 949
950 KASSERT(vp->v_usecount > 0); 950 KASSERT(vp->v_usecount > 0);
951 951
952 mutex_enter(vp->v_interlock); 952 mutex_enter(vp->v_interlock);
953 VSTATE_WAIT_STABLE(vp); 953 VSTATE_WAIT_STABLE(vp);
954 if (VSTATE_GET(vp) == VS_RECLAIMED) { 954 if (VSTATE_GET(vp) == VS_RECLAIMED) {
955 mutex_exit(vp->v_interlock); 955 mutex_exit(vp->v_interlock);
956 return; 956 return;
957 } else if (vp->v_type != VBLK && vp->v_type != VCHR) { 957 } else if (vp->v_type != VBLK && vp->v_type != VCHR) {
958 atomic_inc_uint(&vp->v_usecount); 958 atomic_inc_uint(&vp->v_usecount);
959 mutex_exit(vp->v_interlock); 959 mutex_exit(vp->v_interlock);
960 vgone(vp); 960 vgone(vp);
961 return; 961 return;
962 } else { 962 } else {
963 dev = vp->v_rdev; 963 dev = vp->v_rdev;
964 type = vp->v_type; 964 type = vp->v_type;
965 mutex_exit(vp->v_interlock); 965 mutex_exit(vp->v_interlock);
966 } 966 }
967 967
968 while (spec_node_lookup_by_dev(type, dev, &vq) == 0) { 968 while (spec_node_lookup_by_dev(type, dev, &vq) == 0) {
969 vgone(vq); 969 vgone(vq);
970 } 970 }
971} 971}
972 972
973/* 973/*
974 * Eliminate all activity associated with a vnode in preparation for 974 * Eliminate all activity associated with a vnode in preparation for
975 * reuse. Drops a reference from the vnode. 975 * reuse. Drops a reference from the vnode.
976 */ 976 */
977void 977void
978vgone(vnode_t *vp) 978vgone(vnode_t *vp)
979{ 979{
980 980
981 if (vn_lock(vp, LK_EXCLUSIVE) != 0) { 981 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
982 VSTATE_ASSERT(vp, VS_RECLAIMED); 
983 vrele(vp); 
984 } 
985 
986 mutex_enter(vp->v_interlock); 982 mutex_enter(vp->v_interlock);
987 vcache_reclaim(vp); 983 VSTATE_WAIT_STABLE(vp);
 984 if (VSTATE_GET(vp) == VS_ACTIVE)
 985 vcache_reclaim(vp);
 986 VSTATE_ASSERT(vp, VS_RECLAIMED);
988 vrelel(vp, 0); 987 vrelel(vp, 0);
989} 988}
990 989
991static inline uint32_t 990static inline uint32_t
992vcache_hash(const struct vcache_key *key) 991vcache_hash(const struct vcache_key *key)
993{ 992{
994 uint32_t hash = HASH32_BUF_INIT; 993 uint32_t hash = HASH32_BUF_INIT;
995 994
996 hash = hash32_buf(&key->vk_mount, sizeof(struct mount *), hash); 995 hash = hash32_buf(&key->vk_mount, sizeof(struct mount *), hash);
997 hash = hash32_buf(key->vk_key, key->vk_key_len, hash); 996 hash = hash32_buf(key->vk_key, key->vk_key_len, hash);
998 return hash; 997 return hash;
999} 998}
1000 999
1001static void 1000static void
1002vcache_init(void) 1001vcache_init(void)
1003{ 1002{
1004 1003
1005 vcache_pool = pool_cache_init(sizeof(vnode_impl_t), 0, 0, 0, 1004 vcache_pool = pool_cache_init(sizeof(vnode_impl_t), 0, 0, 0,
1006 "vcachepl", NULL, IPL_NONE, NULL, NULL, NULL); 1005 "vcachepl", NULL, IPL_NONE, NULL, NULL, NULL);
1007 KASSERT(vcache_pool != NULL); 1006 KASSERT(vcache_pool != NULL);
1008 mutex_init(&vcache_lock, MUTEX_DEFAULT, IPL_NONE); 1007 mutex_init(&vcache_lock, MUTEX_DEFAULT, IPL_NONE);
1009 cv_init(&vcache_cv, "vcache"); 1008 cv_init(&vcache_cv, "vcache");
1010 vcache_hashsize = desiredvnodes; 1009 vcache_hashsize = desiredvnodes;
1011 vcache_hashtab = hashinit(desiredvnodes, HASH_SLIST, true, 1010 vcache_hashtab = hashinit(desiredvnodes, HASH_SLIST, true,
1012 &vcache_hashmask); 1011 &vcache_hashmask);
1013} 1012}
1014 1013
1015static void 1014static void
1016vcache_reinit(void) 1015vcache_reinit(void)
1017{ 1016{
1018 int i; 1017 int i;
1019 uint32_t hash; 1018 uint32_t hash;
1020 u_long oldmask, newmask; 1019 u_long oldmask, newmask;
1021 struct hashhead *oldtab, *newtab; 1020 struct hashhead *oldtab, *newtab;
1022 vnode_impl_t *vip; 1021 vnode_impl_t *vip;
1023 1022
1024 newtab = hashinit(desiredvnodes, HASH_SLIST, true, &newmask); 1023 newtab = hashinit(desiredvnodes, HASH_SLIST, true, &newmask);
1025 mutex_enter(&vcache_lock); 1024 mutex_enter(&vcache_lock);
1026 oldtab = vcache_hashtab; 1025 oldtab = vcache_hashtab;
1027 oldmask = vcache_hashmask; 1026 oldmask = vcache_hashmask;
1028 vcache_hashsize = desiredvnodes; 1027 vcache_hashsize = desiredvnodes;
1029 vcache_hashtab = newtab; 1028 vcache_hashtab = newtab;
1030 vcache_hashmask = newmask; 1029 vcache_hashmask = newmask;
1031 for (i = 0; i <= oldmask; i++) { 1030 for (i = 0; i <= oldmask; i++) {
1032 while ((vip = SLIST_FIRST(&oldtab[i])) != NULL) { 1031 while ((vip = SLIST_FIRST(&oldtab[i])) != NULL) {
1033 SLIST_REMOVE(&oldtab[i], vip, vnode_impl, vi_hash); 1032 SLIST_REMOVE(&oldtab[i], vip, vnode_impl, vi_hash);
1034 hash = vcache_hash(&vip->vi_key); 1033 hash = vcache_hash(&vip->vi_key);
1035 SLIST_INSERT_HEAD(&newtab[hash & vcache_hashmask], 1034 SLIST_INSERT_HEAD(&newtab[hash & vcache_hashmask],
1036 vip, vi_hash); 1035 vip, vi_hash);
1037 } 1036 }
1038 } 1037 }
1039 mutex_exit(&vcache_lock); 1038 mutex_exit(&vcache_lock);
1040 hashdone(oldtab, HASH_SLIST, oldmask); 1039 hashdone(oldtab, HASH_SLIST, oldmask);
1041} 1040}
1042 1041
1043static inline vnode_impl_t * 1042static inline vnode_impl_t *
1044vcache_hash_lookup(const struct vcache_key *key, uint32_t hash) 1043vcache_hash_lookup(const struct vcache_key *key, uint32_t hash)
1045{ 1044{
1046 struct hashhead *hashp; 1045 struct hashhead *hashp;
1047 vnode_impl_t *vip; 1046 vnode_impl_t *vip;
1048 1047
1049 KASSERT(mutex_owned(&vcache_lock)); 1048 KASSERT(mutex_owned(&vcache_lock));
1050 1049
1051 hashp = &vcache_hashtab[hash & vcache_hashmask]; 1050 hashp = &vcache_hashtab[hash & vcache_hashmask];
1052 SLIST_FOREACH(vip, hashp, vi_hash) { 1051 SLIST_FOREACH(vip, hashp, vi_hash) {
1053 if (key->vk_mount != vip->vi_key.vk_mount) 1052 if (key->vk_mount != vip->vi_key.vk_mount)
1054 continue; 1053 continue;
1055 if (key->vk_key_len != vip->vi_key.vk_key_len) 1054 if (key->vk_key_len != vip->vi_key.vk_key_len)
1056 continue; 1055 continue;
1057 if (memcmp(key->vk_key, vip->vi_key.vk_key, key->vk_key_len)) 1056 if (memcmp(key->vk_key, vip->vi_key.vk_key, key->vk_key_len))
1058 continue; 1057 continue;
1059 return vip; 1058 return vip;
1060 } 1059 }
1061 return NULL; 1060 return NULL;
1062} 1061}
1063 1062
1064/* 1063/*
1065 * Allocate a new, uninitialized vcache node. 1064 * Allocate a new, uninitialized vcache node.
1066 */ 1065 */
1067static vnode_impl_t * 1066static vnode_impl_t *
1068vcache_alloc(void) 1067vcache_alloc(void)
1069{ 1068{
1070 vnode_impl_t *vip; 1069 vnode_impl_t *vip;
1071 vnode_t *vp; 1070 vnode_t *vp;
1072 1071
1073 vip = pool_cache_get(vcache_pool, PR_WAITOK); 1072 vip = pool_cache_get(vcache_pool, PR_WAITOK);
1074 memset(vip, 0, sizeof(*vip)); 1073 memset(vip, 0, sizeof(*vip));
1075 1074
1076 rw_init(&vip->vi_lock); 1075 rw_init(&vip->vi_lock);
1077 /* SLIST_INIT(&vip->vi_hash); */ 1076 /* SLIST_INIT(&vip->vi_hash); */
1078 /* LIST_INIT(&vip->vi_nclist); */ 1077 /* LIST_INIT(&vip->vi_nclist); */
1079 /* LIST_INIT(&vip->vi_dnclist); */ 1078 /* LIST_INIT(&vip->vi_dnclist); */
1080 1079
1081 vp = VIMPL_TO_VNODE(vip); 1080 vp = VIMPL_TO_VNODE(vip);
1082 uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 0); 1081 uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 0);
1083 cv_init(&vp->v_cv, "vnode"); 1082 cv_init(&vp->v_cv, "vnode");
1084 1083
1085 vp->v_usecount = 1; 1084 vp->v_usecount = 1;
1086 vp->v_type = VNON; 1085 vp->v_type = VNON;
1087 vp->v_size = vp->v_writesize = VSIZENOTSET; 1086 vp->v_size = vp->v_writesize = VSIZENOTSET;
1088 1087
1089 vip->vi_state = VS_LOADING; 1088 vip->vi_state = VS_LOADING;
1090 1089
1091 lru_requeue(vp, &lru_free_list); 1090 lru_requeue(vp, &lru_free_list);
1092 1091
1093 return vip; 1092 return vip;
1094} 1093}
1095 1094
1096/* 1095/*
1097 * Free an unused, unreferenced vcache node. 1096 * Free an unused, unreferenced vcache node.
1098 * v_interlock locked on entry. 1097 * v_interlock locked on entry.
1099 */ 1098 */
1100static void 1099static void
1101vcache_free(vnode_impl_t *vip) 1100vcache_free(vnode_impl_t *vip)
1102{ 1101{
1103 vnode_t *vp; 1102 vnode_t *vp;
1104 1103
1105 vp = VIMPL_TO_VNODE(vip); 1104 vp = VIMPL_TO_VNODE(vip);
1106 KASSERT(mutex_owned(vp->v_interlock)); 1105 KASSERT(mutex_owned(vp->v_interlock));
1107 1106
1108 KASSERT(vp->v_usecount == 0); 1107 KASSERT(vp->v_usecount == 0);
1109 KASSERT(vp->v_holdcnt == 0); 1108 KASSERT(vp->v_holdcnt == 0);
1110 KASSERT(vp->v_writecount == 0); 1109 KASSERT(vp->v_writecount == 0);
1111 lru_requeue(vp, NULL); 1110 lru_requeue(vp, NULL);
1112 mutex_exit(vp->v_interlock); 1111 mutex_exit(vp->v_interlock);
1113 1112
1114 vfs_insmntque(vp, NULL); 1113 vfs_insmntque(vp, NULL);
1115 if (vp->v_type == VBLK || vp->v_type == VCHR) 1114 if (vp->v_type == VBLK || vp->v_type == VCHR)
1116 spec_node_destroy(vp); 1115 spec_node_destroy(vp);
1117 1116
1118 rw_destroy(&vip->vi_lock); 1117 rw_destroy(&vip->vi_lock);
1119 uvm_obj_destroy(&vp->v_uobj, true); 1118 uvm_obj_destroy(&vp->v_uobj, true);
1120 cv_destroy(&vp->v_cv); 1119 cv_destroy(&vp->v_cv);
1121 pool_cache_put(vcache_pool, vip); 1120 pool_cache_put(vcache_pool, vip);
1122} 1121}
1123 1122
1124/* 1123/*
1125 * Try to get an initial reference on this cached vnode. 1124 * Try to get an initial reference on this cached vnode.
1126 * Returns zero on success, ENOENT if the vnode has been reclaimed and 1125 * Returns zero on success, ENOENT if the vnode has been reclaimed and
1127 * EBUSY if the vnode state is unstable. 1126 * EBUSY if the vnode state is unstable.
1128 * 1127 *
1129 * v_interlock locked on entry and unlocked on exit. 1128 * v_interlock locked on entry and unlocked on exit.
1130 */ 1129 */
1131int 1130int
1132vcache_tryvget(vnode_t *vp) 1131vcache_tryvget(vnode_t *vp)
1133{ 1132{
1134 int error = 0; 1133 int error = 0;
1135 1134
1136 KASSERT(mutex_owned(vp->v_interlock)); 1135 KASSERT(mutex_owned(vp->v_interlock));
1137 1136
1138 if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED)) 1137 if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED))
1139 error = ENOENT; 1138 error = ENOENT;
1140 else if (__predict_false(VSTATE_GET(vp) != VS_ACTIVE)) 1139 else if (__predict_false(VSTATE_GET(vp) != VS_ACTIVE))
1141 error = EBUSY; 1140 error = EBUSY;
1142 else if (vp->v_usecount == 0) 1141 else if (vp->v_usecount == 0)
1143 vp->v_usecount = 1; 1142 vp->v_usecount = 1;
1144 else 1143 else
1145 atomic_inc_uint(&vp->v_usecount); 1144 atomic_inc_uint(&vp->v_usecount);
1146 1145
1147 mutex_exit(vp->v_interlock); 1146 mutex_exit(vp->v_interlock);
1148 1147
1149 return error; 1148 return error;
1150} 1149}
1151 1150
1152/* 1151/*
1153 * Try to get an initial reference on this cached vnode. 1152 * Try to get an initial reference on this cached vnode.
1154 * Returns zero on success and ENOENT if the vnode has been reclaimed. 1153 * Returns zero on success and ENOENT if the vnode has been reclaimed.
1155 * Will wait for the vnode state to be stable. 1154 * Will wait for the vnode state to be stable.
1156 * 1155 *
1157 * v_interlock locked on entry and unlocked on exit. 1156 * v_interlock locked on entry and unlocked on exit.
1158 */ 1157 */
1159int 1158int
1160vcache_vget(vnode_t *vp) 1159vcache_vget(vnode_t *vp)
1161{ 1160{
1162 1161
1163 KASSERT(mutex_owned(vp->v_interlock)); 1162 KASSERT(mutex_owned(vp->v_interlock));
1164 1163
1165 /* Increment hold count to prevent vnode from disappearing. */ 1164 /* Increment hold count to prevent vnode from disappearing. */
1166 vp->v_holdcnt++; 1165 vp->v_holdcnt++;
1167 VSTATE_WAIT_STABLE(vp); 1166 VSTATE_WAIT_STABLE(vp);
1168 vp->v_holdcnt--; 1167 vp->v_holdcnt--;
1169 1168
1170 /* If this was the last reference to a reclaimed vnode free it now. */ 1169 /* If this was the last reference to a reclaimed vnode free it now. */
1171 if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED)) { 1170 if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED)) {
1172 if (vp->v_holdcnt == 0 && vp->v_usecount == 0) 1171 if (vp->v_holdcnt == 0 && vp->v_usecount == 0)
1173 vcache_free(VNODE_TO_VIMPL(vp)); 1172 vcache_free(VNODE_TO_VIMPL(vp));
1174 else 1173 else
1175 mutex_exit(vp->v_interlock); 1174 mutex_exit(vp->v_interlock);
1176 return ENOENT; 1175 return ENOENT;
1177 } 1176 }
1178 VSTATE_ASSERT(vp, VS_ACTIVE); 1177 VSTATE_ASSERT(vp, VS_ACTIVE);
1179 if (vp->v_usecount == 0) 1178 if (vp->v_usecount == 0)
1180 vp->v_usecount = 1; 1179 vp->v_usecount = 1;
1181 else 1180 else
1182 atomic_inc_uint(&vp->v_usecount); 1181 atomic_inc_uint(&vp->v_usecount);
1183 1182
1184 mutex_exit(vp->v_interlock); 1183 mutex_exit(vp->v_interlock);
1185 1184
1186 return 0; 1185 return 0;
1187} 1186}
1188 1187
1189/* 1188/*
1190 * Get a vnode / fs node pair by key and return it referenced through vpp. 1189 * Get a vnode / fs node pair by key and return it referenced through vpp.
1191 */ 1190 */
1192int 1191int
1193vcache_get(struct mount *mp, const void *key, size_t key_len, 1192vcache_get(struct mount *mp, const void *key, size_t key_len,
1194 struct vnode **vpp) 1193 struct vnode **vpp)
1195{ 1194{
1196 int error; 1195 int error;
1197 uint32_t hash; 1196 uint32_t hash;
1198 const void *new_key; 1197 const void *new_key;
1199 struct vnode *vp; 1198 struct vnode *vp;
1200 struct vcache_key vcache_key; 1199 struct vcache_key vcache_key;
1201 vnode_impl_t *vip, *new_vip; 1200 vnode_impl_t *vip, *new_vip;
1202 1201
1203 new_key = NULL; 1202 new_key = NULL;
1204 *vpp = NULL; 1203 *vpp = NULL;
1205 1204
1206 vcache_key.vk_mount = mp; 1205 vcache_key.vk_mount = mp;
1207 vcache_key.vk_key = key; 1206 vcache_key.vk_key = key;
1208 vcache_key.vk_key_len = key_len; 1207 vcache_key.vk_key_len = key_len;
1209 hash = vcache_hash(&vcache_key); 1208 hash = vcache_hash(&vcache_key);
1210 1209
1211again: 1210again:
1212 mutex_enter(&vcache_lock); 1211 mutex_enter(&vcache_lock);
1213 vip = vcache_hash_lookup(&vcache_key, hash); 1212 vip = vcache_hash_lookup(&vcache_key, hash);
1214 1213
1215 /* If found, take a reference or retry. */ 1214 /* If found, take a reference or retry. */
1216 if (__predict_true(vip != NULL)) { 1215 if (__predict_true(vip != NULL)) {
1217 /* 1216 /*
1218 * If the vnode is loading we cannot take the v_interlock 1217 * If the vnode is loading we cannot take the v_interlock
1219 * here as it might change during load (see uvm_obj_setlock()). 1218 * here as it might change during load (see uvm_obj_setlock()).
1220 * As changing state from VS_LOADING requires both vcache_lock 1219 * As changing state from VS_LOADING requires both vcache_lock
1221 * and v_interlock it is safe to test with vcache_lock held. 1220 * and v_interlock it is safe to test with vcache_lock held.
1222 * 1221 *
1223 * Wait for vnodes changing state from VS_LOADING and retry. 1222 * Wait for vnodes changing state from VS_LOADING and retry.
1224 */ 1223 */
1225 if (__predict_false(vip->vi_state == VS_LOADING)) { 1224 if (__predict_false(vip->vi_state == VS_LOADING)) {
1226 cv_wait(&vcache_cv, &vcache_lock); 1225 cv_wait(&vcache_cv, &vcache_lock);
1227 mutex_exit(&vcache_lock); 1226 mutex_exit(&vcache_lock);
1228 goto again; 1227 goto again;
1229 } 1228 }
1230 vp = VIMPL_TO_VNODE(vip); 1229 vp = VIMPL_TO_VNODE(vip);
1231 mutex_enter(vp->v_interlock); 1230 mutex_enter(vp->v_interlock);
1232 mutex_exit(&vcache_lock); 1231 mutex_exit(&vcache_lock);
1233 error = vcache_vget(vp); 1232 error = vcache_vget(vp);
1234 if (error == ENOENT) 1233 if (error == ENOENT)
1235 goto again; 1234 goto again;
1236 if (error == 0) 1235 if (error == 0)
1237 *vpp = vp; 1236 *vpp = vp;
1238 KASSERT((error != 0) == (*vpp == NULL)); 1237 KASSERT((error != 0) == (*vpp == NULL));
1239 return error; 1238 return error;
1240 } 1239 }
1241 mutex_exit(&vcache_lock); 1240 mutex_exit(&vcache_lock);
1242 1241
1243 /* Allocate and initialize a new vcache / vnode pair. */ 1242 /* Allocate and initialize a new vcache / vnode pair. */
1244 error = vfs_busy(mp, NULL); 1243 error = vfs_busy(mp, NULL);
1245 if (error) 1244 if (error)
1246 return error; 1245 return error;
1247 new_vip = vcache_alloc(); 1246 new_vip = vcache_alloc();
1248 new_vip->vi_key = vcache_key; 1247 new_vip->vi_key = vcache_key;
1249 vp = VIMPL_TO_VNODE(new_vip); 1248 vp = VIMPL_TO_VNODE(new_vip);
1250 mutex_enter(&vcache_lock); 1249 mutex_enter(&vcache_lock);
1251 vip = vcache_hash_lookup(&vcache_key, hash); 1250 vip = vcache_hash_lookup(&vcache_key, hash);
1252 if (vip == NULL) { 1251 if (vip == NULL) {
1253 SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask], 1252 SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask],
1254 new_vip, vi_hash); 1253 new_vip, vi_hash);
1255 vip = new_vip; 1254 vip = new_vip;
1256 } 1255 }
1257 1256
1258 /* If another thread beat us inserting this node, retry. */ 1257 /* If another thread beat us inserting this node, retry. */
1259 if (vip != new_vip) { 1258 if (vip != new_vip) {
1260 mutex_enter(vp->v_interlock); 1259 mutex_enter(vp->v_interlock);
1261 VSTATE_CHANGE(vp, VS_LOADING, VS_RECLAIMED); 1260 VSTATE_CHANGE(vp, VS_LOADING, VS_RECLAIMED);
1262 mutex_exit(&vcache_lock); 1261 mutex_exit(&vcache_lock);
1263 vrelel(vp, 0); 1262 vrelel(vp, 0);
1264 vfs_unbusy(mp, false, NULL); 1263 vfs_unbusy(mp, false, NULL);
1265 goto again; 1264 goto again;
1266 } 1265 }
1267 mutex_exit(&vcache_lock); 1266 mutex_exit(&vcache_lock);
1268 1267
1269 /* Load the fs node. Exclusive as new_node is VS_LOADING. */ 1268 /* Load the fs node. Exclusive as new_node is VS_LOADING. */
1270 error = VFS_LOADVNODE(mp, vp, key, key_len, &new_key); 1269 error = VFS_LOADVNODE(mp, vp, key, key_len, &new_key);
1271 if (error) { 1270 if (error) {
1272 mutex_enter(&vcache_lock); 1271 mutex_enter(&vcache_lock);
1273 SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask], 1272 SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
1274 new_vip, vnode_impl, vi_hash); 1273 new_vip, vnode_impl, vi_hash);
1275 mutex_enter(vp->v_interlock); 1274 mutex_enter(vp->v_interlock);
1276 VSTATE_CHANGE(vp, VS_LOADING, VS_RECLAIMED); 1275 VSTATE_CHANGE(vp, VS_LOADING, VS_RECLAIMED);
1277 mutex_exit(&vcache_lock); 1276 mutex_exit(&vcache_lock);
1278 vrelel(vp, 0); 1277 vrelel(vp, 0);
1279 vfs_unbusy(mp, false, NULL); 1278 vfs_unbusy(mp, false, NULL);
1280 KASSERT(*vpp == NULL); 1279 KASSERT(*vpp == NULL);
1281 return error; 1280 return error;
1282 } 1281 }
1283 KASSERT(new_key != NULL); 1282 KASSERT(new_key != NULL);
1284 KASSERT(memcmp(key, new_key, key_len) == 0); 1283 KASSERT(memcmp(key, new_key, key_len) == 0);
1285 KASSERT(vp->v_op != NULL); 1284 KASSERT(vp->v_op != NULL);
1286 vfs_insmntque(vp, mp); 1285 vfs_insmntque(vp, mp);
1287 if ((mp->mnt_iflag & IMNT_MPSAFE) != 0) 1286 if ((mp->mnt_iflag & IMNT_MPSAFE) != 0)
1288 vp->v_vflag |= VV_MPSAFE; 1287 vp->v_vflag |= VV_MPSAFE;
1289 vfs_unbusy(mp, true, NULL); 1288 vfs_unbusy(mp, true, NULL);
1290 1289
1291 /* Finished loading, finalize node. */ 1290 /* Finished loading, finalize node. */
1292 mutex_enter(&vcache_lock); 1291 mutex_enter(&vcache_lock);
1293 new_vip->vi_key.vk_key = new_key; 1292 new_vip->vi_key.vk_key = new_key;
1294 mutex_enter(vp->v_interlock); 1293 mutex_enter(vp->v_interlock);
1295 VSTATE_CHANGE(vp, VS_LOADING, VS_ACTIVE); 1294 VSTATE_CHANGE(vp, VS_LOADING, VS_ACTIVE);
1296 mutex_exit(vp->v_interlock); 1295 mutex_exit(vp->v_interlock);
1297 mutex_exit(&vcache_lock); 1296 mutex_exit(&vcache_lock);
1298 *vpp = vp; 1297 *vpp = vp;
1299 return 0; 1298 return 0;
1300} 1299}
1301 1300
1302/* 1301/*
1303 * Create a new vnode / fs node pair and return it referenced through vpp. 1302 * Create a new vnode / fs node pair and return it referenced through vpp.
1304 */ 1303 */
1305int 1304int
1306vcache_new(struct mount *mp, struct vnode *dvp, struct vattr *vap, 1305vcache_new(struct mount *mp, struct vnode *dvp, struct vattr *vap,
1307 kauth_cred_t cred, struct vnode **vpp) 1306 kauth_cred_t cred, struct vnode **vpp)
1308{ 1307{
1309 int error; 1308 int error;
1310 uint32_t hash; 1309 uint32_t hash;
1311 struct vnode *vp, *ovp; 1310 struct vnode *vp, *ovp;
1312 vnode_impl_t *vip, *ovip; 1311 vnode_impl_t *vip, *ovip;
1313 1312
1314 *vpp = NULL; 1313 *vpp = NULL;
1315 1314
1316 /* Allocate and initialize a new vcache / vnode pair. */ 1315 /* Allocate and initialize a new vcache / vnode pair. */
1317 error = vfs_busy(mp, NULL); 1316 error = vfs_busy(mp, NULL);
1318 if (error) 1317 if (error)
1319 return error; 1318 return error;
1320 vip = vcache_alloc(); 1319 vip = vcache_alloc();
1321 vip->vi_key.vk_mount = mp; 1320 vip->vi_key.vk_mount = mp;
1322 vp = VIMPL_TO_VNODE(vip); 1321 vp = VIMPL_TO_VNODE(vip);
1323 1322
1324 /* Create and load the fs node. */ 1323 /* Create and load the fs node. */
1325 error = VFS_NEWVNODE(mp, dvp, vp, vap, cred, 1324 error = VFS_NEWVNODE(mp, dvp, vp, vap, cred,
1326 &vip->vi_key.vk_key_len, &vip->vi_key.vk_key); 1325 &vip->vi_key.vk_key_len, &vip->vi_key.vk_key);
1327 if (error) { 1326 if (error) {
1328 mutex_enter(&vcache_lock); 1327 mutex_enter(&vcache_lock);
1329 mutex_enter(vp->v_interlock); 1328 mutex_enter(vp->v_interlock);
1330 VSTATE_CHANGE(vp, VS_LOADING, VS_RECLAIMED); 1329 VSTATE_CHANGE(vp, VS_LOADING, VS_RECLAIMED);
1331 mutex_exit(&vcache_lock); 1330 mutex_exit(&vcache_lock);
1332 vrelel(vp, 0); 1331 vrelel(vp, 0);
1333 vfs_unbusy(mp, false, NULL); 1332 vfs_unbusy(mp, false, NULL);
1334 KASSERT(*vpp == NULL); 1333 KASSERT(*vpp == NULL);
1335 return error; 1334 return error;
1336 } 1335 }
1337 KASSERT(vip->vi_key.vk_key != NULL); 1336 KASSERT(vip->vi_key.vk_key != NULL);
1338 KASSERT(vp->v_op != NULL); 1337 KASSERT(vp->v_op != NULL);
1339 hash = vcache_hash(&vip->vi_key); 1338 hash = vcache_hash(&vip->vi_key);
1340 1339
1341 /* Wait for previous instance to be reclaimed, then insert new node. */ 1340 /* Wait for previous instance to be reclaimed, then insert new node. */
1342 mutex_enter(&vcache_lock); 1341 mutex_enter(&vcache_lock);
1343 while ((ovip = vcache_hash_lookup(&vip->vi_key, hash))) { 1342 while ((ovip = vcache_hash_lookup(&vip->vi_key, hash))) {
1344 ovp = VIMPL_TO_VNODE(ovip); 1343 ovp = VIMPL_TO_VNODE(ovip);
1345 mutex_enter(ovp->v_interlock); 1344 mutex_enter(ovp->v_interlock);
1346 mutex_exit(&vcache_lock); 1345 mutex_exit(&vcache_lock);
1347 error = vcache_vget(ovp); 1346 error = vcache_vget(ovp);
1348 KASSERT(error == ENOENT); 1347 KASSERT(error == ENOENT);
1349 mutex_enter(&vcache_lock); 1348 mutex_enter(&vcache_lock);
1350 } 1349 }
1351 SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask], 1350 SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask],
1352 vip, vi_hash); 1351 vip, vi_hash);
1353 mutex_exit(&vcache_lock); 1352 mutex_exit(&vcache_lock);
1354 vfs_insmntque(vp, mp); 1353 vfs_insmntque(vp, mp);
1355 if ((mp->mnt_iflag & IMNT_MPSAFE) != 0) 1354 if ((mp->mnt_iflag & IMNT_MPSAFE) != 0)
1356 vp->v_vflag |= VV_MPSAFE; 1355 vp->v_vflag |= VV_MPSAFE;
1357 vfs_unbusy(mp, true, NULL); 1356 vfs_unbusy(mp, true, NULL);
1358 1357
1359 /* Finished loading, finalize node. */ 1358 /* Finished loading, finalize node. */
1360 mutex_enter(&vcache_lock); 1359 mutex_enter(&vcache_lock);
1361 mutex_enter(vp->v_interlock); 1360 mutex_enter(vp->v_interlock);
1362 VSTATE_CHANGE(vp, VS_LOADING, VS_ACTIVE); 1361 VSTATE_CHANGE(vp, VS_LOADING, VS_ACTIVE);
1363 mutex_exit(&vcache_lock); 1362 mutex_exit(&vcache_lock);
1364 mutex_exit(vp->v_interlock); 1363 mutex_exit(vp->v_interlock);
1365 *vpp = vp; 1364 *vpp = vp;
1366 return 0; 1365 return 0;
1367} 1366}
1368 1367
1369/* 1368/*
1370 * Prepare key change: update old cache nodes key and lock new cache node. 1369 * Prepare key change: update old cache nodes key and lock new cache node.
1371 * Return an error if the new node already exists. 1370 * Return an error if the new node already exists.
1372 */ 1371 */
1373int 1372int
1374vcache_rekey_enter(struct mount *mp, struct vnode *vp, 1373vcache_rekey_enter(struct mount *mp, struct vnode *vp,
1375 const void *old_key, size_t old_key_len, 1374 const void *old_key, size_t old_key_len,
1376 const void *new_key, size_t new_key_len) 1375 const void *new_key, size_t new_key_len)
1377{ 1376{
1378 uint32_t old_hash, new_hash; 1377 uint32_t old_hash, new_hash;
1379 struct vcache_key old_vcache_key, new_vcache_key; 1378 struct vcache_key old_vcache_key, new_vcache_key;
1380 vnode_impl_t *vip, *new_vip; 1379 vnode_impl_t *vip, *new_vip;
1381 struct vnode *new_vp; 1380 struct vnode *new_vp;
1382 1381
1383 old_vcache_key.vk_mount = mp; 1382 old_vcache_key.vk_mount = mp;
1384 old_vcache_key.vk_key = old_key; 1383 old_vcache_key.vk_key = old_key;
1385 old_vcache_key.vk_key_len = old_key_len; 1384 old_vcache_key.vk_key_len = old_key_len;
1386 old_hash = vcache_hash(&old_vcache_key); 1385 old_hash = vcache_hash(&old_vcache_key);
1387 1386
1388 new_vcache_key.vk_mount = mp; 1387 new_vcache_key.vk_mount = mp;
1389 new_vcache_key.vk_key = new_key; 1388 new_vcache_key.vk_key = new_key;
1390 new_vcache_key.vk_key_len = new_key_len; 1389 new_vcache_key.vk_key_len = new_key_len;
1391 new_hash = vcache_hash(&new_vcache_key); 1390 new_hash = vcache_hash(&new_vcache_key);
1392 1391
1393 new_vip = vcache_alloc(); 1392 new_vip = vcache_alloc();
1394 new_vip->vi_key = new_vcache_key; 1393 new_vip->vi_key = new_vcache_key;
1395 new_vp = VIMPL_TO_VNODE(new_vip); 1394 new_vp = VIMPL_TO_VNODE(new_vip);
1396 1395
1397 /* Insert locked new node used as placeholder. */ 1396 /* Insert locked new node used as placeholder. */
1398 mutex_enter(&vcache_lock); 1397 mutex_enter(&vcache_lock);
1399 vip = vcache_hash_lookup(&new_vcache_key, new_hash); 1398 vip = vcache_hash_lookup(&new_vcache_key, new_hash);
1400 if (vip != NULL) { 1399 if (vip != NULL) {
1401 mutex_enter(new_vp->v_interlock); 1400 mutex_enter(new_vp->v_interlock);
1402 VSTATE_CHANGE(new_vp, VS_LOADING, VS_RECLAIMED); 1401 VSTATE_CHANGE(new_vp, VS_LOADING, VS_RECLAIMED);
1403 mutex_exit(&vcache_lock); 1402 mutex_exit(&vcache_lock);
1404 vrelel(new_vp, 0); 1403 vrelel(new_vp, 0);
1405 return EEXIST; 1404 return EEXIST;
1406 } 1405 }
1407 SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask], 1406 SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask],
1408 new_vip, vi_hash); 1407 new_vip, vi_hash);
1409 1408
1410 /* Replace old nodes key with the temporary copy. */ 1409 /* Replace old nodes key with the temporary copy. */
1411 vip = vcache_hash_lookup(&old_vcache_key, old_hash); 1410 vip = vcache_hash_lookup(&old_vcache_key, old_hash);
1412 KASSERT(vip != NULL); 1411 KASSERT(vip != NULL);
1413 KASSERT(VIMPL_TO_VNODE(vip) == vp); 1412 KASSERT(VIMPL_TO_VNODE(vip) == vp);
1414 KASSERT(vip->vi_key.vk_key != old_vcache_key.vk_key); 1413 KASSERT(vip->vi_key.vk_key != old_vcache_key.vk_key);
1415 vip->vi_key = old_vcache_key; 1414 vip->vi_key = old_vcache_key;
1416 mutex_exit(&vcache_lock); 1415 mutex_exit(&vcache_lock);
1417 return 0; 1416 return 0;
1418} 1417}
1419 1418
1420/* 1419/*
1421 * Key change complete: update old node and remove placeholder. 1420 * Key change complete: update old node and remove placeholder.
1422 */ 1421 */
1423void 1422void
1424vcache_rekey_exit(struct mount *mp, struct vnode *vp, 1423vcache_rekey_exit(struct mount *mp, struct vnode *vp,
1425 const void *old_key, size_t old_key_len, 1424 const void *old_key, size_t old_key_len,
1426 const void *new_key, size_t new_key_len) 1425 const void *new_key, size_t new_key_len)
1427{ 1426{
1428 uint32_t old_hash, new_hash; 1427 uint32_t old_hash, new_hash;
1429 struct vcache_key old_vcache_key, new_vcache_key; 1428 struct vcache_key old_vcache_key, new_vcache_key;
1430 vnode_impl_t *vip, *new_vip; 1429 vnode_impl_t *vip, *new_vip;
1431 struct vnode *new_vp; 1430 struct vnode *new_vp;
1432 1431
1433 old_vcache_key.vk_mount = mp; 1432 old_vcache_key.vk_mount = mp;
1434 old_vcache_key.vk_key = old_key; 1433 old_vcache_key.vk_key = old_key;
1435 old_vcache_key.vk_key_len = old_key_len; 1434 old_vcache_key.vk_key_len = old_key_len;
1436 old_hash = vcache_hash(&old_vcache_key); 1435 old_hash = vcache_hash(&old_vcache_key);
1437 1436
1438 new_vcache_key.vk_mount = mp; 1437 new_vcache_key.vk_mount = mp;
1439 new_vcache_key.vk_key = new_key; 1438 new_vcache_key.vk_key = new_key;
1440 new_vcache_key.vk_key_len = new_key_len; 1439 new_vcache_key.vk_key_len = new_key_len;
1441 new_hash = vcache_hash(&new_vcache_key); 1440 new_hash = vcache_hash(&new_vcache_key);
1442 1441
1443 mutex_enter(&vcache_lock); 1442 mutex_enter(&vcache_lock);
1444 1443
1445 /* Lookup old and new node. */ 1444 /* Lookup old and new node. */
1446 vip = vcache_hash_lookup(&old_vcache_key, old_hash); 1445 vip = vcache_hash_lookup(&old_vcache_key, old_hash);
1447 KASSERT(vip != NULL); 1446 KASSERT(vip != NULL);
1448 KASSERT(VIMPL_TO_VNODE(vip) == vp); 1447 KASSERT(VIMPL_TO_VNODE(vip) == vp);
1449 1448
1450 new_vip = vcache_hash_lookup(&new_vcache_key, new_hash); 1449 new_vip = vcache_hash_lookup(&new_vcache_key, new_hash);
1451 KASSERT(new_vip != NULL); 1450 KASSERT(new_vip != NULL);
1452 KASSERT(new_vip->vi_key.vk_key_len == new_key_len); 1451 KASSERT(new_vip->vi_key.vk_key_len == new_key_len);
1453 new_vp = VIMPL_TO_VNODE(new_vip); 1452 new_vp = VIMPL_TO_VNODE(new_vip);
1454 mutex_enter(new_vp->v_interlock); 1453 mutex_enter(new_vp->v_interlock);
1455 VSTATE_ASSERT(VIMPL_TO_VNODE(new_vip), VS_LOADING); 1454 VSTATE_ASSERT(VIMPL_TO_VNODE(new_vip), VS_LOADING);
1456 1455
1457 /* Rekey old node and put it onto its new hashlist. */ 1456 /* Rekey old node and put it onto its new hashlist. */
1458 vip->vi_key = new_vcache_key; 1457 vip->vi_key = new_vcache_key;
1459 if (old_hash != new_hash) { 1458 if (old_hash != new_hash) {
1460 SLIST_REMOVE(&vcache_hashtab[old_hash & vcache_hashmask], 1459 SLIST_REMOVE(&vcache_hashtab[old_hash & vcache_hashmask],
1461 vip, vnode_impl, vi_hash); 1460 vip, vnode_impl, vi_hash);
1462 SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask], 1461 SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask],
1463 vip, vi_hash); 1462 vip, vi_hash);
1464 } 1463 }
1465 1464
1466 /* Remove new node used as placeholder. */ 1465 /* Remove new node used as placeholder. */
1467 SLIST_REMOVE(&vcache_hashtab[new_hash & vcache_hashmask], 1466 SLIST_REMOVE(&vcache_hashtab[new_hash & vcache_hashmask],
1468 new_vip, vnode_impl, vi_hash); 1467 new_vip, vnode_impl, vi_hash);
1469 VSTATE_CHANGE(new_vp, VS_LOADING, VS_RECLAIMED); 1468 VSTATE_CHANGE(new_vp, VS_LOADING, VS_RECLAIMED);
1470 mutex_exit(&vcache_lock); 1469 mutex_exit(&vcache_lock);
1471 vrelel(new_vp, 0); 1470 vrelel(new_vp, 0);
1472} 1471}
1473 1472
1474/* 1473/*
1475 * Disassociate the underlying file system from a vnode. 1474 * Disassociate the underlying file system from a vnode.
1476 * 1475 *
1477 * Must be called with vnode locked and will return unlocked. 1476 * Must be called with vnode locked and will return unlocked.
1478 * Must be called with the interlock held, and will return with it held. 1477 * Must be called with the interlock held, and will return with it held.
1479 */ 1478 */
1480static void 1479static void
1481vcache_reclaim(vnode_t *vp) 1480vcache_reclaim(vnode_t *vp)
1482{ 1481{
1483 lwp_t *l = curlwp; 1482 lwp_t *l = curlwp;
1484 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 1483 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
1485 struct mount *mp = vp->v_mount; 1484 struct mount *mp = vp->v_mount;
1486 uint32_t hash; 1485 uint32_t hash;
1487 uint8_t temp_buf[64], *temp_key; 1486 uint8_t temp_buf[64], *temp_key;
1488 size_t temp_key_len; 1487 size_t temp_key_len;
1489 bool recycle, active; 1488 bool recycle, active;
1490 int error; 1489 int error;
1491 1490
1492 KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 || 1491 KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
1493 VOP_ISLOCKED(vp) == LK_EXCLUSIVE); 1492 VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
1494 KASSERT(mutex_owned(vp->v_interlock)); 1493 KASSERT(mutex_owned(vp->v_interlock));
1495 KASSERT(vp->v_usecount != 0); 1494 KASSERT(vp->v_usecount != 0);
1496 1495
1497 active = (vp->v_usecount > 1); 1496 active = (vp->v_usecount > 1);
1498 temp_key_len = vip->vi_key.vk_key_len; 1497 temp_key_len = vip->vi_key.vk_key_len;
1499 /* 1498 /*
1500 * Prevent the vnode from being recycled or brought into use 1499 * Prevent the vnode from being recycled or brought into use
1501 * while we clean it out. 1500 * while we clean it out.
1502 */ 1501 */
1503 VSTATE_CHANGE(vp, VS_ACTIVE, VS_RECLAIMING); 1502 VSTATE_CHANGE(vp, VS_ACTIVE, VS_RECLAIMING);
1504 if (vp->v_iflag & VI_EXECMAP) { 1503 if (vp->v_iflag & VI_EXECMAP) {
1505 atomic_add_int(&uvmexp.execpages, -vp->v_uobj.uo_npages); 1504 atomic_add_int(&uvmexp.execpages, -vp->v_uobj.uo_npages);
1506 atomic_add_int(&uvmexp.filepages, vp->v_uobj.uo_npages); 1505 atomic_add_int(&uvmexp.filepages, vp->v_uobj.uo_npages);
1507 } 1506 }
1508 vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP); 1507 vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP);
1509 mutex_exit(vp->v_interlock); 1508 mutex_exit(vp->v_interlock);
1510 1509
1511 /* Replace the vnode key with a temporary copy. */ 1510 /* Replace the vnode key with a temporary copy. */
1512 if (vip->vi_key.vk_key_len > sizeof(temp_buf)) { 1511 if (vip->vi_key.vk_key_len > sizeof(temp_buf)) {
1513 temp_key = kmem_alloc(temp_key_len, KM_SLEEP); 1512 temp_key = kmem_alloc(temp_key_len, KM_SLEEP);
1514 } else { 1513 } else {
1515 temp_key = temp_buf; 1514 temp_key = temp_buf;
1516 } 1515 }
1517 mutex_enter(&vcache_lock); 1516 mutex_enter(&vcache_lock);
1518 memcpy(temp_key, vip->vi_key.vk_key, temp_key_len); 1517 memcpy(temp_key, vip->vi_key.vk_key, temp_key_len);
1519 vip->vi_key.vk_key = temp_key; 1518 vip->vi_key.vk_key = temp_key;
1520 mutex_exit(&vcache_lock); 1519 mutex_exit(&vcache_lock);
1521 1520
1522 fstrans_start(mp, FSTRANS_LAZY); 1521 fstrans_start(mp, FSTRANS_LAZY);
1523 1522
1524 /* 1523 /*
1525 * Clean out any cached data associated with the vnode. 1524 * Clean out any cached data associated with the vnode.
1526 * If purging an active vnode, it must be closed and 1525 * If purging an active vnode, it must be closed and
1527 * deactivated before being reclaimed. 1526 * deactivated before being reclaimed.
1528 */ 1527 */
1529 error = vinvalbuf(vp, V_SAVE, NOCRED, l, 0, 0); 1528 error = vinvalbuf(vp, V_SAVE, NOCRED, l, 0, 0);
1530 if (error != 0) { 1529 if (error != 0) {
1531 if (wapbl_vphaswapbl(vp)) 1530 if (wapbl_vphaswapbl(vp))
1532 WAPBL_DISCARD(wapbl_vptomp(vp)); 1531 WAPBL_DISCARD(wapbl_vptomp(vp));
1533 error = vinvalbuf(vp, 0, NOCRED, l, 0, 0); 1532 error = vinvalbuf(vp, 0, NOCRED, l, 0, 0);
1534 } 1533 }
1535 KASSERTMSG((error == 0), "vinvalbuf failed: %d", error); 1534 KASSERTMSG((error == 0), "vinvalbuf failed: %d", error);
1536 KASSERT((vp->v_iflag & VI_ONWORKLST) == 0); 1535 KASSERT((vp->v_iflag & VI_ONWORKLST) == 0);
1537 if (active && (vp->v_type == VBLK || vp->v_type == VCHR)) { 1536 if (active && (vp->v_type == VBLK || vp->v_type == VCHR)) {
1538 spec_node_revoke(vp); 1537 spec_node_revoke(vp);
1539 } 1538 }
1540 1539
1541 /* 1540 /*
1542 * Disassociate the underlying file system from the vnode. 1541 * Disassociate the underlying file system from the vnode.
1543 * Note that the VOP_INACTIVE will unlock the vnode. 1542 * Note that the VOP_INACTIVE will unlock the vnode.
1544 */ 1543 */
1545 VOP_INACTIVE(vp, &recycle); 1544 VOP_INACTIVE(vp, &recycle);
1546 if (VOP_RECLAIM(vp)) { 1545 if (VOP_RECLAIM(vp)) {
1547 vnpanic(vp, "%s: cannot reclaim", __func__); 1546 vnpanic(vp, "%s: cannot reclaim", __func__);
1548 } 1547 }
1549 1548
1550 KASSERT(vp->v_data == NULL); 1549 KASSERT(vp->v_data == NULL);
1551 KASSERT(vp->v_uobj.uo_npages == 0); 1550 KASSERT(vp->v_uobj.uo_npages == 0);
1552 1551
1553 if (vp->v_type == VREG && vp->v_ractx != NULL) { 1552 if (vp->v_type == VREG && vp->v_ractx != NULL) {
1554 uvm_ra_freectx(vp->v_ractx); 1553 uvm_ra_freectx(vp->v_ractx);
1555 vp->v_ractx = NULL; 1554 vp->v_ractx = NULL;
1556 } 1555 }
1557 1556
1558 /* Purge name cache. */ 1557 /* Purge name cache. */
1559 cache_purge(vp); 1558 cache_purge(vp);
1560 1559
1561 /* Move to dead mount. */ 1560 /* Move to dead mount. */
1562 vp->v_vflag &= ~VV_ROOT; 1561 vp->v_vflag &= ~VV_ROOT;
1563 atomic_inc_uint(&dead_rootmount->mnt_refcnt); 1562 atomic_inc_uint(&dead_rootmount->mnt_refcnt);
1564 vfs_insmntque(vp, dead_rootmount); 1563 vfs_insmntque(vp, dead_rootmount);
1565 1564
1566 /* Remove from vnode cache. */ 1565 /* Remove from vnode cache. */
1567 hash = vcache_hash(&vip->vi_key); 1566 hash = vcache_hash(&vip->vi_key);
1568 mutex_enter(&vcache_lock); 1567 mutex_enter(&vcache_lock);
1569 KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash)); 1568 KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash));
1570 SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask], 1569 SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
1571 vip, vnode_impl, vi_hash); 1570 vip, vnode_impl, vi_hash);
1572 mutex_exit(&vcache_lock); 1571 mutex_exit(&vcache_lock);
1573 if (temp_key != temp_buf) 1572 if (temp_key != temp_buf)
1574 kmem_free(temp_key, temp_key_len); 1573 kmem_free(temp_key, temp_key_len);
1575 1574
1576 /* Done with purge, notify sleepers of the grim news. */ 1575 /* Done with purge, notify sleepers of the grim news. */
1577 mutex_enter(vp->v_interlock); 1576 mutex_enter(vp->v_interlock);
1578 vp->v_op = dead_vnodeop_p; 1577 vp->v_op = dead_vnodeop_p;
1579 vp->v_vflag |= VV_LOCKSWORK; 1578 vp->v_vflag |= VV_LOCKSWORK;
1580 VSTATE_CHANGE(vp, VS_RECLAIMING, VS_RECLAIMED); 1579 VSTATE_CHANGE(vp, VS_RECLAIMING, VS_RECLAIMED);
1581 vp->v_tag = VT_NON; 1580 vp->v_tag = VT_NON;
1582 KNOTE(&vp->v_klist, NOTE_REVOKE); 1581 KNOTE(&vp->v_klist, NOTE_REVOKE);
1583 1582
1584 fstrans_done(mp); 1583 fstrans_done(mp);
1585 1584
1586 KASSERT((vp->v_iflag & VI_ONWORKLST) == 0); 1585 KASSERT((vp->v_iflag & VI_ONWORKLST) == 0);
1587} 1586}
1588 1587
1589/* 1588/*
1590 * Update outstanding I/O count and do wakeup if requested. 1589 * Update outstanding I/O count and do wakeup if requested.
1591 */ 1590 */
1592void 1591void
1593vwakeup(struct buf *bp) 1592vwakeup(struct buf *bp)
1594{ 1593{
1595 vnode_t *vp; 1594 vnode_t *vp;
1596 1595
1597 if ((vp = bp->b_vp) == NULL) 1596 if ((vp = bp->b_vp) == NULL)
1598 return; 1597 return;
1599 1598
1600 KASSERT(bp->b_objlock == vp->v_interlock); 1599 KASSERT(bp->b_objlock == vp->v_interlock);
1601 KASSERT(mutex_owned(bp->b_objlock)); 1600 KASSERT(mutex_owned(bp->b_objlock));
1602 1601
1603 if (--vp->v_numoutput < 0) 1602 if (--vp->v_numoutput < 0)
1604 vnpanic(vp, "%s: neg numoutput, vp %p", __func__, vp); 1603 vnpanic(vp, "%s: neg numoutput, vp %p", __func__, vp);
1605 if (vp->v_numoutput == 0) 1604 if (vp->v_numoutput == 0)
1606 cv_broadcast(&vp->v_cv); 1605 cv_broadcast(&vp->v_cv);
1607} 1606}
1608 1607
1609/* 1608/*
1610 * Test a vnode for being or becoming dead. Returns one of: 1609 * Test a vnode for being or becoming dead. Returns one of:
1611 * EBUSY: vnode is becoming dead, with "flags == VDEAD_NOWAIT" only. 1610 * EBUSY: vnode is becoming dead, with "flags == VDEAD_NOWAIT" only.
1612 * ENOENT: vnode is dead. 1611 * ENOENT: vnode is dead.
1613 * 0: otherwise. 1612 * 0: otherwise.
1614 * 1613 *
1615 * Whenever this function returns a non-zero value all future 1614 * Whenever this function returns a non-zero value all future
1616 * calls will also return a non-zero value. 1615 * calls will also return a non-zero value.
1617 */ 1616 */
1618int 1617int
1619vdead_check(struct vnode *vp, int flags) 1618vdead_check(struct vnode *vp, int flags)
1620{ 1619{
1621 1620
1622 KASSERT(mutex_owned(vp->v_interlock)); 1621 KASSERT(mutex_owned(vp->v_interlock));
1623 1622
1624 if (! ISSET(flags, VDEAD_NOWAIT)) 1623 if (! ISSET(flags, VDEAD_NOWAIT))
1625 VSTATE_WAIT_STABLE(vp); 1624 VSTATE_WAIT_STABLE(vp);
1626 1625
1627 if (VSTATE_GET(vp) == VS_RECLAIMING) { 1626 if (VSTATE_GET(vp) == VS_RECLAIMING) {
1628 KASSERT(ISSET(flags, VDEAD_NOWAIT)); 1627 KASSERT(ISSET(flags, VDEAD_NOWAIT));
1629 return EBUSY; 1628 return EBUSY;
1630 } else if (VSTATE_GET(vp) == VS_RECLAIMED) { 1629 } else if (VSTATE_GET(vp) == VS_RECLAIMED) {
1631 return ENOENT; 1630 return ENOENT;
1632 } 1631 }
1633 1632
1634 return 0; 1633 return 0;
1635} 1634}
1636 1635
1637int 1636int
1638vfs_drainvnodes(void) 1637vfs_drainvnodes(void)
1639{ 1638{
1640 int i, gen; 1639 int i, gen;
1641 1640
1642 mutex_enter(&vdrain_lock); 1641 mutex_enter(&vdrain_lock);
1643 for (i = 0; i < 2; i++) { 1642 for (i = 0; i < 2; i++) {
1644 gen = vdrain_gen; 1643 gen = vdrain_gen;
1645 while (gen == vdrain_gen) { 1644 while (gen == vdrain_gen) {
1646 cv_broadcast(&vdrain_cv); 1645 cv_broadcast(&vdrain_cv);
1647 cv_wait(&vdrain_gen_cv, &vdrain_lock); 1646 cv_wait(&vdrain_gen_cv, &vdrain_lock);
1648 } 1647 }
1649 } 1648 }
1650 mutex_exit(&vdrain_lock); 1649 mutex_exit(&vdrain_lock);
1651 1650
1652 if (numvnodes >= desiredvnodes) 1651 if (numvnodes >= desiredvnodes)
1653 return EBUSY; 1652 return EBUSY;
1654 1653
1655 if (vcache_hashsize != desiredvnodes) 1654 if (vcache_hashsize != desiredvnodes)
1656 vcache_reinit(); 1655 vcache_reinit();
1657 1656
1658 return 0; 1657 return 0;
1659} 1658}
1660 1659
1661void 1660void
1662vnpanic(vnode_t *vp, const char *fmt, ...) 1661vnpanic(vnode_t *vp, const char *fmt, ...)
1663{ 1662{
1664 va_list ap; 1663 va_list ap;
1665 1664
1666#ifdef DIAGNOSTIC 1665#ifdef DIAGNOSTIC
1667 vprint(NULL, vp); 1666 vprint(NULL, vp);
1668#endif 1667#endif
1669 va_start(ap, fmt); 1668 va_start(ap, fmt);
1670 vpanic(fmt, ap); 1669 vpanic(fmt, ap);
1671 va_end(ap); 1670 va_end(ap);
1672} 1671}