| @@ -1,17 +1,17 @@ | | | @@ -1,17 +1,17 @@ |
1 | /* $NetBSD: kern_lock.c,v 1.164 2019/12/03 15:20:59 riastradh Exp $ */ | | 1 | /* $NetBSD: kern_lock.c,v 1.165 2020/01/17 20:26:22 ad Exp $ */ |
2 | | | 2 | |
3 | /*- | | 3 | /*- |
4 | * Copyright (c) 2002, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc. | | 4 | * Copyright (c) 2002, 2006, 2007, 2008, 2009, 2020 The NetBSD Foundation, Inc. |
5 | * All rights reserved. | | 5 | * All rights reserved. |
6 | * | | 6 | * |
7 | * This code is derived from software contributed to The NetBSD Foundation | | 7 | * This code is derived from software contributed to The NetBSD Foundation |
8 | * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, | | 8 | * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, |
9 | * NASA Ames Research Center, and by Andrew Doran. | | 9 | * NASA Ames Research Center, and by Andrew Doran. |
10 | * | | 10 | * |
11 | * Redistribution and use in source and binary forms, with or without | | 11 | * Redistribution and use in source and binary forms, with or without |
12 | * modification, are permitted provided that the following conditions | | 12 | * modification, are permitted provided that the following conditions |
13 | * are met: | | 13 | * are met: |
14 | * 1. Redistributions of source code must retain the above copyright | | 14 | * 1. Redistributions of source code must retain the above copyright |
15 | * notice, this list of conditions and the following disclaimer. | | 15 | * notice, this list of conditions and the following disclaimer. |
16 | * 2. Redistributions in binary form must reproduce the above copyright | | 16 | * 2. Redistributions in binary form must reproduce the above copyright |
17 | * notice, this list of conditions and the following disclaimer in the | | 17 | * notice, this list of conditions and the following disclaimer in the |
| @@ -21,27 +21,27 @@ | | | @@ -21,27 +21,27 @@ |
21 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED | | 21 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
22 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | | 22 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
23 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS | | 23 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
24 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | | 24 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
25 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | | 25 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
26 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | | 26 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
27 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | | 27 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
28 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | | 28 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
29 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | | 29 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
30 | * POSSIBILITY OF SUCH DAMAGE. | | 30 | * POSSIBILITY OF SUCH DAMAGE. |
31 | */ | | 31 | */ |
32 | | | 32 | |
33 | #include <sys/cdefs.h> | | 33 | #include <sys/cdefs.h> |
34 | __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.164 2019/12/03 15:20:59 riastradh Exp $"); | | 34 | __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.165 2020/01/17 20:26:22 ad Exp $"); |
35 | | | 35 | |
36 | #include <sys/param.h> | | 36 | #include <sys/param.h> |
37 | #include <sys/proc.h> | | 37 | #include <sys/proc.h> |
38 | #include <sys/lock.h> | | 38 | #include <sys/lock.h> |
39 | #include <sys/systm.h> | | 39 | #include <sys/systm.h> |
40 | #include <sys/kernel.h> | | 40 | #include <sys/kernel.h> |
41 | #include <sys/lockdebug.h> | | 41 | #include <sys/lockdebug.h> |
42 | #include <sys/cpu.h> | | 42 | #include <sys/cpu.h> |
43 | #include <sys/syslog.h> | | 43 | #include <sys/syslog.h> |
44 | #include <sys/atomic.h> | | 44 | #include <sys/atomic.h> |
45 | #include <sys/lwp.h> | | 45 | #include <sys/lwp.h> |
46 | #include <sys/pserialize.h> | | 46 | #include <sys/pserialize.h> |
47 | | | 47 | |
| @@ -154,120 +154,134 @@ _kernel_lock_dump(const volatile void *j | | | @@ -154,120 +154,134 @@ _kernel_lock_dump(const volatile void *j |
154 | ci->ci_biglock_count, (long)ci->ci_biglock_wanted); | | 154 | ci->ci_biglock_count, (long)ci->ci_biglock_wanted); |
155 | } | | 155 | } |
156 | | | 156 | |
157 | /* | | 157 | /* |
158 | * Acquire 'nlocks' holds on the kernel lock. | | 158 | * Acquire 'nlocks' holds on the kernel lock. |
159 | */ | | 159 | */ |
160 | void | | 160 | void |
161 | _kernel_lock(int nlocks) | | 161 | _kernel_lock(int nlocks) |
162 | { | | 162 | { |
163 | struct cpu_info *ci; | | 163 | struct cpu_info *ci; |
164 | LOCKSTAT_TIMER(spintime); | | 164 | LOCKSTAT_TIMER(spintime); |
165 | LOCKSTAT_FLAG(lsflag); | | 165 | LOCKSTAT_FLAG(lsflag); |
166 | struct lwp *owant; | | 166 | struct lwp *owant; |
167 | u_int spins; | | 167 | u_int count; |
| | | 168 | #ifdef LOCKDEBUG |
| | | 169 | u_int spins = 0; |
| | | 170 | #endif |
168 | int s; | | 171 | int s; |
169 | struct lwp *l = curlwp; | | 172 | struct lwp *l = curlwp; |
170 | | | 173 | |
171 | _KERNEL_LOCK_ASSERT(nlocks > 0); | | 174 | _KERNEL_LOCK_ASSERT(nlocks > 0); |
172 | | | 175 | |
173 | s = splvm(); | | 176 | s = splvm(); |
174 | ci = curcpu(); | | 177 | ci = curcpu(); |
175 | if (ci->ci_biglock_count != 0) { | | 178 | if (ci->ci_biglock_count != 0) { |
176 | _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock)); | | 179 | _KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock)); |
177 | ci->ci_biglock_count += nlocks; | | 180 | ci->ci_biglock_count += nlocks; |
178 | l->l_blcnt += nlocks; | | 181 | l->l_blcnt += nlocks; |
179 | splx(s); | | 182 | splx(s); |
180 | return; | | 183 | return; |
181 | } | | 184 | } |
182 | | | 185 | |
183 | _KERNEL_LOCK_ASSERT(l->l_blcnt == 0); | | 186 | _KERNEL_LOCK_ASSERT(l->l_blcnt == 0); |
184 | LOCKDEBUG_WANTLOCK(kernel_lock_dodebug, kernel_lock, RETURN_ADDRESS, | | 187 | LOCKDEBUG_WANTLOCK(kernel_lock_dodebug, kernel_lock, RETURN_ADDRESS, |
185 | 0); | | 188 | 0); |
186 | | | 189 | |
187 | if (__cpu_simple_lock_try(kernel_lock)) { | | 190 | if (__predict_true(__cpu_simple_lock_try(kernel_lock))) { |
188 | ci->ci_biglock_count = nlocks; | | 191 | ci->ci_biglock_count = nlocks; |
189 | l->l_blcnt = nlocks; | | 192 | l->l_blcnt = nlocks; |
190 | LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL, | | 193 | LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL, |
191 | RETURN_ADDRESS, 0); | | 194 | RETURN_ADDRESS, 0); |
192 | splx(s); | | 195 | splx(s); |
193 | return; | | 196 | return; |
194 | } | | 197 | } |
195 | | | 198 | |
196 | /* | | 199 | /* |
197 | * To remove the ordering constraint between adaptive mutexes | | 200 | * To remove the ordering constraint between adaptive mutexes |
198 | * and kernel_lock we must make it appear as if this thread is | | 201 | * and kernel_lock we must make it appear as if this thread is |
199 | * blocking. For non-interlocked mutex release, a store fence | | 202 | * blocking. For non-interlocked mutex release, a store fence |
200 | * is required to ensure that the result of any mutex_exit() | | 203 | * is required to ensure that the result of any mutex_exit() |
201 | * by the current LWP becomes visible on the bus before the set | | 204 | * by the current LWP becomes visible on the bus before the set |
202 | * of ci->ci_biglock_wanted becomes visible. | | 205 | * of ci->ci_biglock_wanted becomes visible. |
| | | 206 | * |
| | | 207 | * However, we won't set ci_biglock_wanted until we've spun for |
| | | 208 | * a bit, as we don't want to make any lock waiters in rw_oncpu() |
| | | 209 | * or mutex_oncpu() block prematurely. |
203 | */ | | 210 | */ |
204 | membar_producer(); | | 211 | membar_producer(); |
205 | owant = ci->ci_biglock_wanted; | | 212 | owant = ci->ci_biglock_wanted; |
206 | ci->ci_biglock_wanted = l; | | | |
207 | | | 213 | |
208 | /* | | 214 | /* |
209 | * Spin until we acquire the lock. Once we have it, record the | | 215 | * Spin until we acquire the lock. Once we have it, record the |
210 | * time spent with lockstat. | | 216 | * time spent with lockstat. |
211 | */ | | 217 | */ |
212 | LOCKSTAT_ENTER(lsflag); | | 218 | LOCKSTAT_ENTER(lsflag); |
213 | LOCKSTAT_START_TIMER(lsflag, spintime); | | 219 | LOCKSTAT_START_TIMER(lsflag, spintime); |
214 | | | 220 | |
215 | spins = 0; | | 221 | count = SPINLOCK_BACKOFF_MIN; |
216 | do { | | 222 | do { |
217 | splx(s); | | 223 | splx(s); |
218 | while (__SIMPLELOCK_LOCKED_P(kernel_lock)) { | | 224 | while (__SIMPLELOCK_LOCKED_P(kernel_lock)) { |
| | | 225 | #ifdef LOCKDEBUG |
219 | if (SPINLOCK_SPINOUT(spins)) { | | 226 | if (SPINLOCK_SPINOUT(spins)) { |
220 | extern int start_init_exec; | | 227 | extern int start_init_exec; |
221 | if (!start_init_exec) | | 228 | if (!start_init_exec) |
222 | _KERNEL_LOCK_ABORT("spinout"); | | 229 | _KERNEL_LOCK_ABORT("spinout"); |
223 | } | | 230 | } |
224 | SPINLOCK_BACKOFF_HOOK; | | 231 | #endif |
225 | SPINLOCK_SPIN_HOOK; | | 232 | SPINLOCK_BACKOFF(count); |
| | | 233 | if (count == SPINLOCK_BACKOFF_MAX) { |
| | | 234 | /* Ok, waiting for real. */ |
| | | 235 | ci->ci_biglock_wanted = l; |
| | | 236 | } |
226 | } | | 237 | } |
227 | s = splvm(); | | 238 | s = splvm(); |
228 | } while (!__cpu_simple_lock_try(kernel_lock)); | | 239 | } while (!__cpu_simple_lock_try(kernel_lock)); |
229 | | | 240 | |
230 | ci->ci_biglock_count = nlocks; | | 241 | ci->ci_biglock_count = nlocks; |
231 | l->l_blcnt = nlocks; | | 242 | l->l_blcnt = nlocks; |
| | | 243 | splx(s); |
| | | 244 | |
232 | LOCKSTAT_STOP_TIMER(lsflag, spintime); | | 245 | LOCKSTAT_STOP_TIMER(lsflag, spintime); |
233 | LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL, | | 246 | LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL, |
234 | RETURN_ADDRESS, 0); | | 247 | RETURN_ADDRESS, 0); |
235 | if (owant == NULL) { | | 248 | if (owant == NULL) { |
236 | LOCKSTAT_EVENT_RA(lsflag, kernel_lock, | | 249 | LOCKSTAT_EVENT_RA(lsflag, kernel_lock, |
237 | LB_KERNEL_LOCK | LB_SPIN, 1, spintime, RETURN_ADDRESS); | | 250 | LB_KERNEL_LOCK | LB_SPIN, 1, spintime, RETURN_ADDRESS); |
238 | } | | 251 | } |
239 | LOCKSTAT_EXIT(lsflag); | | 252 | LOCKSTAT_EXIT(lsflag); |
240 | splx(s); | | | |
241 | | | 253 | |
242 | /* | | 254 | /* |
243 | * Now that we have kernel_lock, reset ci_biglock_wanted. This | | 255 | * Now that we have kernel_lock, reset ci_biglock_wanted. This |
244 | * store must be unbuffered (immediately visible on the bus) in | | 256 | * store must be unbuffered (immediately visible on the bus) in |
245 | * order for non-interlocked mutex release to work correctly. | | 257 | * order for non-interlocked mutex release to work correctly. |
246 | * It must be visible before a mutex_exit() can execute on this | | 258 | * It must be visible before a mutex_exit() can execute on this |
247 | * processor. | | 259 | * processor. |
248 | * | | 260 | * |
249 | * Note: only where CAS is available in hardware will this be | | 261 | * Note: only where CAS is available in hardware will this be |
250 | * an unbuffered write, but non-interlocked release cannot be | | 262 | * an unbuffered write, but non-interlocked release cannot be |
251 | * done on CPUs without CAS in hardware. | | 263 | * done on CPUs without CAS in hardware. |
252 | */ | | 264 | */ |
253 | (void)atomic_swap_ptr(&ci->ci_biglock_wanted, owant); | | 265 | (void)atomic_swap_ptr(&ci->ci_biglock_wanted, owant); |
254 | | | 266 | |
255 | /* | | 267 | /* |
256 | * Issue a memory barrier as we have acquired a lock. This also | | 268 | * Issue a memory barrier as we have acquired a lock. This also |
257 | * prevents stores from a following mutex_exit() being reordered | | 269 | * prevents stores from a following mutex_exit() being reordered |
258 | * to occur before our store to ci_biglock_wanted above. | | 270 | * to occur before our store to ci_biglock_wanted above. |
259 | */ | | 271 | */ |
| | | 272 | #ifndef __HAVE_ATOMIC_AS_MEMBAR |
260 | membar_enter(); | | 273 | membar_enter(); |
| | | 274 | #endif |
261 | } | | 275 | } |
262 | | | 276 | |
263 | /* | | 277 | /* |
264 | * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release | | 278 | * Release 'nlocks' holds on the kernel lock. If 'nlocks' is zero, release |
265 | * all holds. | | 279 | * all holds. |
266 | */ | | 280 | */ |
267 | void | | 281 | void |
268 | _kernel_unlock(int nlocks, int *countp) | | 282 | _kernel_unlock(int nlocks, int *countp) |
269 | { | | 283 | { |
270 | struct cpu_info *ci; | | 284 | struct cpu_info *ci; |
271 | u_int olocks; | | 285 | u_int olocks; |
272 | int s; | | 286 | int s; |
273 | struct lwp *l = curlwp; | | 287 | struct lwp *l = curlwp; |