Sun Apr 9 08:17:45 2023 UTC ()
pserialize(9): Micro-optimize pserialize_not_in_read_section_p.

Load l_ncsw to test whether we have been preempted, rather than
loading and storing l_nopreempt (via function call) to prevent it.


(riastradh)
diff -r1.19 -r1.20 src/sys/kern/subr_pserialize.c
cvs diff -r1.19 -r1.20 src/sys/kern/subr_pserialize.c (expand / switch to unified diff)

--- src/sys/kern/subr_pserialize.c 2022/11/15 10:29:56 1.19
+++ src/sys/kern/subr_pserialize.c 2023/04/09 08:17:45 1.20
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: subr_pserialize.c,v 1.19 2022/11/15 10:29:56 macallan Exp $ */ 1/* $NetBSD: subr_pserialize.c,v 1.20 2023/04/09 08:17:45 riastradh Exp $ */
2 2
3/*- 3/*-
4 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 4 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * Redistribution and use in source and binary forms, with or without 7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions 8 * modification, are permitted provided that the following conditions
9 * are met: 9 * are met:
10 * 1. Redistributions of source code must retain the above copyright 10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer. 11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright 12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the 13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution. 14 * documentation and/or other materials provided with the distribution.
@@ -21,34 +21,36 @@ @@ -21,34 +21,36 @@
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE. 26 * POSSIBILITY OF SUCH DAMAGE.
27 */ 27 */
28 28
29/* 29/*
30 * Passive serialization. 30 * Passive serialization.
31 */ 31 */
32 32
33#include <sys/cdefs.h> 33#include <sys/cdefs.h>
34__KERNEL_RCSID(0, "$NetBSD: subr_pserialize.c,v 1.19 2022/11/15 10:29:56 macallan Exp $"); 34__KERNEL_RCSID(0, "$NetBSD: subr_pserialize.c,v 1.20 2023/04/09 08:17:45 riastradh Exp $");
35 35
36#include <sys/param.h> 36#include <sys/param.h>
 37
37#include <sys/atomic.h> 38#include <sys/atomic.h>
38#include <sys/cpu.h> 39#include <sys/cpu.h>
39#include <sys/kernel.h> 
40#include <sys/evcnt.h> 40#include <sys/evcnt.h>
 41#include <sys/kernel.h>
41#include <sys/kmem.h> 42#include <sys/kmem.h>
 43#include <sys/lwp.h>
42#include <sys/mutex.h> 44#include <sys/mutex.h>
43#include <sys/pserialize.h> 45#include <sys/pserialize.h>
44#include <sys/xcall.h> 46#include <sys/xcall.h>
45 47
46struct pserialize { 48struct pserialize {
47 char psz_dummy; 49 char psz_dummy;
48}; 50};
49 51
50static kmutex_t psz_lock __cacheline_aligned; 52static kmutex_t psz_lock __cacheline_aligned;
51static struct evcnt psz_ev_excl __cacheline_aligned = 53static struct evcnt psz_ev_excl __cacheline_aligned =
52 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pserialize", "exclusive access"); 54 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pserialize", "exclusive access");
53EVCNT_ATTACH_STATIC(psz_ev_excl); 55EVCNT_ATTACH_STATIC(psz_ev_excl);
54 56
@@ -162,21 +164,34 @@ pserialize_in_read_section(void) @@ -162,21 +164,34 @@ pserialize_in_read_section(void)
162 164
163/* 165/*
164 * pserialize_not_in_read_section: 166 * pserialize_not_in_read_section:
165 * 167 *
166 * True if the caller is not in a pserialize read section. To be 168 * True if the caller is not in a pserialize read section. To be
167 * used only for diagnostic assertions where we want to guarantee 169 * used only for diagnostic assertions where we want to guarantee
168 * the condition like: 170 * the condition like:
169 * 171 *
170 * KASSERT(pserialize_not_in_read_section()); 172 * KASSERT(pserialize_not_in_read_section());
171 */ 173 */
172bool 174bool
173pserialize_not_in_read_section(void) 175pserialize_not_in_read_section(void)
174{ 176{
 177 struct lwp *l = curlwp;
 178 uint64_t ncsw;
175 bool notin; 179 bool notin;
176 180
177 kpreempt_disable(); 181 ncsw = l->l_ncsw;
178 notin = (curcpu()->ci_psz_read_depth == 0); 182 __insn_barrier();
179 kpreempt_enable(); 183 notin = __predict_true(l->l_cpu->ci_psz_read_depth == 0);
 184 __insn_barrier();
 185
 186 /*
 187 * If we had a context switch, we're definitely not in a
 188 * pserialize read section because pserialize read sections
 189 * block preemption.
 190 */
 191 if (__predict_false(ncsw != l->l_ncsw)) {
 192 KDASSERT(notin);
 193 notin = true;
 194 }
180 195
181 return notin; 196 return notin;
182} 197}