Thu Dec 5 03:21:17 2019 UTC ()
Restore psz_lock just for the event count.

Cost of mutex_enter/exit is negligible compared to the xcall we just
did, so this is not going to meaningfully affect performance.


(riastradh)
diff -r1.15 -r1.16 src/sys/kern/subr_pserialize.c
cvs diff -r1.15 -r1.16 src/sys/kern/subr_pserialize.c (expand / switch to unified diff)

--- src/sys/kern/subr_pserialize.c 2019/12/03 13:30:52 1.15
+++ src/sys/kern/subr_pserialize.c 2019/12/05 03:21:17 1.16
@@ -1,14 +1,14 @@ @@ -1,14 +1,14 @@
1/* $NetBSD: subr_pserialize.c,v 1.15 2019/12/03 13:30:52 martin Exp $ */ 1/* $NetBSD: subr_pserialize.c,v 1.16 2019/12/05 03:21:17 riastradh Exp $ */
2 2
3/*- 3/*-
4 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 4 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * Redistribution and use in source and binary forms, with or without 7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions 8 * modification, are permitted provided that the following conditions
9 * are met: 9 * are met:
10 * 1. Redistributions of source code must retain the above copyright 10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer. 11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright 12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the 13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution. 14 * documentation and/or other materials provided with the distribution.
@@ -21,51 +21,54 @@ @@ -21,51 +21,54 @@
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE. 26 * POSSIBILITY OF SUCH DAMAGE.
27 */ 27 */
28 28
29/* 29/*
30 * Passive serialization. 30 * Passive serialization.
31 */ 31 */
32 32
33#include <sys/cdefs.h> 33#include <sys/cdefs.h>
34__KERNEL_RCSID(0, "$NetBSD: subr_pserialize.c,v 1.15 2019/12/03 13:30:52 martin Exp $"); 34__KERNEL_RCSID(0, "$NetBSD: subr_pserialize.c,v 1.16 2019/12/05 03:21:17 riastradh Exp $");
35 35
36#include <sys/param.h> 36#include <sys/param.h>
37#include <sys/atomic.h> 37#include <sys/atomic.h>
38#include <sys/cpu.h> 38#include <sys/cpu.h>
39#include <sys/evcnt.h> 39#include <sys/evcnt.h>
40#include <sys/kmem.h> 40#include <sys/kmem.h>
 41#include <sys/mutex.h>
41#include <sys/pserialize.h> 42#include <sys/pserialize.h>
42#include <sys/xcall.h> 43#include <sys/xcall.h>
43 44
44struct pserialize { 45struct pserialize {
45 lwp_t * psz_owner; 46 lwp_t * psz_owner;
46}; 47};
47 48
 49static kmutex_t psz_lock __cacheline_aligned;
48static struct evcnt psz_ev_excl __cacheline_aligned; 50static struct evcnt psz_ev_excl __cacheline_aligned;
49 51
50/* 52/*
51 * pserialize_init: 53 * pserialize_init:
52 * 54 *
53 * Initialize passive serialization structures. 55 * Initialize passive serialization structures.
54 */ 56 */
55void 57void
56pserialize_init(void) 58pserialize_init(void)
57{ 59{
58 60
 61 mutex_init(&psz_lock, MUTEX_DEFAULT, IPL_NONE);
59 evcnt_attach_dynamic(&psz_ev_excl, EVCNT_TYPE_MISC, NULL, 62 evcnt_attach_dynamic(&psz_ev_excl, EVCNT_TYPE_MISC, NULL,
60 "pserialize", "exclusive access"); 63 "pserialize", "exclusive access");
61} 64}
62 65
63/* 66/*
64 * pserialize_create: 67 * pserialize_create:
65 * 68 *
66 * Create and initialize a passive serialization object. 69 * Create and initialize a passive serialization object.
67 */ 70 */
68pserialize_t 71pserialize_t
69pserialize_create(void) 72pserialize_create(void)
70{ 73{
71 pserialize_t psz; 74 pserialize_t psz;
@@ -110,30 +113,29 @@ pserialize_perform(pserialize_t psz) @@ -110,30 +113,29 @@ pserialize_perform(pserialize_t psz)
110 psz_ev_excl.ev_count++; 113 psz_ev_excl.ev_count++;
111 return; 114 return;
112 } 115 }
113 116
114 psz->psz_owner = curlwp; 117 psz->psz_owner = curlwp;
115 118
116 /* 119 /*
117 * Broadcast a NOP to all CPUs and wait until all of them complete. 120 * Broadcast a NOP to all CPUs and wait until all of them complete.
118 */ 121 */
119 xc_barrier(XC_HIGHPRI); 122 xc_barrier(XC_HIGHPRI);
120 123
121 KASSERT(psz->psz_owner == curlwp); 124 KASSERT(psz->psz_owner == curlwp);
122 psz->psz_owner = NULL; 125 psz->psz_owner = NULL;
123#ifdef __HAVE_ATOMIC64_LOADSTORE 126 mutex_enter(&psz_lock);
124 atomic_store_relaxed(&psz_ev_excl.ev_count, 127 psz_ev_excl.ev_count++;
125 1 + atomic_load_relaxed(&psz_ev_excl.ev_count)); 128 mutex_exit(&psz_lock);
126#endif 
127} 129}
128 130
129int 131int
130pserialize_read_enter(void) 132pserialize_read_enter(void)
131{ 133{
132 int s; 134 int s;
133 135
134 s = splsoftserial(); 136 s = splsoftserial();
135 curcpu()->ci_psz_read_depth++; 137 curcpu()->ci_psz_read_depth++;
136 __insn_barrier(); 138 __insn_barrier();
137 return s; 139 return s;
138} 140}
139 141