 @@ -1,14 +1,14 @@
-/*	$NetBSD: lockstat.c,v 1.10 2019/02/12 14:31:45 rin Exp $	*/
+/*	$NetBSD: lockstat.c,v 1.11 2022/02/27 14:16:12 riastradh Exp $	*/
 /*
  * CDDL HEADER START
+ *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
+ *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
  * See the License for the specific language governing permissions
  * and limitations under the License.
+ *
 @@ -16,27 +16,27 @@
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
  * CDDL HEADER END
  */
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.10 2019/02/12 14:31:45 rin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.11 2022/02/27 14:16:12 riastradh Exp $");
 #include <sys/types.h>
 #include <sys/proc.h>
 #include <sys/param.h>
 #include <sys/conf.h>
 #include <sys/dtrace.h>
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <sys/systm.h>
 #include <sys/xcall.h>
 #include <sys/atomic.h>
 #define NLOCKSTAT 1
 @@ -62,49 +62,58 @@ lockstat_probe_t lockstat_probes[] = {
 };
 static dtrace_provider_id_t lockstat_id;
 static size_t lockstat_dtrace_count;
 /*ARGSUSED*/
 static int
 lockstat_enable(void *arg, dtrace_id_t id, void *parg)
+{
 	lockstat_probe_t *probe = parg;
 	ASSERT(!lockstat_probemap[LS_COMPRESS(probe->lsp_probe)]);
 	lockstat_probemap[LS_COMPRESS(probe->lsp_probe)] = id;
 	if (lockstat_dtrace_count++ == 0) {
 		LOCKSTAT_ENABLED_UPDATE_BEGIN();
 		lockstat_dtrace_enabled = LB_DTRACE;
-		LOCKSTAT_ENABLED_UPDATE();
+		LOCKSTAT_ENABLED_UPDATE_END();
+	}
 	atomic_store_relaxed(&lockstat_probemap[LS_COMPRESS(probe->lsp_probe)],
 	    id);
 	return 0;
+}
 /*ARGSUSED*/
 static void
 lockstat_disable(void *arg, dtrace_id_t id __unused, void *parg)
+{
 	lockstat_probe_t *probe = parg;
 	ASSERT(lockstat_probemap[LS_COMPRESS(probe->lsp_probe)]);
 	atomic_store_relaxed(&lockstat_probemap[LS_COMPRESS(probe->lsp_probe)],
 );
 	if (--lockstat_dtrace_count == 0) {
 		LOCKSTAT_ENABLED_UPDATE_BEGIN();
 		lockstat_dtrace_enabled = 0;
-		LOCKSTAT_ENABLED_UPDATE();
+		LOCKSTAT_ENABLED_UPDATE_END();
-	lockstat_probemap[LS_COMPRESS(probe->lsp_probe)] = 0;
+		/*
 		 * Wait for all lockstat dtrace probe on all CPUs to
 		 * finish, now that they've been disabled.
 		 */
 		xc_barrier(0);
+	}
+}
 /*ARGSUSED*/
 static void
 lockstat_provide(void *arg, dtrace_probedesc_t *desc)
+{
 	int i = 0;
 	for (i = 0; lockstat_probes[i].lsp_func != NULL; i++) {
 		lockstat_probe_t *probe = &lockstat_probes[i];
 		if (dtrace_probe_lookup(lockstat_id, __UNCONST("kernel"),
 			__UNCONST(probe->lsp_func), __UNCONST(probe->lsp_name))
 @@ -139,48 +148,47 @@ static dtrace_pattr_t lockstat_attr = {
 static dtrace_pops_t lockstat_pops = {
 	lockstat_provide,
 	NULL,
 	lockstat_enable,
 	lockstat_disable,
 	NULL,
 	NULL,
 	NULL,
 	NULL,
 	NULL,
 	lockstat_destroy
 };
 static void
 lockstat_barrier_xc(void *arg0 __unused, void *arg1 __unused)
 	membar_consumer();
 typedef void (*dtrace_probe_func_t)(dtrace_id_t, uintptr_t, uintptr_t,
     uintptr_t, uintptr_t, uintptr_t);
 static bool
 lockstat_cas_probe(dtrace_probe_func_t old, dtrace_probe_func_t new)
+{
 	ASSERT(kernconfig_is_held());
 	if (lockstat_probe_func != old)
 		return false;
 	lockstat_probe_func = new;
 	membar_producer();
-	xc_wait(xc_broadcast(0, lockstat_barrier_xc, NULL, NULL));
+	/*
 	 * Make sure that the probe function is initialized on all CPUs
 	 * before we enable the lockstat probe by setting
 	 * lockstat_probemap[...].
 	 */
 	xc_barrier(0);
 	return true;
+}
 static int
 lockstat_init(void)
+{
 	int error;
 	bool ok;
 	/* Install the probe function.  */
 	ok = lockstat_cas_probe(lockstat_probe_stub, dtrace_probe);
 	if (!ok) {
 		printf("dtrace_lockstat: lockstat probe already installed\n");

 @@ -1,14 +1,14 @@
-/*	$NetBSD: lockstat.c,v 1.27 2020/05/23 23:42:42 ad Exp $	*/
+/*	$NetBSD: lockstat.c,v 1.28 2022/02/27 14:16:12 riastradh Exp $	*/
 /*-
  * Copyright (c) 2006, 2007, 2019 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Andrew Doran.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -31,39 +31,40 @@
 /*
  * Lock statistics driver, providing kernel support for the lockstat(8)
  * command.
+ *
  * We use a global lock word (lockstat_lock) to track device opens.
  * Only one thread can hold the device at a time, providing a global lock.
+ *
  * XXX Timings for contention on sleep locks are currently incorrect.
  * XXX Convert this to use timecounters!
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.27 2020/05/23 23:42:42 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.28 2022/02/27 14:16:12 riastradh Exp $");
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/proc.h>
 #include <sys/resourcevar.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/conf.h>
 #include <sys/cpu.h>
 #include <sys/syslog.h>
 #include <sys/atomic.h>
 #include <sys/xcall.h>
 #include <dev/lockstat.h>
 #include <machine/lock.h>
 #include "ioconf.h"
 #ifndef __HAVE_CPU_COUNTER
 #error CPU counters not available
 #endif
 #if LONG_BIT == 64
 #define	LOCKSTAT_HASH_SHIFT	3
 @@ -91,26 +92,27 @@ typedef struct lslist lslist_t;
 void	lockstat_start(lsenable_t *);
 int	lockstat_alloc(lsenable_t *);
 void	lockstat_init_tables(lsenable_t *);
 int	lockstat_stop(lsdisable_t *);
 void	lockstat_free(void);
 dev_type_open(lockstat_open);
 dev_type_close(lockstat_close);
 dev_type_read(lockstat_read);
 dev_type_ioctl(lockstat_ioctl);
 volatile u_int	lockstat_enabled;
 volatile u_int	lockstat_dev_enabled;
 __cpu_simple_lock_t lockstat_enabled_lock;
 uintptr_t	lockstat_csstart;
 uintptr_t	lockstat_csend;
 uintptr_t	lockstat_csmask;
 uintptr_t	lockstat_lamask;
 uintptr_t	lockstat_lockstart;
 uintptr_t	lockstat_lockend;
 __cpu_simple_lock_t lockstat_lock;
 lwp_t		*lockstat_lwp;
 lsbuf_t		*lockstat_baseb;
 size_t		lockstat_sizeb;
 int		lockstat_busy;
 struct timespec	lockstat_stime;
 @@ -144,26 +146,27 @@ const struct cdevsw lockstat_cdevsw = {
 	.d_flag = D_OTHER | D_MPSAFE
 };
 /*
  * Called when the pseudo-driver is attached.
  */
 void
 lockstatattach(int nunits)
+{
 	(void)nunits;
 	__cpu_simple_lock_init(&lockstat_lock);
 	__cpu_simple_lock_init(&lockstat_enabled_lock);
+}
 /*
  * Prepare the per-CPU tables for use, or clear down tables when tracing is
  * stopped.
  */
 void
 lockstat_init_tables(lsenable_t *le)
+{
 	int i, per, slop, cpuno;
 	CPU_INFO_ITERATOR cii;
 	struct cpu_info *ci;
 	lscpu_t *lc;
 @@ -225,56 +228,72 @@ lockstat_start(lsenable_t *le)
 	else
 		lockstat_csmask = 0;
 	if ((le->le_flags & LE_LOCK) != 0)
 		lockstat_lamask = (uintptr_t)-1LL;
 	else
 		lockstat_lamask = 0;
 	lockstat_csstart = le->le_csstart;
 	lockstat_csend = le->le_csend;
 	lockstat_lockstart = le->le_lockstart;
 	lockstat_lockstart = le->le_lockstart;
 	lockstat_lockend = le->le_lockend;
 	membar_sync();
 	/*
 	 * Ensure everything is initialized on all CPUs, by issuing a
 	 * null xcall with the side effect of a release barrier on this
 	 * CPU and an acquire barrier on all other CPUs, before they
 	 * can witness any flags set in lockstat_dev_enabled -- this
 	 * way we don't need to add any barriers in lockstat_event.
 	 */
 	xc_barrier(0);
 	/*
 	 * Start timing after the xcall, so we don't spuriously count
 	 * xcall communication time, but before flipping the switch, so
 	 * we don't dirty sample with locks taken in the timecounter.
 	 */
 	getnanotime(&lockstat_stime);
 	lockstat_dev_enabled = le->le_mask;
-	LOCKSTAT_ENABLED_UPDATE();
+	LOCKSTAT_ENABLED_UPDATE_BEGIN();
 	atomic_store_relaxed(&lockstat_dev_enabled, le->le_mask);
 	LOCKSTAT_ENABLED_UPDATE_END();
+}
 /*
  * Stop collecting lock statistics.
  */
 int
 lockstat_stop(lsdisable_t *ld)
+{
 	CPU_INFO_ITERATOR cii;
 	struct cpu_info *ci;
 	u_int cpuno, overflow;
 	struct timespec ts;
 	int error;
 	lwp_t *l;
 	/* coverity[assert_side_effect] */
 	KASSERT(lockstat_dev_enabled);
 	/*
-	 * Set enabled false, force a write barrier, and wait for other CPUs
+	 * Disable and wait for other CPUs to exit lockstat_event().
 	 * to exit lockstat_event().
 	 */
-	lockstat_dev_enabled = 0;
+	LOCKSTAT_ENABLED_UPDATE_BEGIN();
-	LOCKSTAT_ENABLED_UPDATE();
+	atomic_store_relaxed(&lockstat_dev_enabled, 0);
 	LOCKSTAT_ENABLED_UPDATE_END();
 	getnanotime(&ts);
-	tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10));
+	xc_barrier(0);
 	/*
 	 * Did we run out of buffers while tracing?
 	 */
 	overflow = 0;
 	for (CPU_INFO_FOREACH(cii, ci))
 		overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow;
 	if (overflow != 0) {
 		error = EOVERFLOW;
 		log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n",
 		    overflow);
 	} else
 @@ -360,32 +379,34 @@ lockstat_free(void)
 void
 lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count,
 	       uint64_t cycles)
+{
 	lslist_t *ll;
 	lscpu_t *lc;
 	lsbuf_t *lb;
 	u_int event;
 	int s;
 #ifdef KDTRACE_HOOKS
 	uint32_t id;
 	CTASSERT((LS_NPROBES & (LS_NPROBES - 1)) == 0);
-	if ((id = lockstat_probemap[LS_COMPRESS(flags)]) != 0)
+	if ((id = atomic_load_relaxed(&lockstat_probemap[LS_COMPRESS(flags)]))
 	    != 0)
 		(*lockstat_probe_func)(id, lock, callsite, flags, count,
 		    cycles);
 #endif
-	if ((flags & lockstat_dev_enabled) != flags || count == 0)
+	if ((flags & atomic_load_relaxed(&lockstat_dev_enabled)) != flags ||
 	    count == 0)
 		return;
 	if (lock < lockstat_lockstart || lock > lockstat_lockend)
 		return;
 	if (callsite < lockstat_csstart || callsite > lockstat_csend)
 		return;
 	callsite &= lockstat_csmask;
 	lock &= lockstat_lamask;
 	/*
 	 * Find the table for this lock+callsite pair, and try to locate a
 	 * buffer with the same key.
 	 */
 @@ -477,27 +498,27 @@ lockstat_ioctl(dev_t dev, u_long cmd, vo
 	switch (cmd) {
 	case IOC_LOCKSTAT_GVERSION:
 		*(int *)data = LS_VERSION;
 		error = 0;
 		break;
 	case IOC_LOCKSTAT_ENABLE:
 		le = (lsenable_t *)data;
 		if (!cpu_hascounter()) {
 			error = ENODEV;
 			break;
+		}
-		if (lockstat_dev_enabled) {
+		if (atomic_load_relaxed(&lockstat_dev_enabled)) {
 			error = EBUSY;
 			break;
+		}
 		/*
 		 * Sanitize the arguments passed in and set up filtering.
 		 */
 		if (le->le_nbufs == 0) {
 			le->le_nbufs = MIN(LOCKSTAT_DEFBUFS * ncpu,
 			    LOCKSTAT_MAXBUFS);
 		} else if (le->le_nbufs > LOCKSTAT_MAXBUFS ||
 		    le->le_nbufs < LOCKSTAT_MINBUFS) {
 			error = EINVAL;
 @@ -514,27 +535,27 @@ lockstat_ioctl(dev_t dev, u_long cmd, vo
 		if ((le->le_mask & LB_EVENT_MASK) == 0)
 			return EINVAL;
 		if ((le->le_mask & LB_LOCK_MASK) == 0)
 			return EINVAL;
 		/*
 		 * Start tracing.
 		 */
 		if ((error = lockstat_alloc(le)) == 0)
 			lockstat_start(le);
 		break;
 	case IOC_LOCKSTAT_DISABLE:
-		if (!lockstat_dev_enabled)
+		if (!atomic_load_relaxed(&lockstat_dev_enabled))
 			error = EINVAL;
 		else
 			error = lockstat_stop((lsdisable_t *)data);
 		break;
 	default:
 		error = ENOTTY;
 		break;
+	}
 	return error;
+}

 @@ -1,14 +1,14 @@
-/*	$NetBSD: lockstat.h,v 1.14 2016/01/24 01:01:11 christos Exp $	*/
+/*	$NetBSD: lockstat.h,v 1.15 2022/02/27 14:16:12 riastradh Exp $	*/
 /*-
  * Copyright (c) 2006 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Andrew Doran.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -28,27 +28,29 @@
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #ifndef _SYS_LOCKSTAT_H_
 #define _SYS_LOCKSTAT_H_
 #ifdef _KERNEL_OPT
 #include "opt_dtrace.h"
 #include <lockstat.h>
 #endif
 #include <sys/types.h>
 #include <sys/ioccom.h>
 #include <sys/lock.h>
 #include <sys/queue.h>
 #include <sys/time.h>
 #if defined(_KERNEL) && defined(__HAVE_CPU_COUNTER)
 #include <machine/cpu_counter.h>
 #endif
 /*
  * Interface version.  The interface is not designed to provide
  * compatibility across NetBSD releases.
  */
 #define	IOC_LOCKSTAT_GVERSION	_IOR('L', 0, int)
 @@ -149,27 +151,27 @@ do {									\
 		    (type), (count), (time));				\
 } while (/* CONSTCOND */ 0);
 #define	LOCKSTAT_EVENT_RA(flag, lock, type, count, time, ra)		\
 do {									\
 	if (__predict_false(flag))					\
 		lockstat_event((uintptr_t)(lock), (uintptr_t)ra,	\
 		    (type), (count), (time));				\
 } while (/* CONSTCOND */ 0);
 #define	LOCKSTAT_TIMER(name)	uint64_t name = 0
 #define	LOCKSTAT_COUNTER(name)	uint64_t name = 0
 #define	LOCKSTAT_FLAG(name)	int name
-#define	LOCKSTAT_ENTER(name)	name = lockstat_enabled
+#define	LOCKSTAT_ENTER(name)	name = atomic_load_relaxed(&lockstat_enabled)
 #define	LOCKSTAT_EXIT(name)
 #define	LOCKSTAT_START_TIMER(flag, name)				\
 do {									\
 	if (__predict_false(flag))					\
 		(name) -= cpu_counter();				\
 } while (/* CONSTCOND */ 0)
 #define	LOCKSTAT_STOP_TIMER(flag, name)					\
 do {									\
 	if (__predict_false(flag))					\
 		(name) += cpu_counter();				\
 } while (/* CONSTCOND */ 0)
 @@ -204,23 +206,32 @@ extern volatile u_int lockstat_dtrace_en
 #define	LS_NPROBES	0x20	/* 5 bits */
 extern uint32_t	lockstat_probemap[];
 extern void	(*lockstat_probe_func)(uint32_t, uintptr_t, uintptr_t,
     uintptr_t, uintptr_t, uintptr_t);
 void		lockstat_probe_stub(uint32_t, uintptr_t, uintptr_t,
     uintptr_t, uintptr_t, uintptr_t);
 #else
 #define KDTRACE_LOCKSTAT_ENABLED 0
 #endif
 #if defined(_KERNEL) && NLOCKSTAT > 0
 extern __cpu_simple_lock_t lockstat_enabled_lock;
 extern volatile u_int	lockstat_enabled;
 extern volatile u_int	lockstat_dev_enabled;
-#define LOCKSTAT_ENABLED_UPDATE() do { \
+#define LOCKSTAT_ENABLED_UPDATE_BEGIN() do				    \
-	lockstat_enabled = lockstat_dev_enabled | KDTRACE_LOCKSTAT_ENABLED; \
+{									    \
-	membar_producer(); \
+	__cpu_simple_lock(&lockstat_enabled_lock);			    \
-    } while (/*CONSTCOND*/0)
+} while (/*CONSTCOND*/0)
 #define LOCKSTAT_ENABLED_UPDATE_END() do				    \
 {									    \
 	atomic_store_relaxed(&lockstat_enabled,				    \
 	    lockstat_dev_enabled | KDTRACE_LOCKSTAT_ENABLED);		    \
 	__cpu_simple_unlock(&lockstat_enabled_lock);			    \
 } while (/*CONSTCOND*/0)
 #endif
 #endif	/* _SYS_LOCKSTAT_H_ */