 @@ -1,17 +1,17 @@
-/*	$NetBSD: kern_lock.c,v 1.147 2008/11/12 12:36:16 ad Exp $	*/
+/*	$NetBSD: kern_lock.c,v 1.148 2009/05/23 17:08:04 ad Exp $	*/
 /*-
- * Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc.
+ * Copyright (c) 2002, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
  * NASA Ames Research Center, and by Andrew Doran.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
 @@ -21,69 +21,81 @@
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.147 2008/11/12 12:36:16 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.148 2009/05/23 17:08:04 ad Exp $");
 #include <sys/param.h>
 #include <sys/proc.h>
 #include <sys/lock.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/lockdebug.h>
 #include <sys/cpu.h>
 #include <sys/syslog.h>
 #include <sys/atomic.h>
 #include <sys/lwp.h>
 #include <machine/stdarg.h>
 #include <machine/lock.h>
 #include <dev/lockstat.h>
 #define	RETURN_ADDRESS	(uintptr_t)__builtin_return_address(0)
 bool	kernel_lock_dodebug;
 __cpu_simple_lock_t kernel_lock[CACHE_LINE_SIZE / sizeof(__cpu_simple_lock_t)]
     __aligned(CACHE_LINE_SIZE);
 void
 assert_sleepable(void)
+{
 	const char *reason;
 	uint64_t pctr;
 	bool idle;
 	if (panicstr != NULL) {
 		return;
+	}
 	LOCKDEBUG_BARRIER(kernel_lock, 1);
 	/*
 	 * Avoid disabling/re-enabling preemption here since this
 	 * routine may be called in delicate situatations.
 	 */
 	do {
 		pctr = lwp_pctr();
 		idle = CURCPU_IDLE_P();
 	} while (pctr != lwp_pctr());
 	reason = NULL;
-	if (CURCPU_IDLE_P() && !cold) {
+	if (idle && !cold) {
 		reason = "idle";
+	}
 	if (cpu_intr_p()) {
 		reason = "interrupt";
+	}
-	if ((curlwp->l_pflag & LP_INTR) != 0) {
+	if (cpu_softintr_p()) {
 		reason = "softint";
+	}
 	if (reason) {
 		panic("%s: %s caller=%p", __func__, reason,
 		    (void *)RETURN_ADDRESS);
+	}
+}
 /*
  * Functions for manipulating the kernel_lock.  We put them here
  * so that they show up in profiles.
  */

 @@ -1,19 +1,22 @@
-/* $NetBSD: kern_tc.c,v 1.38 2009/01/11 02:45:52 christos Exp $ */
+/* $NetBSD: kern_tc.c,v 1.39 2009/05/23 17:08:04 ad Exp $ */
 /*-
- * Copyright (c) 2008 The NetBSD Foundation, Inc.
+ * Copyright (c) 2008, 2009 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Andrew Doran.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 @@ -27,43 +30,44 @@
  */
 /*-
  * ----------------------------------------------------------------------------
  * "THE BEER-WARE LICENSE" (Revision 42):
  * <phk@FreeBSD.ORG> wrote this file.  As long as you retain this notice you
  * can do whatever you want with this stuff. If we meet some day, and you think
  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
  * ---------------------------------------------------------------------------
  */
 #include <sys/cdefs.h>
 /* __FBSDID("$FreeBSD: src/sys/kern/kern_tc.c,v 1.166 2005/09/19 22:16:31 andre Exp $"); */
-__KERNEL_RCSID(0, "$NetBSD: kern_tc.c,v 1.38 2009/01/11 02:45:52 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_tc.c,v 1.39 2009/05/23 17:08:04 ad Exp $");
 #include "opt_ntp.h"
 #include <sys/param.h>
 #include <sys/kernel.h>
 #include <sys/reboot.h>	/* XXX just to get AB_VERBOSE */
 #include <sys/sysctl.h>
 #include <sys/syslog.h>
 #include <sys/systm.h>
 #include <sys/timepps.h>
 #include <sys/timetc.h>
 #include <sys/timex.h>
 #include <sys/evcnt.h>
 #include <sys/kauth.h>
 #include <sys/mutex.h>
 #include <sys/atomic.h>
 #include <sys/xcall.h>
 /*
  * A large step happens on boot.  This constant detects such steps.
  * It is relatively small so that ntp_update_second gets called enough
  * in the typical 'missed a couple of seconds' case, but doesn't loop
  * forever when the time step is large.
  */
 #define LARGE_STEP	200
 /*
  * Implement a dummy timecounter which we can use until we get a real one
  * in the air.  This allows the console and other early stuff to use
  * time services.
 @@ -116,26 +120,27 @@ static struct timehands th0 = {
 static struct timehands *volatile timehands = &th0;
 struct timecounter *timecounter = &dummy_timecounter;
 static struct timecounter *timecounters = &dummy_timecounter;
 time_t time_second = 1;
 time_t time_uptime = 1;
 static struct bintime timebasebin;
 static int timestepwarnings;
 kmutex_t timecounter_lock;
 static u_int timecounter_mods;
 static volatile int timecounter_removals = 1;
 static u_int timecounter_bad;
 #ifdef __FreeBSD__
 SYSCTL_INT(_kern_timecounter, OID_AUTO, stepwarnings, CTLFLAG_RW,
     &timestepwarnings, 0, "");
 #endif /* __FreeBSD__ */
 /*
  * sysctl helper routine for kern.timercounter.hardware
  */
 static int
 sysctl_kern_timecounter_hardware(SYSCTLFN_ARGS)
+{
 @@ -299,35 +304,69 @@ tc_delta(struct timehands *th)
 		 th->th_offset_count) & tc->tc_counter_mask);
+}
 /*
  * Functions for reading the time.  We have to loop until we are sure that
  * the timehands that we operated on was not updated under our feet.  See
  * the comment in <sys/timevar.h> for a description of these 12 functions.
  */
 void
 binuptime(struct bintime *bt)
+{
 	struct timehands *th;
-	u_int gen;
+	lwp_t *l;
 	u_int lgen, gen;
 	TC_COUNT(nbinuptime);
 	/*
 	 * Provide exclusion against tc_detach().
+	 *
 	 * We record the number of timecounter removals before accessing
 	 * timecounter state.  Note that the LWP can be using multiple
 	 * "generations" at once, due to interrupts (interrupted while in
 	 * this function).  Hardware interrupts will borrow the interrupted
 	 * LWP's l_tcgen value for this purpose, and can themselves be
 	 * interrupted by higher priority interrupts.  In this case we need
 	 * to ensure that the oldest generation in use is recorded.
+	 *
 	 * splsched() is too expensive to use, so we take care to structure
 	 * this code in such a way that it is not required.  Likewise, we
 	 * do not disable preemption.
+	 *
 	 * Memory barriers are also too expensive to use for such a
 	 * performance critical function.  The good news is that we do not
 	 * need memory barriers for this type of exclusion, as the thread
 	 * updating timecounter_removals will issue a broadcast cross call
 	 * before inspecting our l_tcgen value (this elides memory ordering
 	 * issues).
 	 */
 	l = curlwp;
 	lgen = l->l_tcgen;
 	if (__predict_true(lgen == 0)) {
 		l->l_tcgen = timecounter_removals;
+	}
 	__insn_barrier();
 	do {
 		th = timehands;
 		gen = th->th_generation;
 		*bt = th->th_offset;
 		bintime_addx(bt, th->th_scale * tc_delta(th));
 	} while (gen == 0 || gen != th->th_generation);
 	__insn_barrier();
 	l->l_tcgen = lgen;
+}
 void
 nanouptime(struct timespec *tsp)
+{
 	struct bintime bt;
 	TC_COUNT(nnanouptime);
 	binuptime(&bt);
 	bintime2timespec(&bt, tsp);
+}
 void
 @@ -533,47 +572,95 @@ tc_gonebad(struct timecounter *tc)
 	tc->tc_quality = -100;
 	membar_producer();
 	atomic_inc_uint(&timecounter_bad);
+}
 /*
  * Stop using a timecounter and remove it from the timecounters list.
  */
 int
 tc_detach(struct timecounter *target)
+{
 	struct timecounter *tc;
 	struct timecounter **tcp = NULL;
-	int rc = 0;
+	int removals;
 	uint64_t where;
 	lwp_t *l;
 	/* First, find the timecounter. */
 	mutex_spin_enter(&timecounter_lock);
 	for (tcp = &timecounters, tc = timecounters;
 	     tc != NULL;
 	     tcp = &tc->tc_next, tc = tc->tc_next) {
 		if (tc == target)
 			break;
+	}
 	if (tc == NULL) {
-		rc = ESRCH;
+		mutex_spin_exit(&timecounter_lock);
-	} else {
+		return ESRCH;
 		*tcp = tc->tc_next;
 		if (timecounter == target) {
-			tc_pick();
+	/* And now, remove it. */
-			tc_windup();
+	*tcp = tc->tc_next;
 	if (timecounter == target) {
-		timecounter_mods++;
+		tc_pick();
 		tc_windup();
+	}
 	timecounter_mods++;
 	removals = timecounter_removals++;
 	mutex_spin_exit(&timecounter_lock);
 	return rc;
 	/*
 	 * We now have to determine if any threads in the system are still
 	 * making use of this timecounter.
+	 *
 	 * We issue a broadcast cross call to elide memory ordering issues,
 	 * then scan all LWPs in the system looking at each's timecounter
 	 * generation number.  We need to see a value of zero (not actively
 	 * using a timecounter) or a value greater than our removal value.
+	 *
 	 * We may race with threads that read `timecounter_removals' and
 	 * and then get preempted before updating `l_tcgen'.  This is not
 	 * a problem, since it means that these threads have not yet started
 	 * accessing timecounter state.  All we do need is one clean
 	 * snapshot of the system where every thread appears not to be using
 	 * old timecounter state.
 	 */
 	for (;;) {
 		where = xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL);
 		xc_wait(where);
 		mutex_enter(proc_lock);
 		LIST_FOREACH(l, &alllwp, l_list) {
 			if (l->l_tcgen == 0 || l->l_tcgen > removals) {
 				/*
 				 * Not using timecounter or old timecounter
 				 * state at time of our xcall or later.
 				 */
 				continue;
+			}
 			break;
+		}
 		mutex_exit(proc_lock);
 		/*
 		 * If the timecounter is still in use, wait at least 10ms
 		 * before retrying.
 		 */
 		if (l == NULL) {
 			return 0;
+		}
 		(void)kpause("tcdetach", false, mstohz(10), NULL);
+	}
+}
 /* Report the frequency of the current timecounter. */
 u_int64_t
 tc_getfrequency(void)
+{
 	return (timehands->th_counter->tc_frequency);
+}
 /*
  * Step our concept of UTC.  This is done by modifying our estimate of
  * when we booted.

 @@ -1,14 +1,14 @@
-/*	$NetBSD: lwp.h,v 1.117 2009/02/04 21:17:39 ad Exp $	*/
+/*	$NetBSD: lwp.h,v 1.118 2009/05/23 17:08:05 ad Exp $	*/
 /*-
  * Copyright (c) 2001, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Nathan J. Williams and Andrew Doran.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -163,26 +163,28 @@ struct lwp {
 	u_short		l_exlocks;	/* !: lockdebug: excl. locks held */
 	u_short		l_unused;	/* !: unused */
 	u_short		l_blcnt;	/* !: count of kernel_lock held */
 	int		l_nopreempt;	/* !: don't preempt me! */
 	u_int		l_dopreempt;	/* s: kernel preemption pending */
 	int		l_pflag;	/* !: LWP private flags */
 	int		l_dupfd;	/* !: side return from cloning devs XXX */
 	const struct sysent * volatile l_sysent;/* !: currently active syscall */
 	struct rusage	l_ru;		/* !: accounting information */
 	uint64_t	l_pfailtime;	/* !: for kernel preemption */
 	uintptr_t	l_pfailaddr;	/* !: for kernel preemption */
 	uintptr_t	l_pfaillock;	/* !: for kernel preemption */
 	_TAILQ_HEAD(,struct lockdebug,volatile) l_ld_locks;/* !: locks held by LWP */
 	int		l_tcgen;	/* !: for timecounter removal */
 	int		l_unused2;	/* !: for future use */
 	/* These are only used by 'options SYSCALL_TIMES' */
 	uint32_t        l_syscall_time; /* !: time epoch for current syscall */
 	uint64_t        *l_syscall_counter; /* !: counter for current process */
 };
 #if !defined(USER_TO_UAREA)
 #if !defined(UAREA_USER_OFFSET)
 #define	UAREA_USER_OFFSET	0
 #endif /* !defined(UAREA_USER_OFFSET) */
 #define	USER_TO_UAREA(user)	((vaddr_t)(user) - UAREA_USER_OFFSET)
 #define	UAREA_TO_USER(uarea)	((struct user *)((uarea) + UAREA_USER_OFFSET))
 #endif /* !defined(UAREA_TO_USER) */
 @@ -296,26 +298,27 @@ void	upcallret(lwp_t *);
 void	lwp_exit(lwp_t *) __dead;
 void	lwp_exit_switchaway(lwp_t *) __dead;
 int	lwp_suspend(lwp_t *, lwp_t *);
 int	lwp_create1(lwp_t *, const void *, size_t, u_long, lwpid_t *);
 void	lwp_update_creds(lwp_t *);
 void	lwp_migrate(lwp_t *, struct cpu_info *);
 lwp_t *lwp_find2(pid_t, lwpid_t);
 lwp_t *lwp_find(proc_t *, int);
 void	lwp_userret(lwp_t *);
 void	lwp_need_userret(lwp_t *);
 void	lwp_free(lwp_t *, bool, bool);
 void	lwp_sys_init(void);
 u_int	lwp_unsleep(lwp_t *, bool);
 uint64_t lwp_pctr(void);
 int	lwp_specific_key_create(specificdata_key_t *, specificdata_dtor_t);
 void	lwp_specific_key_delete(specificdata_key_t);
 void 	lwp_initspecific(lwp_t *);
 void 	lwp_finispecific(lwp_t *);
 void	*lwp_getspecific(specificdata_key_t);
 #if defined(_LWP_API_PRIVATE)
 void	*_lwp_getspecific_by_lwp(lwp_t *, specificdata_key_t);
 #endif
 void	lwp_setspecific(specificdata_key_t, void *);
 /* Syscalls */
 int	lwp_park(struct timespec *, const void *);