 @@ -1,268 +1,269 @@
-/*	$NetBSD: subr_pserialize.c,v 1.1 2011/07/30 17:01:04 christos Exp $	*/
+/*	$NetBSD: subr_pserialize.c,v 1.2 2011/08/01 15:26:31 he Exp $	*/
 /*-
  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 /*
  * Passive serialization.
+ *
  * Implementation accurately matches the lapsed US patent 4809168, therefore
  * code is patent-free in the United States.  Your use of this code is at
  * your own risk.
+ *
  * Note for NetBSD developers: all changes to this source file must be
  * approved by the <core>.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_pserialize.c,v 1.1 2011/07/30 17:01:04 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_pserialize.c,v 1.2 2011/08/01 15:26:31 he Exp $");
 #include <sys/param.h>
 #include <sys/condvar.h>
 #include <sys/cpu.h>
 #include <sys/evcnt.h>
 #include <sys/kmem.h>
 #include <sys/mutex.h>
 #include <sys/pserialize.h>
 #include <sys/queue.h>
 #include <sys/xcall.h>
 struct pserialize {
 	TAILQ_ENTRY(pserialize)	psz_chain;
 	lwp_t *			psz_owner;
 	kcondvar_t		psz_notifier;
 	kcpuset_t *		psz_target;
 	kcpuset_t *		psz_pass;
 };
 static u_int			psz_work_todo	__cacheline_aligned;
 static kmutex_t			psz_lock	__cacheline_aligned;
 static struct evcnt		psz_ev_excl	__cacheline_aligned;
 /*
  * As defined in "Method 1":
  *	q0: "0 MP checkpoints have occured".
  *	q1: "1 MP checkpoint has occured".
  *	q2: "2 MP checkpoints have occured".
  */
 static TAILQ_HEAD(, pserialize)	psz_queue0	__cacheline_aligned;
 static TAILQ_HEAD(, pserialize)	psz_queue1	__cacheline_aligned;
 static TAILQ_HEAD(, pserialize)	psz_queue2	__cacheline_aligned;
 /*
  * pserialize_init:
+ *
  *	Initialize passive serialization structures.
  */
 void
 pserialize_init(void)
+{
 	psz_work_todo = 0;
 	TAILQ_INIT(&psz_queue0);
 	TAILQ_INIT(&psz_queue1);
 	TAILQ_INIT(&psz_queue2);
 	mutex_init(&psz_lock, MUTEX_DEFAULT, IPL_SCHED);
 	evcnt_attach_dynamic(&psz_ev_excl, EVCNT_TYPE_MISC, NULL,
 	    "pserialize", "exclusive access");
+}
 /*
  * pserialize_create:
+ *
  *	Create and initialize a passive serialization object.
  */
 pserialize_t
 pserialize_create(void)
+{
 	pserialize_t psz;
 	psz = kmem_zalloc(sizeof(struct pserialize), KM_SLEEP);
 	cv_init(&psz->psz_notifier, "psrlz");
 	psz->psz_target = kcpuset_create();
 	psz->psz_pass = kcpuset_create();
 	psz->psz_owner = NULL;
 	return psz;
+}
 /*
  * pserialize_destroy:
+ *
  *	Destroy a passive serialization object.
  */
 void
 pserialize_destroy(pserialize_t psz)
+{
 	KASSERT(psz->psz_owner == NULL);
 	cv_destroy(&psz->psz_notifier);
 	kcpuset_destroy(psz->psz_target);
 	kcpuset_destroy(psz->psz_pass);
 	kmem_free(psz, sizeof(struct pserialize));
+}
 /*
  * pserialize_perform:
+ *
  *	Perform the write side of passive serialization.  The calling
  *	thread holds an exclusive lock on the data object(s) being updated.
  *	We wait until every processor in the system has made at least two
  *	passes through cpu_swichto().  The wait is made with the caller's
  *	update lock held, but is short term.
  */
 void
 pserialize_perform(pserialize_t psz)
+{
 	KASSERT(!cpu_intr_p());
 	KASSERT(!cpu_softintr_p());
 	if (__predict_false(panicstr != NULL)) {
 		return;
+	}
 	KASSERT(psz->psz_owner == NULL);
 	KASSERT(kcpuset_iszero(psz->psz_target));
 	KASSERT(ncpu > 0);
 	/*
 	 * Set up the object and put it onto the queue.  The lock
 	 * activity here provides the necessary memory barrier to
 	 * make the caller's data update completely visible to
 	 * other processors.
 	 */
 	psz->psz_owner = curlwp;
 	kcpuset_fill(psz->psz_target);
 	kcpuset_zero(psz->psz_pass);
 	mutex_spin_enter(&psz_lock);
 	TAILQ_INSERT_TAIL(&psz_queue0, psz, psz_chain);
 	psz_work_todo++;
 	mutex_spin_exit(&psz_lock);
 	/*
 	 * Force some context switch activity on every CPU, as the system
 	 * may not be busy.  Note: should pass the point twice.
 	 */
 	xc_broadcast(XC_HIGHPRI, (xcfunc_t)nullop, NULL, NULL);
 	xc_broadcast(XC_HIGHPRI, (xcfunc_t)nullop, NULL, NULL);
 	/*
 	 * Wait for all CPUs to cycle through mi_switch() twice.
 	 * The last one through will remove our update from the
 	 * queue and awaken us.
 	 */
 	mutex_spin_enter(&psz_lock);
 	while (!kcpuset_iszero(psz->psz_target)) {
 		cv_wait(&psz->psz_notifier, &psz_lock);
+	}
 	psz_ev_excl.ev_count++;
 	mutex_spin_exit(&psz_lock);
 	psz->psz_owner = NULL;
+}
 int
 pserialize_read_enter(void)
+{
 	KASSERT(!cpu_intr_p());
 	return splsoftclock();
+}
 void
 pserialize_read_exit(int s)
+{
 	splx(s);
+}
 /*
  * pserialize_switchpoint:
+ *
  *	Monitor system context switch activity.  Called from machine
  *	independent code after mi_switch() returns.
  */
 void
 pserialize_switchpoint(void)
+{
 	pserialize_t psz, next;
 	cpuid_t cid;
 	/*
 	 * If no updates pending, bail out.  No need to lock in order to
 	 * test psz_work_todo; the only ill effect of missing an update
 	 * would be to delay LWPs waiting in pserialize_perform().  That
 	 * will not happen because updates are on the queue before an
 	 * xcall is generated (serialization) to tickle every CPU.
 	 */
 	if (__predict_true(psz_work_todo == 0)) {
 		return;
+	}
 	mutex_spin_enter(&psz_lock);
 	cid = cpu_index(curcpu());
 	/*
 	 * At first, scan through the second queue and update each request,
 	 * if passed all processors, then transfer to the third queue.
 	 */
 	for (psz = TAILQ_FIRST(&psz_queue1); psz != NULL; psz = next) {
 		next = TAILQ_NEXT(psz, psz_chain);
 		if (!kcpuset_match(psz->psz_pass, psz->psz_target)) {
 			kcpuset_set(cid, psz->psz_pass);
 			continue;
+		}
 		kcpuset_zero(psz->psz_pass);
 		TAILQ_REMOVE(&psz_queue1, psz, psz_chain);
 		TAILQ_INSERT_TAIL(&psz_queue2, psz, psz_chain);
+	}
 	/*
 	 * Scan through the first queue and update each request,
 	 * if passed all processors, then move to the second queue.
 	 */
 	for (psz = TAILQ_FIRST(&psz_queue0); psz != NULL; psz = next) {
 		next = TAILQ_NEXT(psz, psz_chain);
 		if (!kcpuset_match(psz->psz_pass, psz->psz_target)) {
 			kcpuset_set(cid, psz->psz_pass);
 			continue;
+		}
 		kcpuset_zero(psz->psz_pass);
 		TAILQ_REMOVE(&psz_queue0, psz, psz_chain);
 		TAILQ_INSERT_TAIL(&psz_queue1, psz, psz_chain);
+	}
 	/*
 	 * Process the third queue: entries have been seen twice on every
 	 * processor, remove from the queue and notify the updating thread.
 	 */
 	while ((psz = TAILQ_FIRST(&psz_queue2)) != NULL) {
 		TAILQ_REMOVE(&psz_queue2, psz, psz_chain);
 		kcpuset_zero(psz->psz_target);
 		cv_signal(&psz->psz_notifier);
 		psz_work_todo--;
+	}
 	mutex_spin_exit(&psz_lock);
+}