 @@ -1,14 +1,14 @@
-/*	$NetBSD: hypervisor_machdep.c,v 1.17 2011/11/19 17:13:39 cherry Exp $	*/
+/*	$NetBSD: hypervisor_machdep.c,v 1.18 2011/12/03 22:41:40 bouyer Exp $	*/
 /*
+ *
  * Copyright (c) 2004 Christian Limpach.
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
 @@ -44,27 +44,27 @@
  * all copies or substantial portions of the Software.
+ *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  * DEALINGS IN THE SOFTWARE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: hypervisor_machdep.c,v 1.17 2011/11/19 17:13:39 cherry Exp $");
+__KERNEL_RCSID(0, "$NetBSD: hypervisor_machdep.c,v 1.18 2011/12/03 22:41:40 bouyer Exp $");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kmem.h>
 #include <uvm/uvm_extern.h>
 #include <machine/vmparam.h>
 #include <machine/pmap.h>
 #include <xen/xen.h>
 #include <xen/hypervisor.h>
 #include <xen/evtchn.h>
 @@ -295,54 +295,74 @@ hypervisor_send_event(struct cpu_info *c
 #endif
 	xen_atomic_set_bit(&s->evtchn_pending[0], ev);
 	xen_atomic_set_bit(&vci->evtchn_pending_sel,
 			   ev >> LONG_SHIFT);
 	xen_atomic_set_bit(&vci->evtchn_upcall_pending, 0);
 	xen_atomic_clear_bit(&s->evtchn_mask[0], ev);
 	if (__predict_true(ci == curcpu())) {
 		hypervisor_force_callback();
 	} else {
-		if (xen_send_ipi(ci, XEN_IPI_HVCB)) {
+		if (__predict_false(xen_send_ipi(ci, XEN_IPI_HVCB))) {
-			panic("xen_send_ipi(cpu%d, XEN_IPI_HVCB) failed\n", (int) ci->ci_cpuid);
+			panic("xen_send_ipi(cpu%d, XEN_IPI_HVCB) failed\n",
 			    (int) ci->ci_cpuid);
+		}
+	}
+}
 void
 hypervisor_unmask_event(unsigned int ev)
+{
 	volatile shared_info_t *s = HYPERVISOR_shared_info;
-	volatile struct vcpu_info *vci = curcpu()->ci_vcpu;
+	CPU_INFO_ITERATOR cii;
 	struct cpu_info *ci;
 	volatile struct vcpu_info *vci;
 #ifdef PORT_DEBUG
 	if (ev == PORT_DEBUG)
 		printf("hypervisor_unmask_event %d\n", ev);
 #endif
 	xen_atomic_clear_bit(&s->evtchn_mask[0], ev);
 	/*
 	 * The following is basically the equivalent of
 	 * 'hw_resend_irq'. Just like a real IO-APIC we 'lose the
 	 * interrupt edge' if the channel is masked.
 	 */
-	if (xen_atomic_test_bit(&s->evtchn_pending[0], ev) &&
+	if (!xen_atomic_test_bit(&s->evtchn_pending[0], ev))
-	    !xen_atomic_test_and_set_bit(&vci->evtchn_pending_sel, ev>>LONG_SHIFT)) {
+		return;
 		xen_atomic_set_bit(&vci->evtchn_upcall_pending, 0);
-		if (!vci->evtchn_upcall_mask)
+	for (CPU_INFO_FOREACH(cii, ci)) {
-			hypervisor_force_callback();
+		if (!xen_atomic_test_bit(&ci->ci_evtmask[0], ev))
 			continue;
 		vci = ci->ci_vcpu;
 		if (!xen_atomic_test_and_set_bit(&vci->evtchn_pending_sel,
 		    ev>>LONG_SHIFT))
 			xen_atomic_set_bit(&vci->evtchn_upcall_pending, 0);
 		if (!vci->evtchn_upcall_mask) {
 			if (__predict_true(ci == curcpu())) {
 				hypervisor_force_callback();
 			} else {
 				if (__predict_false(
 				    xen_send_ipi(ci, XEN_IPI_HVCB))) {
 					panic("xen_send_ipi(cpu%d, "
 					    "XEN_IPI_HVCB) failed\n",
 					    (int) ci->ci_cpuid);
+				}
+			}
+		}
+	}
+}
 void
 hypervisor_mask_event(unsigned int ev)
+{
 	volatile shared_info_t *s = HYPERVISOR_shared_info;
 #ifdef PORT_DEBUG
 	if (ev == PORT_DEBUG)
 		printf("hypervisor_mask_event %d\n", ev);
 #endif
 	xen_atomic_set_bit(&s->evtchn_mask[0], ev);

 @@ -1,14 +1,14 @@
-/*      $NetBSD: xenevt.c,v 1.38 2011/08/11 17:59:00 cherry Exp $      */
+/*      $NetBSD: xenevt.c,v 1.39 2011/12/03 22:41:40 bouyer Exp $      */
 /*
  * Copyright (c) 2005 Manuel Bouyer.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
 @@ -16,27 +16,27 @@
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: xenevt.c,v 1.38 2011/08/11 17:59:00 cherry Exp $");
+__KERNEL_RCSID(0, "$NetBSD: xenevt.c,v 1.39 2011/12/03 22:41:40 bouyer Exp $");
 #include "opt_xen.h"
 #include <sys/param.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/systm.h>
 #include <sys/device.h>
 #include <sys/file.h>
 #include <sys/filedesc.h>
 #include <sys/poll.h>
 #include <sys/select.h>
 #include <sys/proc.h>
 @@ -102,39 +102,41 @@ const struct cdevsw xenevt_cdevsw = {
 #define BYTES_PER_PORT (sizeof(evtchn_port_t) / sizeof(uint8_t))
 struct xenevt_d {
 	kmutex_t lock;
 	kcondvar_t cv;
 	STAILQ_ENTRY(xenevt_d) pendingq;
 	bool pending;
 	evtchn_port_t ring[2048];
 	u_int ring_read; /* pointer of the reader */
 	u_int ring_write; /* pointer of the writer */
 	u_int flags;
 #define XENEVT_F_OVERFLOW 0x01 /* ring overflow */
 #define XENEVT_F_FREE 0x02 /* free entry */
 	struct selinfo sel; /* used by poll */
 	struct cpu_info *ci; /* prefered CPU for events for this device */
 };
 /* event -> user device mapping */
 static struct xenevt_d *devevent[NR_EVENT_CHANNELS];
 /* pending events */
 static void *devevent_sih;
 static kmutex_t devevent_lock;
 static STAILQ_HEAD(, xenevt_d) devevent_pending;
 static void xenevt_donotify(struct xenevt_d *);
 static void xenevt_record(struct xenevt_d *, evtchn_port_t);
 static void xenevt_free(struct xenevt_d *);
 /* pending events */
 long xenevt_ev1;
 long xenevt_ev2[NR_EVENT_CHANNELS];
 static int xenevt_processevt(void *);
 /* called at boot time */
 void
 xenevtattach(int n)
+{
 	struct intrhand *ih;
 	int s;
 	int level = IPL_HIGH;
 @@ -150,46 +152,46 @@ xenevtattach(int n)
 	memset(devevent, 0, sizeof(devevent));
 	xenevt_ev1 = 0;
 	memset(xenevt_ev2, 0, sizeof(xenevt_ev2));
 	/* register a handler at splhigh, so that spllower() will call us */
 	ih = malloc(sizeof (struct intrhand), M_DEVBUF,
 	     M_WAITOK|M_ZERO);
 	if (ih == NULL)
 		panic("can't allocate xenevt interrupt source");
 	ih->ih_level = level;
 	ih->ih_fun = ih->ih_realfun = xenevt_processevt;
 	ih->ih_arg = ih->ih_realarg = NULL;
 	ih->ih_ipl_next = NULL;
-	ih->ih_cpu = curcpu();
+	ih->ih_cpu = &cpu_info_primary;
 #ifdef MULTIPROCESSOR
 	if (!mpsafe) {
 		ih->ih_fun = intr_biglock_wrapper;
 		ih->ih_arg = ih;
+	}
 #endif /* MULTIPROCESSOR */
 	s = splhigh();
 	event_set_iplhandler(ih->ih_cpu, ih, level);
 	splx(s);
+}
 /* register pending event - always called with interrupt disabled */
 void
 xenevt_setipending(int l1, int l2)
+{
-	xenevt_ev1 |= 1UL << l1;
+	atomic_or_ulong(&xenevt_ev1, 1UL << l1);
-	xenevt_ev2[l1] |= 1UL << l2;
+	atomic_or_ulong(&xenevt_ev2[l1], 1UL << l2);
-	curcpu()/*XXX*/->ci_ipending |= 1 << IPL_HIGH;
+	atomic_or_32(&cpu_info_primary.ci_ipending, 1 << IPL_HIGH);
+}
 /* process pending events */
 static int
 xenevt_processevt(void *v)
+{
 	long l1, l2;
 	int l1i, l2i;
 	int port;
 	l1 = xen_atomic_xchg(&xenevt_ev1, 0);
 	while ((l1i = xen_ffs(l1)) != 0) {
 		l1i--;
 @@ -203,109 +205,108 @@ xenevt_processevt(void *v)
+		}
+	}
 	return 0;
+}
 /* event callback, called at splhigh() */
 void
 xenevt_event(int port)
+{
 	struct xenevt_d *d;
 	mutex_enter(&devevent_lock);
 	d = devevent[port];
 	if (d != NULL) {
 		xenevt_record(d, port);
-		if (d->pending) {
+		if (d->pending == false) {
 			STAILQ_INSERT_TAIL(&devevent_pending, d, pendingq);
 			d->pending = true;
 			mutex_exit(&devevent_lock);
 			softint_schedule(devevent_sih);
 			return;
+		}
 		mutex_enter(&devevent_lock);
 		STAILQ_INSERT_TAIL(&devevent_pending, d, pendingq);
 		d->pending = true;
 		mutex_exit(&devevent_lock);
 		softint_schedule(devevent_sih);
+	}
 	mutex_exit(&devevent_lock);
+}
 void
 xenevt_notify(void)
+{
 	struct xenevt_d *d;
 	for (;;) {
 		mutex_enter(&devevent_lock);
 		d = STAILQ_FIRST(&devevent_pending);
 		if (d == NULL) {
 			mutex_exit(&devevent_lock);
 			break;
+		}
 		STAILQ_REMOVE_HEAD(&devevent_pending, pendingq);
 		d->pending = false;
-		mutex_exit(&devevent_lock);
+		mutex_enter(&d->lock);
 		if (d->flags & XENEVT_F_FREE) {
-		xenevt_donotify(d);
+			xenevt_free(d);
 			mutex_exit(&devevent_lock);
 		} else {
 			mutex_exit(&devevent_lock);
 			selnotify(&d->sel, 0, 1);
 			cv_broadcast(&d->cv);
 			mutex_exit(&d->lock);
+		}
+	}
+}
 static void
 xenevt_donotify(struct xenevt_d *d)
 	mutex_enter(&d->lock);
 	selnotify(&d->sel, 0, 1);
 	cv_broadcast(&d->cv);
 	mutex_exit(&d->lock);
 static void
 xenevt_record(struct xenevt_d *d, evtchn_port_t port)
+{
 	/*
 	 * This algorithm overflows for one less slot than available.
 	 * Not really an issue, and the correct algorithm would be more
 	 * complex
 	 */
 	mutex_enter(&d->lock);
 	if (d->ring_read ==
 	    ((d->ring_write + 1) & XENEVT_RING_MASK)) {
 		d->flags |= XENEVT_F_OVERFLOW;
 		printf("xenevt_event: ring overflow port %d\n", port);
 	} else {
 		d->ring[d->ring_write] = port;
 		d->ring_write = (d->ring_write + 1) & XENEVT_RING_MASK;
+	}
 	mutex_exit(&d->lock);
+}
 /* open the xenevt device; this is where we clone */
 int
 xenevtopen(dev_t dev, int flags, int mode, struct lwp *l)
+{
 	struct xenevt_d *d;
 	struct file *fp;
 	int fd, error;
 	switch(minor(dev)) {
 	case DEV_EVT:
 		/* falloc() will use the descriptor for us. */
 		if ((error = fd_allocfile(&fp, &fd)) != 0)
 			return error;
 		d = malloc(sizeof(*d), M_DEVBUF, M_WAITOK | M_ZERO);
-		mutex_init(&d->lock, MUTEX_DEFAULT, IPL_SOFTSERIAL);
+		d->ci = &cpu_info_primary;
 		mutex_init(&d->lock, MUTEX_DEFAULT, IPL_HIGH);
 		cv_init(&d->cv, "xenevt");
 		selinit(&d->sel);
 		return fd_clone(fp, fd, flags, &xenevt_fileops, d);
 	case DEV_XSD:
 		/* no clone for /dev/xsd_kva */
 		return (0);
 	default:
 		break;
+	}
 	return ENODEV;
+}
 /* read from device: only for /dev/xsd_kva, xenevt is done though fread */
 @@ -340,54 +341,73 @@ xenevtread(dev_t dev, struct uio *uio, i
 paddr_t
 xenevtmmap(dev_t dev, off_t off, int prot)
+{
 	if (minor(dev) == DEV_XSD) {
 		/* only one page, so off is always 0 */
 		if (off != 0)
 			return -1;
 		return x86_btop(
 		   xpmap_mtop((paddr_t)xen_start_info.store_mfn << PAGE_SHIFT));
+	}
 	return -1;
+}
-static int
+static void
-xenevt_fclose(struct file *fp)
+xenevt_free(struct xenevt_d *d)
+{
 	struct xenevt_d *d = fp->f_data;
 	int i;
 	KASSERT(mutex_owned(&devevent_lock));
 	KASSERT(mutex_owned(&d->lock));
 	for (i = 0; i < NR_EVENT_CHANNELS; i++ ) {
 		if (devevent[i] == d) {
 			evtchn_op_t op = { .cmd = 0 };
 			int error;
 			hypervisor_mask_event(i);
 			xen_atomic_clear_bit(&d->ci->ci_evtmask[0], i);
 			devevent[i] = NULL;
 			op.cmd = EVTCHNOP_close;
 			op.u.close.port = i;
 			if ((error = HYPERVISOR_event_channel_op(&op))) {
 				printf("xenevt_fclose: error %d from "
 				    "hypervisor\n", -error);
+			}
+		}
+	}
 	mutex_exit(&d->lock);
 	seldestroy(&d->sel);
 	cv_destroy(&d->cv);
 	mutex_destroy(&d->lock);
 	fp->f_data = NULL;
 	free(d, M_DEVBUF);
+}
 static int
 xenevt_fclose(struct file *fp)
+{
 	struct xenevt_d *d = fp->f_data;
 	mutex_enter(&devevent_lock);
 	mutex_enter(&d->lock);
 	if (d->pending) {
 		d->flags |= XENEVT_F_FREE;
 		mutex_exit(&d->lock);
 	} else {
 		xenevt_free(d);
+	}
 	mutex_exit(&devevent_lock);
 	fp->f_data = NULL;
 	return (0);
+}
 static int
 xenevt_fread(struct file *fp, off_t *offp, struct uio *uio,
     kauth_cred_t cred, int flags)
+{
 	struct xenevt_d *d = fp->f_data;
 	int error, ring_read, ring_write;
 	size_t len, uio_len;
 	error = 0;
 	mutex_enter(&d->lock);
 @@ -457,116 +477,141 @@ xenevt_fwrite(struct file *fp, off_t *of
 	int i, nentries, error;
 	if (uio->uio_resid == 0)
 		return (0);
 	nentries = uio->uio_resid / sizeof(uint16_t);
 	if (nentries > NR_EVENT_CHANNELS)
 		return EMSGSIZE;
 	chans = kmem_alloc(nentries * sizeof(uint16_t), KM_SLEEP);
 	if (chans == NULL)
 		return ENOMEM;
 	error = uiomove(chans, uio->uio_resid, uio);
 	if (error)
 		goto out;
 	mutex_enter(&devevent_lock);
 	for (i = 0; i < nentries; i++) {
 		if (chans[i] < NR_EVENT_CHANNELS &&
 		    devevent[chans[i]] == d) {
 			hypervisor_unmask_event(chans[i]);
+		}
+	}
 	mutex_exit(&devevent_lock);
 out:
 	kmem_free(chans, nentries * sizeof(uint16_t));
 	return 0;
+}
 static int
 xenevt_fioctl(struct file *fp, u_long cmd, void *addr)
+{
 	struct xenevt_d *d = fp->f_data;
 	evtchn_op_t op = { .cmd = 0 };
 	int error;
 	switch(cmd) {
 	case EVTCHN_RESET:
 	case IOCTL_EVTCHN_RESET:
 		mutex_enter(&d->lock);
 		d->ring_read = d->ring_write = 0;
 		d->flags = 0;
 		mutex_exit(&d->lock);
 		break;
 	case IOCTL_EVTCHN_BIND_VIRQ:
+	{
 		struct ioctl_evtchn_bind_virq *bind_virq = addr;
 		op.cmd = EVTCHNOP_bind_virq;
 		op.u.bind_virq.virq = bind_virq->virq;
 		op.u.bind_virq.vcpu = 0;
 		if ((error = HYPERVISOR_event_channel_op(&op))) {
 			printf("IOCTL_EVTCHN_BIND_VIRQ failed: virq %d error %d\n", bind_virq->virq, error);
 			return -error;
+		}
 		bind_virq->port = op.u.bind_virq.port;
 		mutex_enter(&devevent_lock);
 		KASSERT(devevent[bind_virq->port] == NULL);
 		devevent[bind_virq->port] = d;
 		mutex_exit(&devevent_lock);
 		xen_atomic_set_bit(&d->ci->ci_evtmask[0], bind_virq->port);
 		hypervisor_unmask_event(bind_virq->port);
 		break;
+	}
 	case IOCTL_EVTCHN_BIND_INTERDOMAIN:
+	{
 		struct ioctl_evtchn_bind_interdomain *bind_intd = addr;
 		op.cmd = EVTCHNOP_bind_interdomain;
 		op.u.bind_interdomain.remote_dom = bind_intd->remote_domain;
 		op.u.bind_interdomain.remote_port = bind_intd->remote_port;
 		if ((error = HYPERVISOR_event_channel_op(&op)))
 			return -error;
 		bind_intd->port = op.u.bind_interdomain.local_port;
 		mutex_enter(&devevent_lock);
 		KASSERT(devevent[bind_intd->port] == NULL);
 		devevent[bind_intd->port] = d;
 		mutex_exit(&devevent_lock);
 		xen_atomic_set_bit(&d->ci->ci_evtmask[0], bind_intd->port);
 		hypervisor_unmask_event(bind_intd->port);
 		break;
+	}
 	case IOCTL_EVTCHN_BIND_UNBOUND_PORT:
+	{
 		struct ioctl_evtchn_bind_unbound_port *bind_unbound = addr;
 		op.cmd = EVTCHNOP_alloc_unbound;
 		op.u.alloc_unbound.dom = DOMID_SELF;
 		op.u.alloc_unbound.remote_dom = bind_unbound->remote_domain;
 		if ((error = HYPERVISOR_event_channel_op(&op)))
 			return -error;
 		bind_unbound->port = op.u.alloc_unbound.port;
 		mutex_enter(&devevent_lock);
 		KASSERT(devevent[bind_unbound->port] == NULL);
 		devevent[bind_unbound->port] = d;
 		mutex_exit(&devevent_lock);
 		xen_atomic_set_bit(&d->ci->ci_evtmask[0], bind_unbound->port);
 		hypervisor_unmask_event(bind_unbound->port);
 		break;
+	}
 	case IOCTL_EVTCHN_UNBIND:
+	{
 		struct ioctl_evtchn_unbind *unbind = addr;
 		if (unbind->port > NR_EVENT_CHANNELS)
 			return EINVAL;
-		if (devevent[unbind->port] != d)
+		mutex_enter(&devevent_lock);
 		if (devevent[unbind->port] != d) {
 			mutex_exit(&devevent_lock);
 			return ENOTCONN;
+		}
 		devevent[unbind->port] = NULL;
 		mutex_exit(&devevent_lock);
 		hypervisor_mask_event(unbind->port);
 		xen_atomic_clear_bit(&d->ci->ci_evtmask[0], unbind->port);
 		op.cmd = EVTCHNOP_close;
 		op.u.close.port = unbind->port;
 		if ((error = HYPERVISOR_event_channel_op(&op)))
 			return -error;
 		break;
+	}
 	case IOCTL_EVTCHN_NOTIFY:
+	{
 		struct ioctl_evtchn_notify *notify = addr;
 		if (notify->port > NR_EVENT_CHANNELS)
 			return EINVAL;
-		if (devevent[notify->port] != d)
+		mutex_enter(&devevent_lock);
 		if (devevent[notify->port] != d) {
 			mutex_exit(&devevent_lock);
 			return ENOTCONN;
+		}
 		hypervisor_notify_via_evtchn(notify->port);
 		mutex_exit(&devevent_lock);
 		break;
+	}
 	case FIONBIO:
 		break;
 	default:
 		return EINVAL;
+	}
 	return 0;
+}
 /*
  * Support for poll() system call
+ *