 @@ -1,14 +1,14 @@
-/*	$NetBSD: evtchn.c,v 1.92 2020/05/04 15:55:56 jdolecek Exp $	*/
+/*	$NetBSD: evtchn.c,v 1.93 2020/05/06 13:43:48 bouyer Exp $	*/
 /*
  * Copyright (c) 2006 Manuel Bouyer.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
 @@ -44,66 +44,64 @@
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: evtchn.c,v 1.92 2020/05/04 15:55:56 jdolecek Exp $");
+__KERNEL_RCSID(0, "$NetBSD: evtchn.c,v 1.93 2020/05/06 13:43:48 bouyer Exp $");
 #include "opt_xen.h"
 #include "isa.h"
 #include "pci.h"
 #include <sys/param.h>
 #include <sys/cpu.h>
 #include <sys/kernel.h>
 #include <sys/systm.h>
 #include <sys/device.h>
 #include <sys/proc.h>
 #include <sys/kmem.h>
 #include <sys/reboot.h>
 #include <sys/mutex.h>
 #include <sys/interrupt.h>
 #include <sys/xcall.h>
 #include <uvm/uvm.h>
 #include <xen/intr.h>
 #include <xen/xen.h>
 #include <xen/hypervisor.h>
 #include <xen/evtchn.h>
 #include <xen/xenfunc.h>
 #define	NR_PIRQS	NR_EVENT_CHANNELS
 /*
  * This lock protects updates to the following mapping and reference-count
  * arrays. The lock does not need to be acquired to read the mapping tables.
  */
 static kmutex_t evtchn_lock;
 /* event handlers */
 struct evtsource *evtsource[NR_EVENT_CHANNELS];
 /* channel locks */
 static kmutex_t evtlock[NR_EVENT_CHANNELS];
 /* Reference counts for bindings to event channels XXX: redo for SMP */
 static uint8_t evtch_bindcount[NR_EVENT_CHANNELS];
 /* event-channel <-> VCPU mapping for IPIs. XXX: redo for SMP. */
 static evtchn_port_t vcpu_ipi_to_evtch[XEN_LEGACY_MAX_VCPUS];
 /* event-channel <-> VCPU mapping for VIRQ_TIMER.  XXX: redo for SMP. */
 static int virq_timer_to_evtch[XEN_LEGACY_MAX_VCPUS];
 /* event-channel <-> VIRQ mapping. */
 static int virq_to_evtch[NR_VIRQS];
 @@ -352,57 +350,54 @@ evtchn_do_event(int evtch, struct intrfr
 		hypervisor_set_ipending(evtsource[evtch]->ev_imask,
 					evtch >> LONG_SHIFT,
 					evtch & LONG_MASK);
 		/* leave masked */
 		return 0;
+	}
 	ci->ci_ilevel = evtsource[evtch]->ev_maxlevel;
 	iplmask = evtsource[evtch]->ev_imask;
 	KASSERT(ci->ci_ilevel >= IPL_VM);
 	KASSERT(cpu_intr_p());
 	x86_enable_intr();
 	mutex_spin_enter(&evtlock[evtch]);
 	ih = evtsource[evtch]->ev_handlers;
 	while (ih != NULL) {
 		KASSERT(ih->ih_cpu == ci);
 #if 0
 		if (ih->ih_cpu != ci) {
 			hypervisor_send_event(ih->ih_cpu, evtch);
 			iplmask &= ~(1 << XEN_IPL2SIR(ih->ih_level));
 			ih = ih->ih_evt_next;
 			continue;
+		}
 #endif
 		if (ih->ih_level <= ilevel) {
 #ifdef IRQ_DEBUG
 		if (evtch == IRQ_DEBUG)
 		    printf("ih->ih_level %d <= ilevel %d\n", ih->ih_level, ilevel);
 #endif
 			x86_disable_intr();
 			hypervisor_set_ipending(iplmask,
 			    evtch >> LONG_SHIFT, evtch & LONG_MASK);
 			/* leave masked */
 			mutex_spin_exit(&evtlock[evtch]);
 			goto splx;
+		}
 		iplmask &= ~(1 << XEN_IPL2SIR(ih->ih_level));
 		ci->ci_ilevel = ih->ih_level;
 		ih_fun = (void *)ih->ih_fun;
 		ih_fun(ih->ih_arg, regs);
 		ih = ih->ih_evt_next;
+	}
 	mutex_spin_exit(&evtlock[evtch]);
 	x86_disable_intr();
 	hypervisor_unmask_event(evtch);
 splx:
 	ci->ci_ilevel = ilevel;
 	return 0;
+}
 #define PRIuCPUID	"lu" /* XXX: move this somewhere more appropriate */
 /* PIC callbacks */
 /* pic "pin"s are conceptually mapped to event port numbers */
 static void
 @@ -763,54 +758,49 @@ pirq_interrupt(void *arg)
 #endif /* NPCI > 0 || NISA > 0 */
 /*
  * Recalculate the interrupt from scratch for an event source.
  */
 static void
 intr_calculatemasks(struct evtsource *evts, int evtch, struct cpu_info *ci)
+{
 	struct intrhand *ih;
 	int cpu_receive = 0;
 #ifdef MULTIPROCESSOR
 	KASSERT(!mutex_owned(&evtlock[evtch]));
 #endif
 	mutex_spin_enter(&evtlock[evtch]);
 	evts->ev_maxlevel = IPL_NONE;
 	evts->ev_imask = 0;
 	for (ih = evts->ev_handlers; ih != NULL; ih = ih->ih_evt_next) {
 		KASSERT(ih->ih_cpu == curcpu());
 		if (ih->ih_level > evts->ev_maxlevel)
 			evts->ev_maxlevel = ih->ih_level;
 		evts->ev_imask |= (1 << XEN_IPL2SIR(ih->ih_level));
 		if (ih->ih_cpu == ci)
 			cpu_receive = 1;
+	}
 	if (cpu_receive)
 		xen_atomic_set_bit(&curcpu()->ci_evtmask[0], evtch);
 	else
 		xen_atomic_clear_bit(&curcpu()->ci_evtmask[0], evtch);
 	mutex_spin_exit(&evtlock[evtch]);
+}
 struct intrhand *
 event_set_handler(int evtch, int (*func)(void *), void *arg, int level,
     const char *intrname, const char *xname, bool mpsafe, bool bind)
+{
 	struct cpu_info *ci = curcpu(); /* XXX: pass in ci ? */
 	struct evtsource *evts;
 	struct intrhand *ih, **ihp;
 	int s;
 	char intrstr_buf[INTRIDBUF];
 #ifdef IRQ_DEBUG
 	printf("event_set_handler IRQ %d handler %p\n", evtch, func);
 #endif
 	KASSERTMSG(evtch >= 0, "negative evtch: %d", evtch);
 	KASSERTMSG(evtch < NR_EVENT_CHANNELS,
 	    "evtch number %d > NR_EVENT_CHANNELS", evtch);
 	KASSERT(xname != NULL);
 #if 0
 	printf("event_set_handler evtch %d handler %p level %d\n", evtch,
 @@ -826,200 +816,228 @@ event_set_handler(int evtch, int (*func)
 	ih->ih_fun = ih->ih_realfun = func;
 	ih->ih_arg = ih->ih_realarg = arg;
 	ih->ih_evt_next = NULL;
 	ih->ih_next = NULL;
 	ih->ih_cpu = ci;
 	ih->ih_pin = evtch;
 #ifdef MULTIPROCESSOR
 	if (!mpsafe) {
 		ih->ih_fun = xen_intr_biglock_wrapper;
 		ih->ih_arg = ih;
+	}
 #endif /* MULTIPROCESSOR */
-	s = splhigh();
+	mutex_enter(&cpu_lock);
 	/* register per-cpu handler for spllower() */
 	event_set_iplhandler(ci, ih, level);
 	/* register handler for event channel */
 	if (evtsource[evtch] == NULL) {
 		evtchn_op_t op;
 		if (intrname == NULL)
 			intrname = intr_create_intrid(-1, &xen_pic, evtch,
 			    intrstr_buf, sizeof(intrstr_buf));
 		evts = kmem_zalloc(sizeof (struct evtsource),
 		    KM_NOSLEEP);
 		if (evts == NULL)
 			panic("can't allocate fixed interrupt source");
 		evts->ev_handlers = ih;
 		/*
 		 * XXX: We're assuming here that ci is the same cpu as
 		 * the one on which this event/port is bound on. The
 		 * api needs to be reshuffled so that this assumption
 		 * is more explicitly implemented.
 		 */
 		evts->ev_cpu = ci;
 		mutex_init(&evtlock[evtch], MUTEX_DEFAULT, IPL_HIGH);
 		evtsource[evtch] = evts;
 		strlcpy(evts->ev_intrname, intrname, sizeof(evts->ev_intrname));
 		evcnt_attach_dynamic(&evts->ev_evcnt, EVCNT_TYPE_INTR, NULL,
 		    device_xname(ci->ci_dev), evts->ev_intrname);
 		if (bind) {
 			op.cmd = EVTCHNOP_bind_vcpu;
 			op.u.bind_vcpu.port = evtch;
 			op.u.bind_vcpu.vcpu = ci->ci_vcpuid;
 			if (HYPERVISOR_event_channel_op(&op) != 0) {
 				panic("Failed to bind event %d to VCPU  %s %d",
 				    evtch, device_xname(ci->ci_dev),
 				    ci->ci_vcpuid);
+			}
+		}
 #ifndef XENPV
 		evts->ev_isl = intr_allocate_io_intrsource(intrname);
 		evts->ev_isl->is_pic = &xen_pic;
 		evts->ev_isl->is_handlers = evts->ev_handlers;
 #endif
 	} else {
 		evts = evtsource[evtch];
 		/* sort by IPL order, higher first */
 		mutex_spin_enter(&evtlock[evtch]);
 		for (ihp = &evts->ev_handlers; ; ihp = &((*ihp)->ih_evt_next)) {
 			if ((*ihp)->ih_level < ih->ih_level) {
 				/* insert before *ihp */
 				ih->ih_evt_next = *ihp;
 				*ihp = ih;
 				break;
+			}
 			if ((*ihp)->ih_evt_next == NULL) {
 				(*ihp)->ih_evt_next = ih;
 				break;
+			}
+		}
 		mutex_spin_exit(&evtlock[evtch]);
 #ifndef XENPV
 		mutex_enter(&cpu_lock);
 		evts->ev_isl->is_handlers = evts->ev_handlers;
 		mutex_exit(&cpu_lock);
 #endif
+	}
 	const u_long psl = x86_read_psl();
 	x86_disable_intr();
 	/* register per-cpu handler for spllower() */
 	event_set_iplhandler(ci, ih, level);
 	intr_calculatemasks(evts, evtch, ci);
 	x86_write_psl(psl);
 	// append device name
 	if (evts->ev_xname[0] != '\0')
 		strlcat(evts->ev_xname, ", ", sizeof(evts->ev_xname));
 	strlcat(evts->ev_xname, xname, sizeof(evts->ev_xname));
 	intr_calculatemasks(evts, evtch, ci);
 	splx(s);
 #ifndef XENPV
 	mutex_enter(&cpu_lock);
 	if (evts->ev_isl == NULL) {
 		evts->ev_isl = intr_allocate_io_intrsource(intrname);
 		evts->ev_isl->is_pic = &xen_pic;
 	evts->ev_isl->is_handlers = evts->ev_handlers;
 	mutex_exit(&cpu_lock);
 #endif
 	return ih;
+}
 void
 event_set_iplhandler(struct cpu_info *ci,
 		     struct intrhand *ih,
 		     int level)
+{
 	struct intrsource *ipls;
 	int sir = XEN_IPL2SIR(level);
 	KASSERT(sir >= SIR_XENIPL_VM && sir <= SIR_XENIPL_HIGH);
 	KASSERT(ci == ih->ih_cpu);
 	KASSERT(ci == curcpu());
 	if (ci->ci_isources[sir] == NULL) {
 		ipls = kmem_zalloc(sizeof (struct intrsource),
 		    KM_NOSLEEP);
 		if (ipls == NULL)
 			panic("can't allocate fixed interrupt source");
 		ipls->is_recurse = xenev_stubs[level - IPL_VM].ist_recurse;
 		ipls->is_resume = xenev_stubs[level - IPL_VM].ist_resume;
 		ipls->is_handlers = ih;
 		ipls->is_pic = &xen_pic;
 		ci->ci_isources[sir] = ipls;
 	} else {
 		ipls = ci->ci_isources[sir];
 		ih->ih_next = ipls->is_handlers;
 		ipls->is_handlers = ih;
+	}
 	x86_intr_calculatemasks(ci);
+}
-int
+/*
-event_remove_handler(int evtch, int (*func)(void *), void *arg)
+ * Called on bound CPU to handle event_remove_handler()
  * caller (on initiating CPU) holds cpu_lock on our behalf
  * arg1: evtch
  * arg2: struct intrhand *ih
  */
 static void
 event_remove_handler_xcall(void *arg1, void *arg2)
+{
 	struct intrsource *ipls;
 	struct evtsource *evts;
 	struct intrhand *ih;
 	struct intrhand **ihp;
 	struct cpu_info *ci;
 	struct intrhand *ih = arg2;
 	int evtch = (intptr_t)(arg1);
 	evts = evtsource[evtch];
-	if (evts == NULL)
+	KASSERT(evts != NULL);
-		return ENOENT;
+	KASSERT(ih != NULL);
 	ci = ih->ih_cpu;
 	KASSERT(ci == curcpu());
-	mutex_spin_enter(&evtlock[evtch]);
+	const u_long psl = x86_read_psl();
-	for (ihp = &evts->ev_handlers, ih = evts->ev_handlers;
+	x86_disable_intr();
 	    ih != NULL;
-	    ihp = &ih->ih_evt_next, ih = ih->ih_evt_next) {
+	for (ihp = &evts->ev_handlers; *ihp != NULL;
-		if (ih->ih_realfun == func && ih->ih_realarg == arg)
+	    ihp = &(*ihp)->ih_evt_next) {
 		if ((*ihp) == ih)
 			break;
+	}
-	if (ih == NULL) {
+	if (*(ihp) == NULL) {
-		mutex_spin_exit(&evtlock[evtch]);
+		panic("event_remove_handler_xcall: not in ev_handlers");
 		return ENOENT;
+	}
 	ci = ih->ih_cpu;
 	*ihp = ih->ih_evt_next;
 	int sir = XEN_IPL2SIR(ih->ih_level);
 	KASSERT(sir >= SIR_XENIPL_VM && sir <= SIR_XENIPL_HIGH);
 	ipls = ci->ci_isources[sir];
-	for (ihp = &ipls->is_handlers, ih = ipls->is_handlers;
+	for (ihp = &ipls->is_handlers; *ihp != NULL; ihp = &(*ihp)->ih_next) {
-	    ih != NULL;
+		if (*ihp == ih)
 	    ihp = &ih->ih_next, ih = ih->ih_next) {
 		if (ih->ih_realfun == func && ih->ih_realarg == arg)
 			break;
+	}
-	if (ih == NULL)
+	if (*ihp == NULL)
-		panic("event_remove_handler");
+		panic("event_remove_handler_xcall: not in is_handlers");
 	*ihp = ih->ih_next;
-	mutex_spin_exit(&evtlock[evtch]);
+	intr_calculatemasks(evts, evtch, ci);
 #ifndef XENPV
 	mutex_enter(&cpu_lock);
 	evts->ev_isl->is_handlers = evts->ev_handlers;
 	mutex_exit(&cpu_lock);
 #endif
 	if (evts->ev_handlers == NULL)
 		xen_atomic_clear_bit(&ci->ci_evtmask[0], evtch);
 	x86_write_psl(psl);
+}
 int
 event_remove_handler(int evtch, int (*func)(void *), void *arg)
+{
 	struct intrhand *ih;
 	struct cpu_info *ci;
 	struct evtsource *evts;
 	mutex_enter(&cpu_lock);
 	evts = evtsource[evtch];
 	if (evts == NULL)
 		return ENOENT;
 	for (ih = evts->ev_handlers; ih != NULL; ih = ih->ih_evt_next) {
 		if (ih->ih_realfun == func && ih->ih_realarg == arg)
 			break;
+	}
 	if (ih == NULL) {
 		mutex_exit(&cpu_lock);
 		return ENOENT;
+	}
 	ci = ih->ih_cpu;
 	if (ci == curcpu() || !mp_online) {
 		event_remove_handler_xcall((void *)(intptr_t)evtch, ih);
 	} else {
 		uint64_t where = xc_unicast(0, event_remove_handler_xcall,
 		    (void *)(intptr_t)evtch, ih, ci);
 		xc_wait(where);
+	}
 	kmem_free(ih, sizeof (struct intrhand));
 	if (evts->ev_handlers == NULL) {
 #ifndef XENPV
 		KASSERT(evts->ev_isl->is_handlers == NULL);
 		mutex_enter(&cpu_lock);
 		intr_free_io_intrsource(evts->ev_intrname);
 		mutex_exit(&cpu_lock);
 #endif
 		xen_atomic_clear_bit(&ci->ci_evtmask[0], evtch);
 		evcnt_detach(&evts->ev_evcnt);
 		kmem_free(evts, sizeof (struct evtsource));
 		evtsource[evtch] = NULL;
 	} else {
 		intr_calculatemasks(evts, evtch, ci);
+	}
 	mutex_exit(&cpu_lock);
 	return 0;
+}
 int
 xen_debug_handler(void *arg)
+{
 	struct cpu_info *ci = curcpu();
 	int i;
 	int xci_ilevel = ci->ci_ilevel;
 	int xci_ipending = ci->ci_ipending;
 	int xci_idepth = ci->ci_idepth;
 	u_long upcall_pending = ci->ci_vcpu->evtchn_upcall_pending;
 	u_long upcall_mask = ci->ci_vcpu->evtchn_upcall_mask;