 @@ -1,14 +1,14 @@
-/*	$NetBSD: pic.c,v 1.70 2021/03/27 12:15:09 jmcneill Exp $	*/
+/*	$NetBSD: pic.c,v 1.71 2021/08/08 19:28:08 skrll Exp $	*/
 /*-
  * Copyright (c) 2008 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Matt Thomas.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
 @@ -23,27 +23,27 @@
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #define _INTR_PRIVATE
 #include "opt_ddb.h"
 #include "opt_multiprocessor.h"
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.70 2021/03/27 12:15:09 jmcneill Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.71 2021/08/08 19:28:08 skrll Exp $");
 #include <sys/param.h>
 #include <sys/atomic.h>
 #include <sys/cpu.h>
 #include <sys/evcnt.h>
 #include <sys/interrupt.h>
 #include <sys/intr.h>
 #include <sys/ipi.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/mutex.h>
 #include <sys/once.h>
 #include <sys/xcall.h>
 @@ -55,57 +55,35 @@ __KERNEL_RCSID(0, "$NetBSD: pic.c,v 1.70
 #ifdef DDB
 #include <arm/db_machdep.h>
 #endif
 #include <arm/pic/picvar.h>
 #if defined(__HAVE_PIC_PENDING_INTRS)
 /*
  * This implementation of pending interrupts on a MULTIPROCESSOR system makes
  * the assumption that a PIC (pic_softc) shall only have all its interrupts
  * come from the same CPU.  In other words, interrupts from a single PIC will
  * not be distributed among multiple CPUs.
  */
 struct pic_pending {
 	volatile uint32_t blocked_pics;
 	volatile uint32_t pending_pics;
 	volatile uint32_t pending_ipls;
 };
 static uint32_t
 	pic_find_pending_irqs_by_ipl(struct pic_softc *, size_t, uint32_t, int);
 static struct pic_softc *
-	pic_list_find_pic_by_pending_ipl(struct pic_pending *, uint32_t);
+	pic_list_find_pic_by_pending_ipl(struct cpu_info *, uint32_t);
 static void
-	pic_deliver_irqs(struct pic_pending *, struct pic_softc *, int, void *);
+	pic_deliver_irqs(struct cpu_info *, struct pic_softc *, int, void *);
 static void
-	pic_list_deliver_irqs(struct pic_pending *, register_t, int, void *);
+	pic_list_deliver_irqs(struct cpu_info *, register_t, int, void *);
 #ifdef MULTIPROCESSOR
 percpu_t *pic_pending_percpu;
 static struct pic_pending *
 pic_pending_get(void)
 	return percpu_getref(pic_pending_percpu);
 static void
 pic_pending_put(struct pic_pending *pend)
 	percpu_putref(pic_pending_percpu);
 #else
 struct pic_pending pic_pending;
 #define	pic_pending_get()	(&pic_pending)
 #define	pic_pending_put(pend)	__nothing
 #endif /* MULTIPROCESSOR */
 #endif /* __HAVE_PIC_PENDING_INTRS */
 struct pic_softc *pic_list[PIC_MAXPICS];
 #if PIC_MAXPICS > 32
 #error PIC_MAXPICS > 32 not supported
 #endif
 struct intrsource *pic_sources[PIC_MAXMAXSOURCES];
 struct intrsource *pic__iplsources[PIC_MAXMAXSOURCES];
 struct intrsource **pic_iplsource[NIPL] = {
 	[0 ... NIPL-1] = pic__iplsources,
 };
 size_t pic_ipl_offset[NIPL+1];
 @@ -254,81 +232,80 @@ pic_handle_intr(void *arg)
 	struct pic_softc * const pic = arg;
 	int rv;
 	rv = (*pic->pic_ops->pic_find_pending_irqs)(pic);
 	return rv > 0;
+}
 #if defined(__HAVE_PIC_PENDING_INTRS)
 void
 pic_mark_pending_source(struct pic_softc *pic, struct intrsource *is)
+{
 	const uint32_t ipl_mask = __BIT(is->is_ipl);
 	struct cpu_info * const ci = curcpu();
 	atomic_or_32(&pic->pic_pending_irqs[is->is_irq >> 5],
 	    __BIT(is->is_irq & 0x1f));
 	atomic_or_32(&pic->pic_pending_ipls, ipl_mask);
-	struct pic_pending *pend = pic_pending_get();
+	ci->ci_pending_ipls |= ipl_mask;
-	atomic_or_32(&pend->pending_ipls, ipl_mask);
+	ci->ci_pending_pics |= __BIT(pic->pic_id);
 	atomic_or_32(&pend->pending_pics, __BIT(pic->pic_id));
 	pic_pending_put(pend);
+}
 void
 pic_mark_pending(struct pic_softc *pic, int irq)
+{
 	struct intrsource * const is = pic->pic_sources[irq];
 	KASSERT(irq < pic->pic_maxsources);
 	KASSERT(is != NULL);
 	pic_mark_pending_source(pic, is);
+}
 uint32_t
 pic_mark_pending_sources(struct pic_softc *pic, size_t irq_base,
-	uint32_t pending)
+    uint32_t pending)
+{
 	struct intrsource ** const isbase = &pic->pic_sources[irq_base];
 	struct cpu_info * const ci = curcpu();
 	struct intrsource *is;
 	volatile uint32_t *ipending = &pic->pic_pending_irqs[irq_base >> 5];
 	uint32_t ipl_mask = 0;
 	if (pending == 0)
 		return ipl_mask;
 	KASSERT((irq_base & 31) == 0);
 	(*pic->pic_ops->pic_block_irqs)(pic, irq_base, pending);
 	atomic_or_32(ipending, pending);
 	while (pending != 0) {
 		int n = ffs(pending);
 		if (n-- == 0)
 			break;
 		is = isbase[n];
 		KASSERT(is != NULL);
 		KASSERT(irq_base <= is->is_irq && is->is_irq < irq_base + 32);
 		pending &= ~__BIT(n);
 		ipl_mask |= __BIT(is->is_ipl);
+	}
 	atomic_or_32(&pic->pic_pending_ipls, ipl_mask);
-	struct pic_pending *pend = pic_pending_get();
+	ci->ci_pending_ipls |= ipl_mask;
-	atomic_or_32(&pend->pending_ipls, ipl_mask);
+	ci->ci_pending_pics |= __BIT(pic->pic_id);
 	atomic_or_32(&pend->pending_pics, __BIT(pic->pic_id));
 	pic_pending_put(pend);
 	return ipl_mask;
+}
 uint32_t
 pic_find_pending_irqs_by_ipl(struct pic_softc *pic, size_t irq_base,
 	uint32_t pending, int ipl)
+{
 	uint32_t ipl_irq_mask = 0;
 	uint32_t irq_mask;
 	for (;;) {
 		int irq = ffs(pending);
 		if (irq-- == 0)
 @@ -377,27 +354,27 @@ pic_dispatch(struct intrsource *is, void
 		KERNEL_UNLOCK_ONE(NULL);
 	} else
 #endif
 		(void)(*func)(arg);
 	struct pic_percpu * const pcpu = percpu_getref(is->is_pic->pic_percpu);
 	KASSERT(pcpu->pcpu_magic == PICPERCPU_MAGIC);
 	pcpu->pcpu_evs[is->is_irq].ev_count++;
 	percpu_putref(is->is_pic->pic_percpu);
+}
 #if defined(__HAVE_PIC_PENDING_INTRS)
 void
-pic_deliver_irqs(struct pic_pending *pend, struct pic_softc *pic, int ipl,
+pic_deliver_irqs(struct cpu_info *ci, struct pic_softc *pic, int ipl,
     void *frame)
+{
 	const uint32_t ipl_mask = __BIT(ipl);
 	struct intrsource *is;
 	volatile uint32_t *ipending = pic->pic_pending_irqs;
 	volatile uint32_t *iblocked = pic->pic_blocked_irqs;
 	size_t irq_base;
 #if PIC_MAXSOURCES > 32
 	size_t irq_count;
 	int poi = 0;		/* Possibility of interrupting */
 #endif
 	uint32_t pending_irqs;
 	uint32_t blocked_irqs;
 @@ -458,45 +435,45 @@ pic_deliver_irqs(struct pic_pending *pen
 				 * DISABLE_INTERRUPT().
 				 */
 				poi = 1;
 #endif
 				blocked_irqs |= __BIT(irq);
 			} else {
 				KASSERT(0);
+			}
 			pending_irqs = pic_find_pending_irqs_by_ipl(pic,
 			    irq_base, *ipending, ipl);
 		} while (pending_irqs);
 		if (blocked_irqs) {
 			atomic_or_32(iblocked, blocked_irqs);
-			atomic_or_32(&pend->blocked_pics, __BIT(pic->pic_id));
+			ci->ci_blocked_pics |= __BIT(pic->pic_id);
+		}
+	}
 	KASSERT(progress);
 	/*
 	 * Since interrupts are disabled, we don't have to be too careful
 	 * about these.
 	 */
 	if (atomic_and_32_nv(&pic->pic_pending_ipls, ~ipl_mask) == 0)
-		atomic_and_32(&pend->pending_pics, ~__BIT(pic->pic_id));
+		ci->ci_pending_pics &= ~__BIT(pic->pic_id);
+}
 static void
-pic_list_unblock_irqs(struct pic_pending *pend)
+pic_list_unblock_irqs(struct cpu_info *ci)
+{
-	uint32_t blocked_pics = pend->blocked_pics;
+	uint32_t blocked_pics = ci->ci_blocked_pics;
-	pend->blocked_pics = 0;
+	ci->ci_blocked_pics = 0;
 	for (;;) {
 		struct pic_softc *pic;
 #if PIC_MAXSOURCES > 32
 		volatile uint32_t *iblocked;
 		uint32_t blocked;
 		size_t irq_base;
 #endif
 		int pic_id = ffs(blocked_pics);
 		if (pic_id-- == 0)
 			return;
 @@ -513,83 +490,81 @@ pic_list_unblock_irqs(struct pic_pending
+			}
+		}
 #else
 		KASSERT(pic->pic_blocked_irqs[0] != 0);
 		(*pic->pic_ops->pic_unblock_irqs)(pic,
 , pic->pic_blocked_irqs[0]);
 		pic->pic_blocked_irqs[0] = 0;
 #endif
 		blocked_pics &= ~__BIT(pic_id);
+	}
+}
 struct pic_softc *
-pic_list_find_pic_by_pending_ipl(struct pic_pending *pend, uint32_t ipl_mask)
+pic_list_find_pic_by_pending_ipl(struct cpu_info *ci, uint32_t ipl_mask)
+{
-	uint32_t pending_pics = pend->pending_pics;
+	uint32_t pending_pics = ci->ci_pending_pics;
 	struct pic_softc *pic;
 	for (;;) {
 		int pic_id = ffs(pending_pics);
 		if (pic_id-- == 0)
 			return NULL;
 		pic = pic_list[pic_id];
 		KASSERT(pic != NULL);
 		if (pic->pic_pending_ipls & ipl_mask)
 			return pic;
 		pending_pics &= ~__BIT(pic_id);
+	}
+}
 void
-pic_list_deliver_irqs(struct pic_pending *pend, register_t psw, int ipl,
+pic_list_deliver_irqs(struct cpu_info *ci, register_t psw, int ipl,
     void *frame)
+{
 	const uint32_t ipl_mask = __BIT(ipl);
 	struct pic_softc *pic;
-	while ((pic = pic_list_find_pic_by_pending_ipl(pend, ipl_mask)) != NULL) {
+	while ((pic = pic_list_find_pic_by_pending_ipl(ci, ipl_mask)) != NULL) {
-		pic_deliver_irqs(pend, pic, ipl, frame);
+		pic_deliver_irqs(ci, pic, ipl, frame);
 		KASSERT((pic->pic_pending_ipls & ipl_mask) == 0);
+	}
-	atomic_and_32(&pend->pending_ipls, ~ipl_mask);
+	ci->ci_pending_ipls &= ~ipl_mask;
+}
 #endif /* __HAVE_PIC_PENDING_INTRS */
 void
 pic_do_pending_ints(register_t psw, int newipl, void *frame)
+{
 	struct cpu_info * const ci = curcpu();
 	if (__predict_false(newipl == IPL_HIGH)) {
 		KASSERTMSG(ci->ci_cpl == IPL_HIGH, "cpl %d", ci->ci_cpl);
 		return;
+	}
 #if defined(__HAVE_PIC_PENDING_INTRS)
-	struct pic_pending *pend = pic_pending_get();
+	while ((ci->ci_pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) {
-	while ((pend->pending_ipls & ~__BIT(newipl)) > __BIT(newipl)) {
+		KASSERT(ci->ci_pending_ipls < __BIT(NIPL));
 		KASSERT(pend->pending_ipls < __BIT(NIPL));
 		for (;;) {
-			int ipl = 31 - __builtin_clz(pend->pending_ipls);
+			int ipl = 31 - __builtin_clz(ci->ci_pending_ipls);
 			KASSERT(ipl < NIPL);
 			if (ipl <= newipl)
 				break;
 			pic_set_priority(ci, ipl);
-			pic_list_deliver_irqs(pend, psw, ipl, frame);
+			pic_list_deliver_irqs(ci, psw, ipl, frame);
-			pic_list_unblock_irqs(pend);
+			pic_list_unblock_irqs(ci);
+		}
+	}
 	pic_pending_put(pend);
 #endif /* __HAVE_PIC_PENDING_INTRS */
 #ifdef __HAVE_PREEMPTION
 	if (newipl == IPL_NONE && (ci->ci_astpending & __BIT(1))) {
 		pic_set_priority(ci, IPL_SCHED);
 		kpreempt(0);
+	}
 #endif
 	if (ci->ci_cpl != newipl)
 		pic_set_priority(ci, newipl);
+}
 static void
 pic_percpu_allocate(void *v0, void *v1, struct cpu_info *ci)
 @@ -632,31 +607,26 @@ pic_init(void)
+}
 int
 pic_add(struct pic_softc *pic, int irqbase)
+{
 	int slot, maybe_slot = -1;
 	size_t sourcebase;
 	static ONCE_DECL(pic_once);
 	RUN_ONCE(&pic_once, pic_init);
 	KASSERT(strlen(pic->pic_name) > 0);
 #if defined(__HAVE_PIC_PENDING_INTRS) && defined(MULTIPROCESSOR)
 	if (__predict_false(pic_pending_percpu == NULL))
 		pic_pending_percpu = percpu_alloc(sizeof(struct pic_pending));
 #endif /* __HAVE_PIC_PENDING_INTRS && MULTIPROCESSOR */
 	mutex_enter(&pic_lock);
 	if (irqbase == PIC_IRQBASE_ALLOC) {
 		irqbase = pic_lastbase;
+	}
 	for (slot = 0; slot < PIC_MAXPICS; slot++) {
 		struct pic_softc * const xpic = pic_list[slot];
 		if (xpic == NULL) {
 			if (maybe_slot < 0)
 				maybe_slot = slot;
 			if (irqbase < 0)
 				break;
 			continue;
+		}

 @@ -1,14 +1,14 @@
-/* $NetBSD: cpu.h,v 1.36 2021/05/29 06:54:20 skrll Exp $ */
+/* $NetBSD: cpu.h,v 1.37 2021/08/08 19:28:08 skrll Exp $ */
 /*-
  * Copyright (c) 2014, 2020 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Matt Thomas of 3am Software Foundry.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -94,26 +94,29 @@ struct cpu_info {
 	 * largely CPU-private.
 	 */
 	struct lwp *ci_softlwps[SOFTINT_COUNT] __aligned(COHERENCY_UNIT);
 	uint64_t ci_lastintr;
 	int ci_mtx_oldspl;
 	int ci_mtx_count;
 	int ci_cpl;		/* current processor level (spl) */
 	int ci_hwpl;		/* current hardware priority */
 	volatile u_int ci_softints;
 	volatile u_int ci_intr_depth;
 	volatile uint32_t ci_blocked_pics;
 	volatile uint32_t ci_pending_pics;
 	volatile uint32_t ci_pending_ipls;
 	int ci_kfpu_spl;
 	/* event counters */
 	struct evcnt ci_vfp_use;
 	struct evcnt ci_vfp_reuse;
 	struct evcnt ci_vfp_save;
 	struct evcnt ci_vfp_release;
 	struct evcnt ci_uct_trap;
 	struct evcnt ci_intr_preempt;
 	/* FDT or similar supplied "cpu capacity" */
 	uint32_t ci_capacity_dmips_mhz;

 @@ -1,14 +1,14 @@
-/*	$NetBSD: cpu.h,v 1.117 2021/03/27 12:15:08 jmcneill Exp $	*/
+/*	$NetBSD: cpu.h,v 1.118 2021/08/08 19:28:08 skrll Exp $	*/
 /*
  * Copyright (c) 1994-1996 Mark Brinicombe.
  * Copyright (c) 1994 Brini.
  * All rights reserved.
+ *
  * This code is derived from software written for Brini by Mark Brinicombe
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -180,26 +180,29 @@ struct cpu_info {
 	 * largely CPU-private.
 	 */
 	lwp_t *		ci_softlwps[SOFTINT_COUNT] __aligned(COHERENCY_UNIT);
 	struct cpu_softc *
 			ci_softc;	/* platform softc */
 	int		ci_cpl;		/* current processor level (spl) */
 	int		ci_hwpl;	/* current hardware priority */
 	int		ci_kfpu_spl;
 	volatile u_int	ci_intr_depth;	/* */
 	volatile u_int	ci_softints;
 	volatile uint32_t ci_blocked_pics;
 	volatile uint32_t ci_pending_pics;
 	volatile uint32_t ci_pending_ipls;
 	lwp_t *		ci_lastlwp;	/* last lwp */
 	struct evcnt	ci_arm700bugcount;
 	int32_t		ci_mtx_count;
 	int		ci_mtx_oldspl;
 	register_t	ci_undefsave[3];
 	uint32_t	ci_vfp_id;
 	uint64_t	ci_lastintr;
 	struct pmap_tlb_info *
 			ci_tlb_info;
 	struct pmap *	ci_pmap_lastuser;