 @@ -1,14 +1,14 @@
-/*	$NetBSD: octeon_intr.c,v 1.14 2020/06/23 05:14:39 simonb Exp $	*/
+/*	$NetBSD: octeon_intr.c,v 1.15 2020/07/16 21:33:50 jmcneill Exp $	*/
 /*
  * Copyright 2001, 2002 Wasabi Systems, Inc.
  * All rights reserved.
+ *
  * Written by Jason R. Thorpe and Simon Burge for Wasabi Systems, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
 @@ -34,27 +34,27 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 /*
  * Platform-specific interrupt support for the MIPS Malta.
  */
 #include "opt_multiprocessor.h"
 #include "cpunode.h"
 #define __INTR_PRIVATE
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: octeon_intr.c,v 1.14 2020/06/23 05:14:39 simonb Exp $");
+__KERNEL_RCSID(0, "$NetBSD: octeon_intr.c,v 1.15 2020/07/16 21:33:50 jmcneill Exp $");
 #include <sys/param.h>
 #include <sys/cpu.h>
 #include <sys/systm.h>
 #include <sys/device.h>
 #include <sys/intr.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/atomic.h>
 #include <lib/libkern/libkern.h>
 #include <mips/locore.h>
 @@ -70,27 +70,27 @@ static const struct ipl_sr_map octeon_ip
     .sr_bits = {
 	[IPL_NONE] =		0,
 	[IPL_SOFTCLOCK] =	MIPS_SOFT_INT_MASK_0,
 	[IPL_SOFTNET] =		MIPS_SOFT_INT_MASK,
 	[IPL_VM] =		MIPS_SOFT_INT_MASK | MIPS_INT_MASK_0,
 	[IPL_SCHED] =		MIPS_SOFT_INT_MASK | MIPS_INT_MASK_0
 				    | MIPS_INT_MASK_1 | MIPS_INT_MASK_5,
 	[IPL_DDB] =		MIPS_SOFT_INT_MASK | MIPS_INT_MASK_0
 				    | MIPS_INT_MASK_1 | MIPS_INT_MASK_5,
 	[IPL_HIGH] =		MIPS_INT_MASK,
     },
 };
-const char * const octeon_intrnames[NIRQS] = {
+const char * octeon_intrnames[NIRQS] = {
 	"workq 0",
 	"workq 1",
 	"workq 2",
 	"workq 3",
 	"workq 4",
 	"workq 5",
 	"workq 6",
 	"workq 7",
 	"workq 8",
 	"workq 9",
 	"workq 10",
 	"workq 11",
 	"workq 12",
 @@ -181,52 +181,56 @@ struct octeon_intrhand *octciu_intrs[NIR
 #endif
 };
 kmutex_t octeon_intr_lock;
 #define X(a)	MIPS_PHYS_TO_XKPHYS(OCTEON_CCA_NONE, (a))
 struct cpu_softc octeon_cpu0_softc = {
 	.cpu_ci = &cpu_info_store,
 	.cpu_int0_sum0 = X(CIU_INT0_SUM0),
 	.cpu_int1_sum0 = X(CIU_INT1_SUM0),
 	.cpu_int2_sum0 = X(CIU_INT4_SUM0),
 	.cpu_int_sum1 = X(CIU_INT_SUM1),
 	.cpu_int0_en0 = X(CIU_INT0_EN0),
 	.cpu_int1_en0 = X(CIU_INT1_EN0),
 	.cpu_int2_en0 = X(CIU_INT4_EN00),
 	.cpu_int0_en1 = X(CIU_INT0_EN1),
 	.cpu_int1_en1 = X(CIU_INT1_EN1),
 	.cpu_int2_en1 = X(CIU_INT4_EN01),
 	.cpu_int32_en = X(CIU_INT32_EN0),
 	.cpu_wdog = X(CIU_WDOG0),
 	.cpu_pp_poke = X(CIU_PP_POKE0),
 #ifdef MULTIPROCESSOR
 	.cpu_mbox_set = X(CIU_MBOX_SET0),
 	.cpu_mbox_clr = X(CIU_MBOX_CLR0),
 #endif
 };
 #ifdef MULTIPROCESSOR
 /* XXX limit of two CPUs ... */
 struct cpu_softc octeon_cpu1_softc = {
 	.cpu_int0_sum0 = X(CIU_INT2_SUM0),
 	.cpu_int1_sum0 = X(CIU_INT3_SUM0),
 	.cpu_int2_sum0 = X(CIU_INT4_SUM1),
 	.cpu_int_sum1 = X(CIU_INT_SUM1),
 	.cpu_int0_en0 = X(CIU_INT2_EN0),
 	.cpu_int1_en0 = X(CIU_INT3_EN0),
 	.cpu_int2_en0 = X(CIU_INT4_EN10),
 	.cpu_int0_en1 = X(CIU_INT2_EN1),
 	.cpu_int1_en1 = X(CIU_INT3_EN1),
 	.cpu_int2_en1 = X(CIU_INT4_EN11),
 	.cpu_int32_en = X(CIU_INT32_EN1),
 	.cpu_wdog = X(CIU_WDOG(1)),
 	.cpu_pp_poke = X(CIU_PP_POKE1),
 @@ -334,35 +338,37 @@ octeon_intr_init(struct cpu_info *ci)
 	if (ci->ci_dev)
 		aprint_verbose_dev(ci->ci_dev,
 		    "enabling intr masks %#"PRIx64"/%#"PRIx64"/%#"PRIx64"\n",
 		    cpu->cpu_int0_enable0, cpu->cpu_int1_enable0,
 		    cpu->cpu_int2_enable0);
 	mips3_sd(cpu->cpu_int0_en0, cpu->cpu_int0_enable0);
 	mips3_sd(cpu->cpu_int1_en0, cpu->cpu_int1_enable0);
 	mips3_sd(cpu->cpu_int2_en0, cpu->cpu_int2_enable0);
 	mips3_sd(cpu->cpu_int32_en, 0);
-	mips3_sd(cpu->cpu_int0_en1, 0);	// WDOG IPL2
+	mips3_sd(cpu->cpu_int0_en1, cpu->cpu_int0_enable1);
-	mips3_sd(cpu->cpu_int1_en1, 0);	// WDOG IPL3
+	mips3_sd(cpu->cpu_int1_en1, cpu->cpu_int1_enable1);
-	mips3_sd(cpu->cpu_int2_en1, 0);	// WDOG IPL4
+	mips3_sd(cpu->cpu_int2_en1, cpu->cpu_int2_enable1);
 #ifdef MULTIPROCESSOR
 	mips3_sd(cpu->cpu_mbox_clr, __BITS(31,0));
 #endif
-	for (size_t i = 0; i < NIRQS; i++) {
+	for (int i = 0; i < NIRQS; i++) {
 		if (octeon_intrnames[i] == NULL)
 			octeon_intrnames[i] = kmem_asprintf("irq %d", i);
 		evcnt_attach_dynamic(&cpu->cpu_intr_evs[i],
 		    EVCNT_TYPE_INTR, NULL, xname, octeon_intrnames[i]);
+	}
+}
 void
 octeon_cal_timer(int corefreq)
+{
 	/* Compute the number of cycles per second. */
 	curcpu()->ci_cpu_freq = corefreq;
 	/* Compute the number of ticks for hz. */
 	curcpu()->ci_cycles_per_hz = (curcpu()->ci_cpu_freq + hz / 2) / hz;
 @@ -401,159 +407,217 @@ octeon_intr_establish(int irq, int ipl,
 	/*
 	 * First, make it known.
 	 */
 	KASSERTMSG(octciu_intrs[irq] == NULL, "irq %d in use! (%p)",
 	    irq, octciu_intrs[irq]);
 	octciu_intrs[irq] = ih;
 	membar_producer();
 	/*
 	 * Now enable it.
 	 */
-	const uint64_t irq_mask = __BIT(irq);
+	const int bank = irq / 64;
 	const uint64_t irq_mask = __BIT(irq % 64);
 	struct cpu_softc * const cpu0 = &octeon_cpu0_softc;
 #if MULTIPROCESSOR
 	struct cpu_softc * const cpu1 = &octeon_cpu1_softc;
 #endif
 	switch (ipl) {
 	case IPL_VM:
-		cpu0->cpu_int0_enable0 |= irq_mask;
+		if (bank == 0) {
-		mips3_sd(cpu0->cpu_int0_en0, cpu0->cpu_int0_enable0);
+			cpu0->cpu_int0_enable0 |= irq_mask;
 			mips3_sd(cpu0->cpu_int0_en0, cpu0->cpu_int0_enable0);
 		} else {
 			cpu0->cpu_int0_enable1 |= irq_mask;
 			mips3_sd(cpu0->cpu_int0_en1, cpu0->cpu_int0_enable1);
+		}
 		break;
 	case IPL_SCHED:
-		cpu0->cpu_int1_enable0 |= irq_mask;
+		if (bank == 0) {
-		mips3_sd(cpu0->cpu_int1_en0, cpu0->cpu_int1_enable0);
+			cpu0->cpu_int1_enable0 |= irq_mask;
 			mips3_sd(cpu0->cpu_int1_en0, cpu0->cpu_int1_enable0);
 #ifdef MULTIPROCESSOR
 			cpu1->cpu_int1_enable0 = cpu0->cpu_int1_enable0;
 			mips3_sd(cpu1->cpu_int1_en0, cpu1->cpu_int1_enable0);
 #endif
 		} else {
 			cpu0->cpu_int1_enable1 |= irq_mask;
 			mips3_sd(cpu0->cpu_int1_en1, cpu0->cpu_int1_enable1);
 #ifdef MULTIPROCESSOR
-		cpu1->cpu_int1_enable0 = cpu0->cpu_int1_enable0;
+			cpu1->cpu_int1_enable1 = cpu0->cpu_int1_enable1;
-		mips3_sd(cpu1->cpu_int1_en0, cpu1->cpu_int1_enable0);
+			mips3_sd(cpu1->cpu_int1_en1, cpu1->cpu_int1_enable1);
 #endif
+		}
 		break;
 	case IPL_DDB:
 	case IPL_HIGH:
-		cpu0->cpu_int2_enable0 |= irq_mask;
+		if (bank == 0) {
-		mips3_sd(cpu0->cpu_int2_en0, cpu0->cpu_int2_enable0);
+			cpu0->cpu_int2_enable0 |= irq_mask;
 			mips3_sd(cpu0->cpu_int2_en0, cpu0->cpu_int2_enable0);
 #ifdef MULTIPROCESSOR
 			cpu1->cpu_int2_enable0 = cpu0->cpu_int2_enable0;
 			mips3_sd(cpu1->cpu_int2_en0, cpu1->cpu_int2_enable0);
 #endif
 		} else {
 			cpu0->cpu_int2_enable1 |= irq_mask;
 			mips3_sd(cpu0->cpu_int2_en1, cpu0->cpu_int2_enable1);
 #ifdef MULTIPROCESSOR
-		cpu1->cpu_int2_enable0 = cpu0->cpu_int2_enable0;
+			cpu1->cpu_int2_enable1 = cpu0->cpu_int2_enable1;
-		mips3_sd(cpu1->cpu_int2_en0, cpu1->cpu_int2_enable0);
+			mips3_sd(cpu1->cpu_int2_en1, cpu1->cpu_int2_enable1);
 #endif
+		}
 		break;
+	}
 	mutex_exit(&octeon_intr_lock);
 	return ih;
+}
 void
 octeon_intr_disestablish(void *cookie)
+{
 	struct octeon_intrhand * const ih = cookie;
 	const int irq = ih->ih_irq & (NIRQS-1);
 	const int ipl = ih->ih_ipl;
 	mutex_enter(&octeon_intr_lock);
 	/*
 	 * First disable it.
 	 */
-	const uint64_t irq_mask = ~__BIT(irq);
+	const int bank = irq / 64;
 	const uint64_t irq_mask = ~__BIT(irq % 64);
 	struct cpu_softc * const cpu0 = &octeon_cpu0_softc;
 #if MULTIPROCESSOR
 	struct cpu_softc * const cpu1 = &octeon_cpu1_softc;
 #endif
 	switch (ipl) {
 	case IPL_VM:
-		cpu0->cpu_int0_enable0 &= ~irq_mask;
+		if (bank == 0) {
-		mips3_sd(cpu0->cpu_int0_en0, cpu0->cpu_int0_enable0);
+			cpu0->cpu_int0_enable0 &= ~irq_mask;
 			mips3_sd(cpu0->cpu_int0_en0, cpu0->cpu_int0_enable0);
 		} else {
 			cpu0->cpu_int0_enable1 &= ~irq_mask;
 			mips3_sd(cpu0->cpu_int0_en1, cpu0->cpu_int0_enable1);
+		}
 		break;
 	case IPL_SCHED:
-		cpu0->cpu_int1_enable0 &= ~irq_mask;
+		if (bank == 0) {
-		mips3_sd(cpu0->cpu_int1_en0, cpu0->cpu_int1_enable0);
+			cpu0->cpu_int1_enable0 &= ~irq_mask;
 			mips3_sd(cpu0->cpu_int1_en0, cpu0->cpu_int1_enable0);
 #ifdef MULTIPROCESSOR
 			cpu1->cpu_int1_enable0 = cpu0->cpu_int1_enable0;
 			mips3_sd(cpu1->cpu_int1_en0, cpu1->cpu_int1_enable0);
 #endif
 		} else {
 			cpu0->cpu_int1_enable1 &= ~irq_mask;
 			mips3_sd(cpu0->cpu_int1_en1, cpu0->cpu_int1_enable1);
 #ifdef MULTIPROCESSOR
-		cpu1->cpu_int1_enable0 = cpu0->cpu_int1_enable0;
+			cpu1->cpu_int1_enable1 = cpu0->cpu_int1_enable1;
-		mips3_sd(cpu1->cpu_int1_en0, cpu1->cpu_int1_enable0);
+			mips3_sd(cpu1->cpu_int1_en1, cpu1->cpu_int1_enable1);
 #endif
+		}
 		break;
 	case IPL_DDB:
 	case IPL_HIGH:
-		cpu0->cpu_int2_enable0 &= ~irq_mask;
+		if (bank == 0) {
-		mips3_sd(cpu0->cpu_int2_en0, cpu0->cpu_int2_enable0);
+			cpu0->cpu_int2_enable0 &= ~irq_mask;
 			mips3_sd(cpu0->cpu_int2_en0, cpu0->cpu_int2_enable0);
 #ifdef MULTIPROCESSOR
 			cpu1->cpu_int2_enable0 = cpu0->cpu_int2_enable0;
 			mips3_sd(cpu1->cpu_int2_en0, cpu1->cpu_int2_enable0);
 #endif
 		} else {
 			cpu0->cpu_int2_enable1 &= ~irq_mask;
 			mips3_sd(cpu0->cpu_int2_en1, cpu0->cpu_int2_enable1);
 #ifdef MULTIPROCESSOR
-		cpu1->cpu_int2_enable0 = cpu0->cpu_int2_enable0;
+			cpu1->cpu_int2_enable1 = cpu0->cpu_int2_enable1;
-		mips3_sd(cpu1->cpu_int2_en0, cpu1->cpu_int2_enable0);
+			mips3_sd(cpu1->cpu_int2_en1, cpu1->cpu_int2_enable1);
 #endif
+		}
 		break;
+	}
 	/*
 	 * Now remove it since we shouldn't get interrupts for it.
 	 */
 	octciu_intrs[irq] = NULL;
 	mutex_exit(&octeon_intr_lock);
 	kmem_free(ih, sizeof(*ih));
+}
 void
 octeon_iointr(int ipl, vaddr_t pc, uint32_t ipending)
+{
 	struct cpu_info * const ci = curcpu();
 	struct cpu_softc * const cpu = ci->ci_softc;
 	int bank;
 	KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE);
 	KASSERT((ipending & ~MIPS_INT_MASK) == 0);
 	KASSERT(ipending & MIPS_HARD_INT_MASK);
-	uint64_t hwpend = 0;
+	uint64_t hwpend[2] = { 0, 0 };
 	const uint64_t sum1 = mips3_ld(cpu->cpu_int_sum1);
 	if (ipending & MIPS_INT_MASK_2) {
-		hwpend = mips3_ld(cpu->cpu_int2_sum0)
+		hwpend[0] = mips3_ld(cpu->cpu_int2_sum0)
 		    & cpu->cpu_int2_enable0;
 		hwpend[1] = sum1 & cpu->cpu_int2_enable1;
 	} else if (ipending & MIPS_INT_MASK_1) {
-		hwpend = mips3_ld(cpu->cpu_int1_sum0)
+		hwpend[0] = mips3_ld(cpu->cpu_int1_sum0)
 		    & cpu->cpu_int1_enable0;
 		hwpend[1] = sum1 & cpu->cpu_int1_enable1;
 	} else if (ipending & MIPS_INT_MASK_0) {
-		hwpend = mips3_ld(cpu->cpu_int0_sum0)
+		hwpend[0] = mips3_ld(cpu->cpu_int0_sum0)
 		    & cpu->cpu_int0_enable0;
 		hwpend[1] = sum1 & cpu->cpu_int0_enable1;
 	} else {
 		panic("octeon_iointr: unexpected ipending %#x", ipending);
+	}
-	while (hwpend != 0) {
+	for (bank = 0; bank <= 1; bank++) {
-		const int irq = ffs64(hwpend) - 1;
+		while (hwpend[bank] != 0) {
-		hwpend &= ~__BIT(irq);
+			const int bit = ffs64(hwpend[bank]) - 1;
 			const int irq = (bank * 64) + bit;
-		struct octeon_intrhand * const ih = octciu_intrs[irq];
+			hwpend[bank] &= ~__BIT(bit);
 		cpu->cpu_intr_evs[irq].ev_count++;
-		if (__predict_true(ih != NULL)) {
+			struct octeon_intrhand * const ih = octciu_intrs[irq];
-#ifdef MULTIPROCESSOR
+			cpu->cpu_intr_evs[irq].ev_count++;
-			if (ipl == IPL_VM) {
+			if (__predict_true(ih != NULL)) {
-				KERNEL_LOCK(1, NULL);
+#ifdef MULTIPROCESSOR
-#endif
+				if (ipl == IPL_VM) {
-				(*ih->ih_func)(ih->ih_arg);
+					KERNEL_LOCK(1, NULL);
-#ifdef MULTIPROCESSOR
+#endif
-				KERNEL_UNLOCK_ONE(NULL);
+					(*ih->ih_func)(ih->ih_arg);
-			} else {
+#ifdef MULTIPROCESSOR
-				(*ih->ih_func)(ih->ih_arg);
+					KERNEL_UNLOCK_ONE(NULL);
 				} else {
 					(*ih->ih_func)(ih->ih_arg);
+				}
 #endif
-			KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE);
+				KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE);
+			}
+		}
+	}
 	KDASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE);
+}
 #ifdef MULTIPROCESSOR
 __CTASSERT(NIPIS < 16);
 int
 octeon_ipi_intr(void *arg)
+{
 	struct cpu_info * const ci = curcpu();
 	struct cpu_softc * const cpu = ci->ci_softc;

 @@ -1,14 +1,14 @@
-/*	$NetBSD: octeonvar.h,v 1.13 2020/06/23 05:15:33 simonb Exp $	*/
+/*	$NetBSD: octeonvar.h,v 1.14 2020/07/16 21:33:50 jmcneill Exp $	*/
 /*-
  * Copyright (c) 2001 The NetBSD Foundation, Inc.
  * All rights reserved.
+ *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Jason R. Thorpe.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
 @@ -67,51 +67,57 @@ struct octeon_config {
 	struct mips_pci_chipset mc_pc;
 	struct mips_bus_dma_tag mc_iobus_dmat;
 	struct mips_bus_dma_tag mc_bootbus_dmat;
 	struct mips_bus_dma_tag mc_core1_dmat;
 	struct mips_bus_dma_tag mc_fpa_dmat;
 	struct extent *mc_io_ex;
 	struct extent *mc_mem_ex;
 	int	mc_mallocsafe;
 };
-#define NIRQS	64
+#define NIRQS	128
 struct cpu_softc {
 	struct cpu_info *cpu_ci;
 	uint64_t cpu_int0_sum0;
 	uint64_t cpu_int1_sum0;
 	uint64_t cpu_int2_sum0;
 	uint64_t cpu_int_sum1;
 	uint64_t cpu_int0_en0;
 	uint64_t cpu_int1_en0;
 	uint64_t cpu_int2_en0;
 	uint64_t cpu_int0_en1;
 	uint64_t cpu_int1_en1;
 	uint64_t cpu_int2_en1;
 	uint64_t cpu_int32_en;
 	struct evcnt cpu_intr_evs[NIRQS];
 	uint64_t cpu_int0_enable0;
 	uint64_t cpu_int1_enable0;
 	uint64_t cpu_int2_enable0;
 	uint64_t cpu_int0_enable1;
 	uint64_t cpu_int1_enable1;
 	uint64_t cpu_int2_enable1;
 	void *cpu_wdog_sih;		// wdog softint handler
 	uint64_t cpu_wdog;
 	uint64_t cpu_pp_poke;
 #ifdef MULTIPROCESSOR
 	uint64_t cpu_mbox_set;
 	uint64_t cpu_mbox_clr;
 #endif
 };
 /*
  * FPA map
  */