 @@ -1,2136 +1,2136 @@
-/*	$NetBSD: if_aq.c,v 1.17.2.2 2020/07/07 10:29:05 martin Exp $	*/
+/*	$NetBSD: if_aq.c,v 1.17.2.3 2020/07/07 12:02:29 martin Exp $	*/
 /**
  * aQuantia Corporation Network Driver
  * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
+ *
  *   (1) Redistributions of source code must retain the above
  *   copyright notice, this list of conditions and the following
  *   disclaimer.
+ *
  *   (2) Redistributions in binary form must reproduce the above
  *   copyright notice, this list of conditions and the following
  *   disclaimer in the documentation and/or other materials provided
  *   with the distribution.
+ *
  *   (3) The name of the author may not be used to endorse or promote
  *   products derived from this software without specific prior
  *   written permission.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
  * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
  */
 /*-
  * Copyright (c) 2020 Ryo Shimizu <ryo@nerv.org>
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_aq.c,v 1.17.2.2 2020/07/07 10:29:05 martin Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_aq.c,v 1.17.2.3 2020/07/07 12:02:29 martin Exp $");
 #ifdef _KERNEL_OPT
 #include "opt_if_aq.h"
 #include "sysmon_envsys.h"
 #endif
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/bitops.h>
 #include <sys/cprng.h>
 #include <sys/cpu.h>
 #include <sys/interrupt.h>
 #include <sys/module.h>
 #include <sys/pcq.h>
 #include <net/bpf.h>
 #include <net/if.h>
 #include <net/if_dl.h>
 #include <net/if_media.h>
 #include <net/if_ether.h>
 #include <net/rss_config.h>
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pcireg.h>
 #include <dev/pci/pcidevs.h>
 #include <dev/sysmon/sysmonvar.h>
 /* driver configuration */
 #define CONFIG_INTR_MODERATION_ENABLE	true	/* delayed interrupt */
 #undef CONFIG_LRO_SUPPORT			/* no LRO not suppoted */
 #undef CONFIG_NO_TXRX_INDEPENDENT		/* share TX/RX interrupts */
 #define AQ_NINTR_MAX			(AQ_RSSQUEUE_MAX + AQ_RSSQUEUE_MAX + 1)
 					/* TX + RX + LINK. must be <= 32 */
 #define AQ_LINKSTAT_IRQ			31	/* for legacy mode */
 #define AQ_TXD_NUM			2048	/* per ring. 8*n && 32~8184 */
 #define AQ_RXD_NUM			2048	/* per ring. 8*n && 32~8184 */
 /* minimum required to send a packet (vlan needs additional TX descriptor) */
 #define AQ_TXD_MIN			(1 + 1)
 /* hardware specification */
 #define AQ_RINGS_NUM			32
 #define AQ_RSSQUEUE_MAX			8
 #define AQ_RX_DESCRIPTOR_MIN		32
 #define AQ_TX_DESCRIPTOR_MIN		32
 #define AQ_RX_DESCRIPTOR_MAX		8184
 #define AQ_TX_DESCRIPTOR_MAX		8184
 #define AQ_TRAFFICCLASS_NUM		8
 #define AQ_RSS_HASHKEY_SIZE		40
 #define AQ_RSS_INDIRECTION_TABLE_MAX	64
 /*
  * TERMINOLOGY
  *	MPI = MAC PHY INTERFACE?
  *	RPO = RX Protocol Offloading
  *	TPO = TX Protocol Offloading
  *	RPF = RX Packet Filter
  *	TPB = TX Packet buffer
  *	RPB = RX Packet buffer
  */
 /* registers */
 #define AQ_FW_SOFTRESET_REG			0x0000
 #define  AQ_FW_SOFTRESET_RESET			__BIT(15) /* soft reset bit */
 #define  AQ_FW_SOFTRESET_DIS			__BIT(14) /* reset disable */
 #define AQ_FW_VERSION_REG			0x0018
 #define AQ_HW_REVISION_REG			0x001c
 #define AQ_GLB_NVR_INTERFACE1_REG		0x0100
 #define AQ_FW_MBOX_CMD_REG			0x0200
 #define  AQ_FW_MBOX_CMD_EXECUTE			0x00008000
 #define  AQ_FW_MBOX_CMD_BUSY			0x00000100
 #define AQ_FW_MBOX_ADDR_REG			0x0208
 #define AQ_FW_MBOX_VAL_REG			0x020c
 #define FW2X_LED_MIN_VERSION			0x03010026	/* >= 3.1.38 */
 #define FW2X_LED_REG				0x031c
 #define  FW2X_LED_DEFAULT			0x00000000
 #define  FW2X_LED_NONE				0x0000003f
 #define  FW2X_LINKLED				__BITS(0,1)
 #define   FW2X_LINKLED_ACTIVE			0
 #define   FW2X_LINKLED_ON			1
 #define   FW2X_LINKLED_BLINK			2
 #define   FW2X_LINKLED_OFF			3
 #define  FW2X_STATUSLED				__BITS(2,5)
 #define   FW2X_STATUSLED_ORANGE			0
 #define   FW2X_STATUSLED_ORANGE_BLINK		2
 #define   FW2X_STATUSLED_OFF			3
 #define   FW2X_STATUSLED_GREEN			4
 #define   FW2X_STATUSLED_ORANGE_GREEN_BLINK	8
 #define   FW2X_STATUSLED_GREEN_BLINK		10
 #define FW_MPI_MBOX_ADDR_REG			0x0360
 #define FW1X_MPI_INIT1_REG			0x0364
 #define FW1X_MPI_CONTROL_REG			0x0368
 #define FW1X_MPI_STATE_REG			0x036c
 #define  FW1X_MPI_STATE_MODE			__BITS(7,0)
 #define  FW1X_MPI_STATE_SPEED			__BITS(32,16)
 #define  FW1X_MPI_STATE_DISABLE_DIRTYWAKE	__BITS(25)
 #define  FW1X_MPI_STATE_DOWNSHIFT		__BITS(31,28)
 #define FW1X_MPI_INIT2_REG			0x0370
 #define FW1X_MPI_EFUSEADDR_REG			0x0374
 #define FW2X_MPI_EFUSEADDR_REG			0x0364
 #define FW2X_MPI_CONTROL_REG			0x0368	/* 64bit */
 #define FW2X_MPI_STATE_REG			0x0370	/* 64bit */
 #define FW_BOOT_EXIT_CODE_REG			0x0388
 #define  RBL_STATUS_DEAD			0x0000dead
 #define  RBL_STATUS_SUCCESS			0x0000abba
 #define  RBL_STATUS_FAILURE			0x00000bad
 #define  RBL_STATUS_HOST_BOOT			0x0000f1a7
 #define AQ_FW_GLB_CPU_SEM_REG(i)		(0x03a0 + (i) * 4)
 #define AQ_FW_SEM_RAM_REG			AQ_FW_GLB_CPU_SEM_REG(2)
 #define AQ_FW_GLB_CTL2_REG			0x0404
 #define  AQ_FW_GLB_CTL2_MCP_UP_FORCE_INTERRUPT	__BIT(1)
 #define AQ_GLB_GENERAL_PROVISIONING9_REG	0x0520
 #define AQ_GLB_NVR_PROVISIONING2_REG		0x0534
 #define FW_MPI_DAISY_CHAIN_STATUS_REG		0x0704
 #define AQ_PCI_REG_CONTROL_6_REG		0x1014
 // msix bitmap */
 #define AQ_INTR_STATUS_REG			0x2000	/* intr status */
 #define AQ_INTR_STATUS_CLR_REG			0x2050	/* intr status clear */
 #define AQ_INTR_MASK_REG			0x2060	/* intr mask set */
 #define AQ_INTR_MASK_CLR_REG			0x2070	/* intr mask clear */
 #define AQ_INTR_AUTOMASK_REG			0x2090
 /* AQ_INTR_IRQ_MAP_TXRX_REG[AQ_RINGS_NUM] 0x2100-0x2140 */
 #define AQ_INTR_IRQ_MAP_TXRX_REG(i)		(0x2100 + ((i) / 2) * 4)
 #define AQ_INTR_IRQ_MAP_TX_REG(i)		AQ_INTR_IRQ_MAP_TXRX_REG(i)
 #define  AQ_INTR_IRQ_MAP_TX_IRQMAP(i)		(__BITS(28,24) >> (((i) & 1)*8))
 #define  AQ_INTR_IRQ_MAP_TX_EN(i)		(__BIT(31)     >> (((i) & 1)*8))
 #define AQ_INTR_IRQ_MAP_RX_REG(i)		AQ_INTR_IRQ_MAP_TXRX_REG(i)
 #define  AQ_INTR_IRQ_MAP_RX_IRQMAP(i)		(__BITS(12,8)  >> (((i) & 1)*8))
 #define  AQ_INTR_IRQ_MAP_RX_EN(i)		(__BIT(15)     >> (((i) & 1)*8))
 /* AQ_GEN_INTR_MAP_REG[AQ_RINGS_NUM] 0x2180-0x2200 */
 #define AQ_GEN_INTR_MAP_REG(i)			(0x2180 + (i) * 4)
 #define  AQ_B0_ERR_INT				8U
 #define AQ_INTR_CTRL_REG			0x2300
 #define  AQ_INTR_CTRL_IRQMODE			__BITS(1,0)
 #define  AQ_INTR_CTRL_IRQMODE_LEGACY		0
 #define  AQ_INTR_CTRL_IRQMODE_MSI		1
 #define  AQ_INTR_CTRL_IRQMODE_MSIX		2
 #define  AQ_INTR_CTRL_MULTIVEC			__BIT(2)
 #define  AQ_INTR_CTRL_AUTO_MASK			__BIT(5)
 #define  AQ_INTR_CTRL_CLR_ON_READ		__BIT(7)
 #define  AQ_INTR_CTRL_RESET_DIS			__BIT(29)
 #define  AQ_INTR_CTRL_RESET_IRQ			__BIT(31)
 #define AQ_MBOXIF_POWER_GATING_CONTROL_REG	0x32a8
 #define FW_MPI_RESETCTRL_REG			0x4000
 #define  FW_MPI_RESETCTRL_RESET_DIS		__BIT(29)
 #define RX_SYSCONTROL_REG			0x5000
 #define  RX_SYSCONTROL_RPB_DMA_LOOPBACK		__BIT(6)
 #define  RX_SYSCONTROL_RPF_TPO_LOOPBACK		__BIT(8)
 #define  RX_SYSCONTROL_RESET_DIS		__BIT(29)
 #define RX_TCP_RSS_HASH_REG			0x5040
 #define  RX_TCP_RSS_HASH_RPF2			__BITS(19,16)
 #define  RX_TCP_RSS_HASH_TYPE			__BITS(15,0)
 /* for RPF_*_REG.ACTION */
 #define RPF_ACTION_DISCARD			0
 #define RPF_ACTION_HOST				1
 #define RPF_ACTION_MANAGEMENT			2
 #define RPF_ACTION_HOST_MANAGEMENT		3
 #define RPF_ACTION_WOL				4
 #define RPF_L2BC_REG				0x5100
 #define  RPF_L2BC_EN				__BIT(0)
 #define  RPF_L2BC_PROMISC			__BIT(3)
 #define  RPF_L2BC_ACTION			__BITS(12,14)
 #define  RPF_L2BC_THRESHOLD			__BITS(31,16)
 /* RPF_L2UC_*_REG[34] (actual [38]?) */
 #define RPF_L2UC_LSW_REG(i)			(0x5110 + (i) * 8)
 #define RPF_L2UC_MSW_REG(i)			(0x5114 + (i) * 8)
 #define  RPF_L2UC_MSW_MACADDR_HI		__BITS(15,0)
 #define  RPF_L2UC_MSW_ACTION			__BITS(18,16)
 #define  RPF_L2UC_MSW_EN			__BIT(31)
 #define AQ_HW_MAC_OWN			0	/* index of own address */
 #define AQ_HW_MAC_NUM			34
 /* RPF_MCAST_FILTER_REG[8] 0x5250-0x5270 */
 #define RPF_MCAST_FILTER_REG(i)			(0x5250 + (i) * 4)
 #define  RPF_MCAST_FILTER_EN			__BIT(31)
 #define RPF_MCAST_FILTER_MASK_REG		0x5270
 #define  RPF_MCAST_FILTER_MASK_ALLMULTI		__BIT(14)
 #define RPF_VLAN_MODE_REG			0x5280
 #define  RPF_VLAN_MODE_PROMISC			__BIT(1)
 #define  RPF_VLAN_MODE_ACCEPT_UNTAGGED		__BIT(2)
 #define  RPF_VLAN_MODE_UNTAGGED_ACTION		__BITS(5,3)
 #define RPF_VLAN_TPID_REG			0x5284
 #define  RPF_VLAN_TPID_OUTER			__BITS(31,16)
 #define  RPF_VLAN_TPID_INNER			__BITS(15,0)
 /* RPF_VLAN_FILTER_REG[RPF_VLAN_MAX_FILTERS] 0x5290-0x52d0 */
 #define RPF_VLAN_MAX_FILTERS			16
 #define RPF_VLAN_FILTER_REG(i)			(0x5290 + (i) * 4)
 #define  RPF_VLAN_FILTER_EN			__BIT(31)
 #define  RPF_VLAN_FILTER_RXQ_EN			__BIT(28)
 #define  RPF_VLAN_FILTER_RXQ			__BITS(24,20)
 #define  RPF_VLAN_FILTER_ACTION			__BITS(18,16)
 #define  RPF_VLAN_FILTER_ID			__BITS(11,0)
 /* RPF_ETHERTYPE_FILTER_REG[AQ_RINGS_NUM] 0x5300-0x5380 */
 #define RPF_ETHERTYPE_FILTER_REG(i)		(0x5300 + (i) * 4)
 #define  RPF_ETHERTYPE_FILTER_EN		__BIT(31)
 #define  RPF_ETHERTYPE_FILTER_PRIO_EN		__BIT(30)
 #define  RPF_ETHERTYPE_FILTER_RXQF_EN		__BIT(29)
 #define  RPF_ETHERTYPE_FILTER_PRIO		__BITS(28,26)
 #define  RPF_ETHERTYPE_FILTER_RXQF		__BITS(24,20)
 #define  RPF_ETHERTYPE_FILTER_MNG_RXQF		__BIT(19)
 #define  RPF_ETHERTYPE_FILTER_ACTION		__BITS(18,16)
 #define  RPF_ETHERTYPE_FILTER_VAL		__BITS(15,0)
 /* RPF_L3_FILTER_REG[8] 0x5380-0x53a0 */
 #define RPF_L3_FILTER_REG(i)			(0x5380 + (i) * 4)
 #define  RPF_L3_FILTER_L4_EN			__BIT(31)
 #define  RPF_L3_FILTER_IPV6_EN			__BIT(30)
 #define  RPF_L3_FILTER_SRCADDR_EN		__BIT(29)
 #define  RPF_L3_FILTER_DSTADDR_EN		__BIT(28)
 #define  RPF_L3_FILTER_L4_SRCPORT_EN		__BIT(27)
 #define  RPF_L3_FILTER_L4_DSTPORT_EN		__BIT(26)
 #define  RPF_L3_FILTER_L4_PROTO_EN		__BIT(25)
 #define  RPF_L3_FILTER_ARP_EN			__BIT(24)
 #define  RPF_L3_FILTER_L4_RXQUEUE_EN		__BIT(23)
 #define  RPF_L3_FILTER_L4_RXQUEUE_MANAGEMENT_EN	__BIT(22)
 #define  RPF_L3_FILTER_L4_ACTION		__BITS(16,18)
 #define  RPF_L3_FILTER_L4_RXQUEUE		__BITS(12,8)
 #define  RPF_L3_FILTER_L4_PROTO			__BITS(2,0)
 #define   RPF_L3_FILTER_L4_PROTO_TCP		0
 #define   RPF_L3_FILTER_L4_PROTO_UDP		1
 #define   RPF_L3_FILTER_L4_PROTO_SCTP		2
 #define   RPF_L3_FILTER_L4_PROTO_ICMP		3
 /* parameters of RPF_L3_FILTER_REG[8] */
 #define RPF_L3_FILTER_SRCADDR_REG(i)		(0x53b0 + (i) * 4)
 #define RPF_L3_FILTER_DSTADDR_REG(i)		(0x53d0 + (i) * 4)
 #define RPF_L3_FILTER_L4_SRCPORT_REG(i)		(0x5400 + (i) * 4)
 #define RPF_L3_FILTER_L4_DSTPORT_REG(i)		(0x5420 + (i) * 4)
 #define RX_FLR_RSS_CONTROL1_REG			0x54c0
 #define  RX_FLR_RSS_CONTROL1_EN			__BIT(31)
 #define RPF_RPB_RX_TC_UPT_REG			0x54c4
 #define  RPF_RPB_RX_TC_UPT_MASK(i)		(0x00000007 << ((i) * 4))
 #define RPF_RSS_KEY_ADDR_REG			0x54d0
 #define  RPF_RSS_KEY_ADDR			__BITS(4,0)
 #define  RPF_RSS_KEY_WR_EN			__BIT(5)
 #define RPF_RSS_KEY_WR_DATA_REG			0x54d4
 #define RPF_RSS_KEY_RD_DATA_REG			0x54d8
 #define RPF_RSS_REDIR_ADDR_REG			0x54e0
 #define  RPF_RSS_REDIR_ADDR			__BITS(3,0)
 #define  RPF_RSS_REDIR_WR_EN			__BIT(4)
 #define RPF_RSS_REDIR_WR_DATA_REG		0x54e4
 #define  RPF_RSS_REDIR_WR_DATA			__BITS(15,0)
 #define RPO_HWCSUM_REG				0x5580
 #define  RPO_HWCSUM_IP4CSUM_EN			__BIT(1)
 #define  RPO_HWCSUM_L4CSUM_EN			__BIT(0) /* TCP/UDP/SCTP */
 #define RPO_LRO_ENABLE_REG			0x5590
 #define RPO_LRO_CONF_REG			0x5594
 #define  RPO_LRO_CONF_QSESSION_LIMIT		__BITS(13,12)
 #define  RPO_LRO_CONF_TOTAL_DESC_LIMIT		__BITS(6,5)
 #define  RPO_LRO_CONF_PATCHOPTIMIZATION_EN	__BIT(15)
 #define  RPO_LRO_CONF_MIN_PAYLOAD_OF_FIRST_PKT	__BITS(4,0)
 #define RPO_LRO_RSC_MAX_REG			0x5598
 /* RPO_LRO_LDES_MAX_REG[32/8] 0x55a0-0x55b0 */
 #define RPO_LRO_LDES_MAX_REG(i)			(0x55a0 + (i / 8) * 4)
 #define  RPO_LRO_LDES_MAX_MASK(i)		(0x00000003 << ((i & 7) * 4))
 #define RPO_LRO_TB_DIV_REG			0x5620
 #define  RPO_LRO_TB_DIV				__BITS(20,31)
 #define RPO_LRO_INACTIVE_IVAL_REG		0x5620
 #define  RPO_LRO_INACTIVE_IVAL			__BITS(10,19)
 #define RPO_LRO_MAX_COALESCING_IVAL_REG		0x5620
 #define  RPO_LRO_MAX_COALESCING_IVAL		__BITS(9,0)
 #define RPB_RPF_RX_REG				0x5700
 #define  RPB_RPF_RX_TC_MODE			__BIT(8)
 #define  RPB_RPF_RX_FC_MODE			__BITS(5,4)
 #define  RPB_RPF_RX_BUF_EN			__BIT(0)
 /* RPB_RXB_BUFSIZE_REG[AQ_TRAFFICCLASS_NUM] 0x5710-0x5790 */
 #define RPB_RXB_BUFSIZE_REG(i)			(0x5710 + (i) * 0x10)
 #define  RPB_RXB_BUFSIZE			__BITS(8,0)
 #define RPB_RXB_XOFF_REG(i)			(0x5714 + (i) * 0x10)
 #define  RPB_RXB_XOFF_EN			__BIT(31)
 #define  RPB_RXB_XOFF_THRESH_HI			__BITS(29,16)
 #define  RPB_RXB_XOFF_THRESH_LO			__BITS(13,0)
 #define RX_DMA_DESC_CACHE_INIT_REG		0x5a00
 #define  RX_DMA_DESC_CACHE_INIT			__BIT(0)
 #define RX_DMA_INT_DESC_WRWB_EN_REG		0x05a30
 #define  RX_DMA_INT_DESC_WRWB_EN		__BIT(2)
 #define  RX_DMA_INT_DESC_MODERATE_EN		__BIT(3)
 /* RX_INTR_MODERATION_CTL_REG[AQ_RINGS_NUM] 0x5a40-0x5ac0 */
 #define RX_INTR_MODERATION_CTL_REG(i)		(0x5a40 + (i) * 4)
 #define  RX_INTR_MODERATION_CTL_EN		__BIT(1)
 #define  RX_INTR_MODERATION_CTL_MIN		__BITS(15,8)
 #define  RX_INTR_MODERATION_CTL_MAX		__BITS(24,16)
 /* RX_DMA_DESC_*[AQ_RINGS_NUM] 0x5b00-0x5f00 */
 #define RX_DMA_DESC_BASE_ADDRLSW_REG(i)		(0x5b00 + (i) * 0x20)
 #define RX_DMA_DESC_BASE_ADDRMSW_REG(i)		(0x5b04 + (i) * 0x20)
 #define RX_DMA_DESC_REG(i)			(0x5b08 + (i) * 0x20)
 #define  RX_DMA_DESC_LEN			__BITS(12,3)	/* RXD_NUM/8 */
 #define  RX_DMA_DESC_RESET			__BIT(25)
 #define  RX_DMA_DESC_HEADER_SPLIT		__BIT(28)
 #define  RX_DMA_DESC_VLAN_STRIP			__BIT(29)
 #define  RX_DMA_DESC_EN				__BIT(31)
 #define RX_DMA_DESC_HEAD_PTR_REG(i)		(0x5b0c + (i) * 0x20)
 #define  RX_DMA_DESC_HEAD_PTR			__BITS(12,0)
 #define RX_DMA_DESC_TAIL_PTR_REG(i)		(0x5b10 + (i) * 0x20)
 #define RX_DMA_DESC_BUFSIZE_REG(i)		(0x5b18 + (i) * 0x20)
 #define  RX_DMA_DESC_BUFSIZE_DATA		__BITS(4,0)
 #define  RX_DMA_DESC_BUFSIZE_HDR		__BITS(12,8)
 /* RX_DMA_DCAD_REG[AQ_RINGS_NUM] 0x6100-0x6180 */
 #define RX_DMA_DCAD_REG(i)			(0x6100 + (i) * 4)
 #define  RX_DMA_DCAD_CPUID			__BITS(7,0)
 #define  RX_DMA_DCAD_PAYLOAD_EN			__BIT(29)
 #define  RX_DMA_DCAD_HEADER_EN			__BIT(30)
 #define  RX_DMA_DCAD_DESC_EN			__BIT(31)
 #define RX_DMA_DCA_REG				0x6180
 #define  RX_DMA_DCA_EN				__BIT(31)
 #define  RX_DMA_DCA_MODE			__BITS(3,0)
 /* counters */
 #define RX_DMA_GOOD_PKT_COUNTERLSW		0x6800
 #define RX_DMA_GOOD_OCTET_COUNTERLSW		0x6808
 #define RX_DMA_DROP_PKT_CNT_REG			0x6818
 #define RX_DMA_COALESCED_PKT_CNT_REG		0x6820
 #define TX_SYSCONTROL_REG			0x7000
 #define  TX_SYSCONTROL_TPB_DMA_LOOPBACK		__BIT(6)
 #define  TX_SYSCONTROL_TPO_PKT_LOOPBACK		__BIT(7)
 #define  TX_SYSCONTROL_RESET_DIS		__BIT(29)
 #define TX_TPO2_REG				0x7040
 #define  TX_TPO2_EN				__BIT(16)
 #define TPS_DESC_VM_ARB_MODE_REG		0x7300
 #define  TPS_DESC_VM_ARB_MODE			__BIT(0)
 #define TPS_DESC_RATE_REG			0x7310
 #define  TPS_DESC_RATE_TA_RST			__BIT(31)
 #define  TPS_DESC_RATE_LIM			__BITS(10,0)
 #define TPS_DESC_TC_ARB_MODE_REG		0x7200
 #define  TPS_DESC_TC_ARB_MODE			__BITS(1,0)
 #define TPS_DATA_TC_ARB_MODE_REG		0x7100
 #define  TPS_DATA_TC_ARB_MODE			__BIT(0)
 /* TPS_DATA_TCT_REG[AQ_TRAFFICCLASS_NUM] 0x7110-0x7130 */
 #define TPS_DATA_TCT_REG(i)			(0x7110 + (i) * 4)
 #define  TPS_DATA_TCT_CREDIT_MAX		__BITS(16,27)
 #define  TPS_DATA_TCT_WEIGHT			__BITS(8,0)
 /* TPS_DATA_TCT_REG[AQ_TRAFFICCLASS_NUM] 0x7210-0x7230 */
 #define TPS_DESC_TCT_REG(i)			(0x7210 + (i) * 4)
 #define  TPS_DESC_TCT_CREDIT_MAX		__BITS(16,27)
 #define  TPS_DESC_TCT_WEIGHT			__BITS(8,0)
 #define AQ_HW_TXBUF_MAX		160
 #define AQ_HW_RXBUF_MAX		320
 #define TPO_HWCSUM_REG				0x7800
 #define  TPO_HWCSUM_IP4CSUM_EN			__BIT(1)
 #define  TPO_HWCSUM_L4CSUM_EN			__BIT(0) /* TCP/UDP/SCTP */
 #define TDM_LSO_EN_REG				0x7810
 #define THM_LSO_TCP_FLAG1_REG			0x7820
 #define  THM_LSO_TCP_FLAG1_FIRST		__BITS(11,0)
 #define  THM_LSO_TCP_FLAG1_MID			__BITS(27,16)
 #define THM_LSO_TCP_FLAG2_REG			0x7824
 #define  THM_LSO_TCP_FLAG2_LAST			__BITS(11,0)
 #define TPB_TX_BUF_REG				0x7900
 #define  TPB_TX_BUF_EN				__BIT(0)
 #define  TPB_TX_BUF_SCP_INS_EN			__BIT(2)
 #define  TPB_TX_BUF_TC_MODE_EN			__BIT(8)
 /* TPB_TXB_BUFSIZE_REG[AQ_TRAFFICCLASS_NUM] 0x7910-7990 */
 #define TPB_TXB_BUFSIZE_REG(i)			(0x7910 + (i) * 0x10)
 #define  TPB_TXB_BUFSIZE			__BITS(7,0)
 #define TPB_TXB_THRESH_REG(i)			(0x7914 + (i) * 0x10)
 #define  TPB_TXB_THRESH_HI			__BITS(16,28)
 #define  TPB_TXB_THRESH_LO			__BITS(12,0)
 #define AQ_HW_TX_DMA_TOTAL_REQ_LIMIT_REG	0x7b20
 #define TX_DMA_INT_DESC_WRWB_EN_REG		0x7b40
 #define  TX_DMA_INT_DESC_WRWB_EN		__BIT(1)
 #define  TX_DMA_INT_DESC_MODERATE_EN		__BIT(4)
 /* TX_DMA_DESC_*[AQ_RINGS_NUM] 0x7c00-0x8400 */
 #define TX_DMA_DESC_BASE_ADDRLSW_REG(i)		(0x7c00 + (i) * 0x40)
 #define TX_DMA_DESC_BASE_ADDRMSW_REG(i)		(0x7c04 + (i) * 0x40)
 #define TX_DMA_DESC_REG(i)			(0x7c08 + (i) * 0x40)
 #define  TX_DMA_DESC_LEN			__BITS(12, 3)	/* TXD_NUM/8 */
 #define  TX_DMA_DESC_EN				__BIT(31)
 #define TX_DMA_DESC_HEAD_PTR_REG(i)		(0x7c0c + (i) * 0x40)
 #define  TX_DMA_DESC_HEAD_PTR			__BITS(12,0)
 #define TX_DMA_DESC_TAIL_PTR_REG(i)		(0x7c10 + (i) * 0x40)
 #define TX_DMA_DESC_WRWB_THRESH_REG(i)		(0x7c18 + (i) * 0x40)
 #define  TX_DMA_DESC_WRWB_THRESH		__BITS(14,8)
 /* TDM_DCAD_REG[AQ_RINGS_NUM] 0x8400-0x8480 */
 #define TDM_DCAD_REG(i)				(0x8400 + (i) * 4)
 #define  TDM_DCAD_CPUID				__BITS(7,0)
 #define  TDM_DCAD_CPUID_EN			__BIT(31)
 #define TDM_DCA_REG				0x8480
 #define  TDM_DCA_EN				__BIT(31)
 #define  TDM_DCA_MODE				__BITS(3,0)
 /* TX_INTR_MODERATION_CTL_REG[AQ_RINGS_NUM] 0x8980-0x8a00 */
 #define TX_INTR_MODERATION_CTL_REG(i)		(0x8980 + (i) * 4)
 #define  TX_INTR_MODERATION_CTL_EN		__BIT(1)
 #define  TX_INTR_MODERATION_CTL_MIN		__BITS(15,8)
 #define  TX_INTR_MODERATION_CTL_MAX		__BITS(24,16)
 #define FW1X_CTRL_10G				__BIT(0)
 #define FW1X_CTRL_5G				__BIT(1)
 #define FW1X_CTRL_5GSR				__BIT(2)
 #define FW1X_CTRL_2G5				__BIT(3)
 #define FW1X_CTRL_1G				__BIT(4)
 #define FW1X_CTRL_100M				__BIT(5)
 #define FW2X_CTRL_10BASET_HD			__BIT(0)
 #define FW2X_CTRL_10BASET_FD			__BIT(1)
 #define FW2X_CTRL_100BASETX_HD			__BIT(2)
 #define FW2X_CTRL_100BASET4_HD			__BIT(3)
 #define FW2X_CTRL_100BASET2_HD			__BIT(4)
 #define FW2X_CTRL_100BASETX_FD			__BIT(5)
 #define FW2X_CTRL_100BASET2_FD			__BIT(6)
 #define FW2X_CTRL_1000BASET_HD			__BIT(7)
 #define FW2X_CTRL_1000BASET_FD			__BIT(8)
 #define FW2X_CTRL_2P5GBASET_FD			__BIT(9)
 #define FW2X_CTRL_5GBASET_FD			__BIT(10)
 #define FW2X_CTRL_10GBASET_FD			__BIT(11)
 #define FW2X_CTRL_RESERVED1			__BIT(32)
 #define FW2X_CTRL_10BASET_EEE			__BIT(33)
 #define FW2X_CTRL_RESERVED2			__BIT(34)
 #define FW2X_CTRL_PAUSE				__BIT(35)
 #define FW2X_CTRL_ASYMMETRIC_PAUSE		__BIT(36)
 #define FW2X_CTRL_100BASETX_EEE			__BIT(37)
 #define FW2X_CTRL_RESERVED3			__BIT(38)
 #define FW2X_CTRL_RESERVED4			__BIT(39)
 #define FW2X_CTRL_1000BASET_FD_EEE		__BIT(40)
 #define FW2X_CTRL_2P5GBASET_FD_EEE		__BIT(41)
 #define FW2X_CTRL_5GBASET_FD_EEE		__BIT(42)
 #define FW2X_CTRL_10GBASET_FD_EEE		__BIT(43)
 #define FW2X_CTRL_RESERVED5			__BIT(44)
 #define FW2X_CTRL_RESERVED6			__BIT(45)
 #define FW2X_CTRL_RESERVED7			__BIT(46)
 #define FW2X_CTRL_RESERVED8			__BIT(47)
 #define FW2X_CTRL_RESERVED9			__BIT(48)
 #define FW2X_CTRL_CABLE_DIAG			__BIT(49)
 #define FW2X_CTRL_TEMPERATURE			__BIT(50)
 #define FW2X_CTRL_DOWNSHIFT			__BIT(51)
 #define FW2X_CTRL_PTP_AVB_EN			__BIT(52)
 #define FW2X_CTRL_MEDIA_DETECT			__BIT(53)
 #define FW2X_CTRL_LINK_DROP			__BIT(54)
 #define FW2X_CTRL_SLEEP_PROXY			__BIT(55)
 #define FW2X_CTRL_WOL				__BIT(56)
 #define FW2X_CTRL_MAC_STOP			__BIT(57)
 #define FW2X_CTRL_EXT_LOOPBACK			__BIT(58)
 #define FW2X_CTRL_INT_LOOPBACK			__BIT(59)
 #define FW2X_CTRL_EFUSE_AGENT			__BIT(60)
 #define FW2X_CTRL_WOL_TIMER			__BIT(61)
 #define FW2X_CTRL_STATISTICS			__BIT(62)
 #define FW2X_CTRL_TRANSACTION_ID		__BIT(63)
 #define FW2X_SNPRINTB			\
 	"\177\020"			\
 	"b\x23" "PAUSE\0"		\
 	"b\x24" "ASYMMETRIC-PAUSE\0"	\
 	"b\x31" "CABLE-DIAG\0"		\
 	"b\x32" "TEMPERATURE\0"		\
 	"b\x33" "DOWNSHIFT\0"		\
 	"b\x34" "PTP-AVB\0"		\
 	"b\x35" "MEDIA-DETECT\0"	\
 	"b\x36" "LINK-DROP\0"		\
 	"b\x37" "SLEEP-PROXY\0"		\
 	"b\x38" "WOL\0"			\
 	"b\x39" "MAC-STOP\0"		\
 	"b\x3a" "EXT-LOOPBACK\0"	\
 	"b\x3b" "INT-LOOPBACK\0"	\
 	"b\x3c" "EFUSE-AGENT\0"		\
 	"b\x3d" "WOL-TIMER\0"		\
 	"b\x3e" "STATISTICS\0"		\
 	"b\x3f" "TRANSACTION-ID\0"	\
 	"\0"
 #define FW2X_CTRL_RATE_100M			FW2X_CTRL_100BASETX_FD
 #define FW2X_CTRL_RATE_1G			FW2X_CTRL_1000BASET_FD
 #define FW2X_CTRL_RATE_2G5			FW2X_CTRL_2P5GBASET_FD
 #define FW2X_CTRL_RATE_5G			FW2X_CTRL_5GBASET_FD
 #define FW2X_CTRL_RATE_10G			FW2X_CTRL_10GBASET_FD
 #define FW2X_CTRL_RATE_MASK		\
 	(FW2X_CTRL_RATE_100M |		\
 	 FW2X_CTRL_RATE_1G |		\
 	 FW2X_CTRL_RATE_2G5 |		\
 	 FW2X_CTRL_RATE_5G |		\
 	 FW2X_CTRL_RATE_10G)
 #define FW2X_CTRL_EEE_MASK		\
 	(FW2X_CTRL_10BASET_EEE |	\
 	 FW2X_CTRL_100BASETX_EEE |	\
 	 FW2X_CTRL_1000BASET_FD_EEE |	\
 	 FW2X_CTRL_2P5GBASET_FD_EEE |	\
 	 FW2X_CTRL_5GBASET_FD_EEE |	\
 	 FW2X_CTRL_10GBASET_FD_EEE)
 typedef enum aq_fw_bootloader_mode {
 	FW_BOOT_MODE_UNKNOWN = 0,
 	FW_BOOT_MODE_FLB,
 	FW_BOOT_MODE_RBL_FLASH,
 	FW_BOOT_MODE_RBL_HOST_BOOTLOAD
 } aq_fw_bootloader_mode_t;
 #define AQ_WRITE_REG(sc, reg, val)				\
 	bus_space_write_4((sc)->sc_iot, (sc)->sc_ioh, (reg), (val))
 #define AQ_READ_REG(sc, reg)					\
 	bus_space_read_4((sc)->sc_iot, (sc)->sc_ioh, (reg))
 #define AQ_READ64_REG(sc, reg)					\
 	((uint64_t)AQ_READ_REG(sc, reg) |			\
 	(((uint64_t)AQ_READ_REG(sc, (reg) + 4)) << 32))
 #define AQ_WRITE64_REG(sc, reg, val)				\
 	do {							\
 		AQ_WRITE_REG(sc, reg, (uint32_t)val);		\
 		AQ_WRITE_REG(sc, reg + 4, (uint32_t)(val >> 32)); \
 	} while (/* CONSTCOND */0)
 #define AQ_READ_REG_BIT(sc, reg, mask)				\
 	__SHIFTOUT(AQ_READ_REG(sc, reg), mask)
 #define AQ_WRITE_REG_BIT(sc, reg, mask, val)			\
 	do {							\
 		uint32_t _v;					\
 		_v = AQ_READ_REG((sc), (reg));			\
 		_v &= ~(mask);					\
 		if ((val) != 0)					\
 			_v |= __SHIFTIN((val), (mask));		\
 		AQ_WRITE_REG((sc), (reg), _v);			\
 	} while (/* CONSTCOND */ 0)
 #define WAIT_FOR(expr, us, n, errp)				\
 	do {							\
 		unsigned int _n;				\
 		for (_n = n; (!(expr)) && _n != 0; --_n) {	\
 			delay((us));				\
 		}						\
 		if ((errp != NULL)) {				\
 			if (_n == 0)				\
 				*(errp) = ETIMEDOUT;		\
 			else					\
 				*(errp) = 0;			\
 		}						\
 	} while (/* CONSTCOND */ 0)
 #define msec_delay(x)	DELAY(1000 * (x))
 typedef struct aq_mailbox_header {
 	uint32_t version;
 	uint32_t transaction_id;
 	int32_t error;
 } __packed aq_mailbox_header_t;
 typedef struct aq_hw_stats_s {
 	uint32_t uprc;
 	uint32_t mprc;
 	uint32_t bprc;
 	uint32_t erpt;
 	uint32_t uptc;
 	uint32_t mptc;
 	uint32_t bptc;
 	uint32_t erpr;
 	uint32_t mbtc;
 	uint32_t bbtc;
 	uint32_t mbrc;
 	uint32_t bbrc;
 	uint32_t ubrc;
 	uint32_t ubtc;
 	uint32_t ptc;
 	uint32_t prc;
 	uint32_t dpc;	/* not exists in fw2x_msm_statistics */
 	uint32_t cprc;	/* not exists in fw2x_msm_statistics */
 } __packed aq_hw_stats_s_t;
 typedef struct fw1x_mailbox {
 	aq_mailbox_header_t header;
 	aq_hw_stats_s_t msm;
 } __packed fw1x_mailbox_t;
 typedef struct fw2x_msm_statistics {
 	uint32_t uprc;
 	uint32_t mprc;
 	uint32_t bprc;
 	uint32_t erpt;
 	uint32_t uptc;
 	uint32_t mptc;
 	uint32_t bptc;
 	uint32_t erpr;
 	uint32_t mbtc;
 	uint32_t bbtc;
 	uint32_t mbrc;
 	uint32_t bbrc;
 	uint32_t ubrc;
 	uint32_t ubtc;
 	uint32_t ptc;
 	uint32_t prc;
 } __packed fw2x_msm_statistics_t;
 typedef struct fw2x_phy_cable_diag_data {
 	uint32_t lane_data[4];
 } __packed fw2x_phy_cable_diag_data_t;
 typedef struct fw2x_capabilities {
 	uint32_t caps_lo;
 	uint32_t caps_hi;
 } __packed fw2x_capabilities_t;
 typedef struct fw2x_mailbox {		/* struct fwHostInterface */
 	aq_mailbox_header_t header;
 	fw2x_msm_statistics_t msm;	/* msmStatistics_t msm; */
 	uint32_t phy_info1;
 #define PHYINFO1_FAULT_CODE	__BITS(31,16)
 #define PHYINFO1_PHY_H_BIT	__BITS(0,15)
 	uint32_t phy_info2;
 #define PHYINFO2_TEMPERATURE	__BITS(15,0)
 #define PHYINFO2_CABLE_LEN	__BITS(23,16)
 	fw2x_phy_cable_diag_data_t diag_data;
 	uint32_t reserved[8];
 	fw2x_capabilities_t caps;
 	/* ... */
 } __packed fw2x_mailbox_t;
 typedef enum aq_link_speed {
 	AQ_LINK_NONE	= 0,
 	AQ_LINK_100M	= (1 << 0),
 	AQ_LINK_1G	= (1 << 1),
 	AQ_LINK_2G5	= (1 << 2),
 	AQ_LINK_5G	= (1 << 3),
 	AQ_LINK_10G	= (1 << 4)
 } aq_link_speed_t;
 #define AQ_LINK_ALL	(AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | \
 			 AQ_LINK_5G | AQ_LINK_10G )
 #define AQ_LINK_AUTO	AQ_LINK_ALL
 typedef enum aq_link_fc {
 	AQ_FC_NONE = 0,
 	AQ_FC_RX = __BIT(0),
 	AQ_FC_TX = __BIT(1),
 	AQ_FC_ALL = (AQ_FC_RX | AQ_FC_TX)
 } aq_link_fc_t;
 typedef enum aq_link_eee {
 	AQ_EEE_DISABLE = 0,
 	AQ_EEE_ENABLE = 1
 } aq_link_eee_t;
 typedef enum aq_hw_fw_mpi_state {
 	MPI_DEINIT	= 0,
 	MPI_RESET	= 1,
 	MPI_INIT	= 2,
 	MPI_POWER	= 4
 } aq_hw_fw_mpi_state_t;
 enum aq_media_type {
 	AQ_MEDIA_TYPE_UNKNOWN = 0,
 	AQ_MEDIA_TYPE_FIBRE,
 	AQ_MEDIA_TYPE_TP
 };
 struct aq_rx_desc_read {
 	uint64_t buf_addr;
 	uint64_t hdr_addr;
 } __packed;
 struct aq_rx_desc_wb {
 	uint32_t type;
 #define RXDESC_TYPE_RSSTYPE		__BITS(3,0)
 #define  RXDESC_TYPE_RSSTYPE_NONE		0
 #define  RXDESC_TYPE_RSSTYPE_IPV4		2
 #define  RXDESC_TYPE_RSSTYPE_IPV6		3
 #define  RXDESC_TYPE_RSSTYPE_IPV4_TCP		4
 #define  RXDESC_TYPE_RSSTYPE_IPV6_TCP		5
 #define  RXDESC_TYPE_RSSTYPE_IPV4_UDP		6
 #define  RXDESC_TYPE_RSSTYPE_IPV6_UDP		7
 #define RXDESC_TYPE_PKTTYPE_ETHER	__BITS(5,4)
 #define  RXDESC_TYPE_PKTTYPE_ETHER_IPV4		0
 #define  RXDESC_TYPE_PKTTYPE_ETHER_IPV6		1
 #define  RXDESC_TYPE_PKTTYPE_ETHER_OTHERS	2
 #define  RXDESC_TYPE_PKTTYPE_ETHER_ARP		3
 #define RXDESC_TYPE_PKTTYPE_PROTO	__BITS(8,6)
 #define  RXDESC_TYPE_PKTTYPE_PROTO_TCP		0
 #define  RXDESC_TYPE_PKTTYPE_PROTO_UDP		1
 #define  RXDESC_TYPE_PKTTYPE_PROTO_SCTP		2
 #define  RXDESC_TYPE_PKTTYPE_PROTO_ICMP		3
 #define  RXDESC_TYPE_PKTTYPE_PROTO_OTHERS	4
 #define RXDESC_TYPE_PKTTYPE_VLAN	__BIT(9)
 #define RXDESC_TYPE_PKTTYPE_VLAN_DOUBLE	__BIT(10)
 #define RXDESC_TYPE_MAC_DMA_ERR		__BIT(12)
 #define RXDESC_TYPE_RESERVED		__BITS(18,13)
 #define RXDESC_TYPE_IPV4_CSUM_CHECKED	__BIT(19)	/* PKTTYPE_ETHER_IPV4 */
 #define RXDESC_TYPE_TCPUDP_CSUM_CHECKED	__BIT(20)
 #define RXDESC_TYPE_SPH			__BIT(21)
 #define RXDESC_TYPE_HDR_LEN		__BITS(31,22)
 	uint32_t rss_hash;
 	uint16_t status;
 #define RXDESC_STATUS_DD		__BIT(0)
 #define RXDESC_STATUS_EOP		__BIT(1)
 #define RXDESC_STATUS_MACERR		__BIT(2)
 #define RXDESC_STATUS_IPV4_CSUM_NG	__BIT(3)
 #define RXDESC_STATUS_TCPUDP_CSUM_ERROR	__BIT(4)
 #define RXDESC_STATUS_TCPUDP_CSUM_OK	__BIT(5)
 #define RXDESC_STATUS_STAT		__BITS(2,5)
 #define RXDESC_STATUS_ESTAT		__BITS(6,11)
 #define RXDESC_STATUS_RSC_CNT		__BITS(12,15)
 	uint16_t pkt_len;
 	uint16_t next_desc_ptr;
 	uint16_t vlan;
 } __packed;
 typedef union aq_rx_desc {
 	struct aq_rx_desc_read read;
 	struct aq_rx_desc_wb wb;
 } __packed aq_rx_desc_t;
 typedef struct aq_tx_desc {
 	uint64_t buf_addr;
 	uint32_t ctl1;
 #define AQ_TXDESC_CTL1_TYPE_MASK	0x00000003
 #define AQ_TXDESC_CTL1_TYPE_TXD		0x00000001
 #define AQ_TXDESC_CTL1_TYPE_TXC		0x00000002
 #define AQ_TXDESC_CTL1_BLEN		__BITS(19,4)	/* TXD */
 #define AQ_TXDESC_CTL1_DD		__BIT(20)	/* TXD */
 #define AQ_TXDESC_CTL1_EOP		__BIT(21)	/* TXD */
 #define AQ_TXDESC_CTL1_CMD_VLAN		__BIT(22)	/* TXD */
 #define AQ_TXDESC_CTL1_CMD_FCS		__BIT(23)	/* TXD */
 #define AQ_TXDESC_CTL1_CMD_IP4CSUM	__BIT(24)	/* TXD */
 #define AQ_TXDESC_CTL1_CMD_L4CSUM	__BIT(25)	/* TXD */
 #define AQ_TXDESC_CTL1_CMD_LSO		__BIT(26)	/* TXD */
 #define AQ_TXDESC_CTL1_CMD_WB		__BIT(27)	/* TXD */
 #define AQ_TXDESC_CTL1_CMD_VXLAN	__BIT(28)	/* TXD */
 #define AQ_TXDESC_CTL1_VID		__BITS(15,4)	/* TXC */
 #define AQ_TXDESC_CTL1_LSO_IPV6		__BIT(21)	/* TXC */
 #define AQ_TXDESC_CTL1_LSO_TCP		__BIT(22)	/* TXC */
 	uint32_t ctl2;
 #define AQ_TXDESC_CTL2_LEN		__BITS(31,14)
 #define AQ_TXDESC_CTL2_CTX_EN		__BIT(13)
 #define AQ_TXDESC_CTL2_CTX_IDX		__BIT(12)
 } __packed aq_tx_desc_t;
 struct aq_txring {
 	struct aq_softc *txr_sc;
 	int txr_index;
 	kmutex_t txr_mutex;
 	bool txr_active;
 	pcq_t *txr_pcq;
 	void *txr_softint;
 	aq_tx_desc_t *txr_txdesc;	/* aq_tx_desc_t[AQ_TXD_NUM] */
 	bus_dmamap_t txr_txdesc_dmamap;
 	bus_dma_segment_t txr_txdesc_seg[1];
 	bus_size_t txr_txdesc_size;
 	struct {
 		struct mbuf *m;
 		bus_dmamap_t dmamap;
 	} txr_mbufs[AQ_TXD_NUM];
 	unsigned int txr_prodidx;
 	unsigned int txr_considx;
 	int txr_nfree;
 	/* counters */
 	uint64_t txr_opackets;
 	uint64_t txr_obytes;
 	uint64_t txr_omcasts;
 	uint64_t txr_oerrors;
 };
 struct aq_rxring {
 	struct aq_softc *rxr_sc;
 	int rxr_index;
 	kmutex_t rxr_mutex;
 	bool rxr_active;
 	aq_rx_desc_t *rxr_rxdesc;	/* aq_rx_desc_t[AQ_RXD_NUM] */
 	bus_dmamap_t rxr_rxdesc_dmamap;
 	bus_dma_segment_t rxr_rxdesc_seg[1];
 	bus_size_t rxr_rxdesc_size;
 	struct {
 		struct mbuf *m;
 		bus_dmamap_t dmamap;
 	} rxr_mbufs[AQ_RXD_NUM];
 	unsigned int rxr_readidx;
 	/* counters */
 	uint64_t rxr_ipackets;
 	uint64_t rxr_ibytes;
 	uint64_t rxr_ierrors;
 	uint64_t rxr_iqdrops;
 };
 struct aq_queue {
 	struct aq_softc *sc;
 	struct aq_txring txring;
 	struct aq_rxring rxring;
 };
 struct aq_softc;
 struct aq_firmware_ops {
 	int (*reset)(struct aq_softc *);
 	int (*set_mode)(struct aq_softc *, aq_hw_fw_mpi_state_t,
 	    aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
 	int (*get_mode)(struct aq_softc *, aq_hw_fw_mpi_state_t *,
 	    aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
 	int (*get_stats)(struct aq_softc *, aq_hw_stats_s_t *);
 #if NSYSMON_ENVSYS > 0
 	int (*get_temperature)(struct aq_softc *, uint32_t *);
 #endif
 };
 #ifdef AQ_EVENT_COUNTERS
 #define AQ_EVCNT_DECL(name)						\
 	char sc_evcount_##name##_name[32];				\
 	struct evcnt sc_evcount_##name##_ev;
 #define AQ_EVCNT_ATTACH(sc, name, desc, evtype)				\
 	do {								\
 		snprintf((sc)->sc_evcount_##name##_name,		\
 		    sizeof((sc)->sc_evcount_##name##_name),		\
 		    "%s", desc);					\
 		evcnt_attach_dynamic(&(sc)->sc_evcount_##name##_ev,	\
 		    (evtype), NULL, device_xname((sc)->sc_dev),		\
 		    (sc)->sc_evcount_##name##_name);			\
 	} while (/*CONSTCOND*/0)
 #define AQ_EVCNT_ATTACH_MISC(sc, name, desc)				\
 	AQ_EVCNT_ATTACH(sc, name, desc, EVCNT_TYPE_MISC)
 #define AQ_EVCNT_DETACH(sc, name)					\
 	evcnt_detach(&(sc)->sc_evcount_##name##_ev)
 #define AQ_EVCNT_ADD(sc, name, val)					\
 	((sc)->sc_evcount_##name##_ev.ev_count += (val))
 #endif /* AQ_EVENT_COUNTERS */
 #define AQ_LOCK(sc)		mutex_enter(&(sc)->sc_mutex);
 #define AQ_UNLOCK(sc)		mutex_exit(&(sc)->sc_mutex);
 /* lock for FW2X_MPI_{CONTROL,STATE]_REG read-modify-write */
 #define AQ_MPI_LOCK(sc)		mutex_enter(&(sc)->sc_mpi_mutex);
 #define AQ_MPI_UNLOCK(sc)	mutex_exit(&(sc)->sc_mpi_mutex);
 struct aq_softc {
 	device_t sc_dev;
 	bus_space_tag_t sc_iot;
 	bus_space_handle_t sc_ioh;
 	bus_size_t sc_iosize;
-	bus_dma_tag_t sc_dmat;;
+	bus_dma_tag_t sc_dmat;
 	void *sc_ihs[AQ_NINTR_MAX];
 	pci_intr_handle_t *sc_intrs;
 	int sc_tx_irq[AQ_RSSQUEUE_MAX];
 	int sc_rx_irq[AQ_RSSQUEUE_MAX];
 	int sc_linkstat_irq;
 	bool sc_use_txrx_independent_intr;
 	bool sc_poll_linkstat;
 	bool sc_detect_linkstat;
 #if NSYSMON_ENVSYS > 0
 	struct sysmon_envsys *sc_sme;
 	envsys_data_t sc_sensor_temp;
 #endif
 	callout_t sc_tick_ch;
 	int sc_nintrs;
 	bool sc_msix;
 	struct aq_queue sc_queue[AQ_RSSQUEUE_MAX];
 	int sc_nqueues;
 	pci_chipset_tag_t sc_pc;
 	pcitag_t sc_pcitag;
 	uint16_t sc_product;
 	uint16_t sc_revision;
 	kmutex_t sc_mutex;
 	kmutex_t sc_mpi_mutex;
-	struct aq_firmware_ops *sc_fw_ops;
+	const struct aq_firmware_ops *sc_fw_ops;
 	uint64_t sc_fw_caps;
 	enum aq_media_type sc_media_type;
 	aq_link_speed_t sc_available_rates;
 	aq_link_speed_t sc_link_rate;
 	aq_link_fc_t sc_link_fc;
 	aq_link_eee_t sc_link_eee;
 	uint32_t sc_fw_version;
 #define FW_VERSION_MAJOR(sc)	(((sc)->sc_fw_version >> 24) & 0xff)
 #define FW_VERSION_MINOR(sc)	(((sc)->sc_fw_version >> 16) & 0xff)
 #define FW_VERSION_BUILD(sc)	((sc)->sc_fw_version & 0xffff)
 	uint32_t sc_features;
 #define FEATURES_MIPS		0x00000001
 #define FEATURES_TPO2		0x00000002
 #define FEATURES_RPF2		0x00000004
 #define FEATURES_MPI_AQ		0x00000008
 #define FEATURES_REV_A0		0x10000000
 #define FEATURES_REV_A		(FEATURES_REV_A0)
 #define FEATURES_REV_B0		0x20000000
 #define FEATURES_REV_B1		0x40000000
 #define FEATURES_REV_B		(FEATURES_REV_B0|FEATURES_REV_B1)
 	uint32_t sc_mbox_addr;
 	bool sc_rbl_enabled;
 	bool sc_fast_start_enabled;
 	bool sc_flash_present;
 	bool sc_intr_moderation_enable;
 	bool sc_rss_enable;
 	struct ethercom sc_ethercom;
 	struct ether_addr sc_enaddr;
 	struct ifmedia sc_media;
 	int sc_ec_capenable;		/* last ec_capenable */
 	unsigned short sc_if_flags;	/* last if_flags */
 #ifdef AQ_EVENT_COUNTERS
 	aq_hw_stats_s_t sc_statistics[2];
 	int sc_statistics_idx;
 	bool sc_poll_statistics;
 	AQ_EVCNT_DECL(uprc);
 	AQ_EVCNT_DECL(mprc);
 	AQ_EVCNT_DECL(bprc);
 	AQ_EVCNT_DECL(erpt);
 	AQ_EVCNT_DECL(uptc);
 	AQ_EVCNT_DECL(mptc);
 	AQ_EVCNT_DECL(bptc);
 	AQ_EVCNT_DECL(erpr);
 	AQ_EVCNT_DECL(mbtc);
 	AQ_EVCNT_DECL(bbtc);
 	AQ_EVCNT_DECL(mbrc);
 	AQ_EVCNT_DECL(bbrc);
 	AQ_EVCNT_DECL(ubrc);
 	AQ_EVCNT_DECL(ubtc);
 	AQ_EVCNT_DECL(ptc);
 	AQ_EVCNT_DECL(prc);
 	AQ_EVCNT_DECL(dpc);
 	AQ_EVCNT_DECL(cprc);
 #endif
 };
 static int aq_match(device_t, cfdata_t, void *);
 static void aq_attach(device_t, device_t, void *);
 static int aq_detach(device_t, int);
 static int aq_setup_msix(struct aq_softc *, struct pci_attach_args *, int,
     bool, bool);
 static int aq_setup_legacy(struct aq_softc *, struct pci_attach_args *,
     pci_intr_type_t);
 static int aq_establish_msix_intr(struct aq_softc *, bool, bool);
 static int aq_ifmedia_change(struct ifnet * const);
 static void aq_ifmedia_status(struct ifnet * const, struct ifmediareq *);
 static int aq_vlan_cb(struct ethercom *ec, uint16_t vid, bool set);
 static int aq_ifflags_cb(struct ethercom *);
 static int aq_init(struct ifnet *);
 static void aq_send_common_locked(struct ifnet *, struct aq_softc *,
     struct aq_txring *, bool);
 static int aq_transmit(struct ifnet *, struct mbuf *);
 static void aq_deferred_transmit(void *);
 static void aq_start(struct ifnet *);
 static void aq_stop(struct ifnet *, int);
 static void aq_watchdog(struct ifnet *);
 static int aq_ioctl(struct ifnet *, unsigned long, void *);
 static int aq_txrx_rings_alloc(struct aq_softc *);
 static void aq_txrx_rings_free(struct aq_softc *);
 static int aq_tx_pcq_alloc(struct aq_softc *, struct aq_txring *);
 static void aq_tx_pcq_free(struct aq_softc *, struct aq_txring *);
 static void aq_initmedia(struct aq_softc *);
 static void aq_enable_intr(struct aq_softc *, bool, bool);
 #if NSYSMON_ENVSYS > 0
 static void aq_temp_refresh(struct sysmon_envsys *, envsys_data_t *);
 #endif
 static void aq_tick(void *);
 static int aq_legacy_intr(void *);
 static int aq_link_intr(void *);
 static int aq_txrx_intr(void *);
 static int aq_tx_intr(void *);
 static int aq_rx_intr(void *);
 static int aq_set_linkmode(struct aq_softc *, aq_link_speed_t, aq_link_fc_t,
     aq_link_eee_t);
 static int aq_get_linkmode(struct aq_softc *, aq_link_speed_t *, aq_link_fc_t *,
     aq_link_eee_t *);
 static int aq_fw_reset(struct aq_softc *);
 static int aq_fw_version_init(struct aq_softc *);
 static int aq_hw_init(struct aq_softc *);
 static int aq_hw_init_ucp(struct aq_softc *);
 static int aq_hw_reset(struct aq_softc *);
 static int aq_fw_downld_dwords(struct aq_softc *, uint32_t, uint32_t *,
     uint32_t);
 static int aq_get_mac_addr(struct aq_softc *);
 static int aq_init_rss(struct aq_softc *);
 static int aq_set_capability(struct aq_softc *);
 static int fw1x_reset(struct aq_softc *);
 static int fw1x_set_mode(struct aq_softc *, aq_hw_fw_mpi_state_t,
     aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
 static int fw1x_get_mode(struct aq_softc *, aq_hw_fw_mpi_state_t *,
     aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
 static int fw1x_get_stats(struct aq_softc *, aq_hw_stats_s_t *);
 static int fw2x_reset(struct aq_softc *);
 static int fw2x_set_mode(struct aq_softc *, aq_hw_fw_mpi_state_t,
     aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
 static int fw2x_get_mode(struct aq_softc *, aq_hw_fw_mpi_state_t *,
     aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
 static int fw2x_get_stats(struct aq_softc *, aq_hw_stats_s_t *);
 #if NSYSMON_ENVSYS > 0
 static int fw2x_get_temperature(struct aq_softc *, uint32_t *);
 #endif
-static struct aq_firmware_ops aq_fw1x_ops = {
+static const struct aq_firmware_ops aq_fw1x_ops = {
 	.reset = fw1x_reset,
 	.set_mode = fw1x_set_mode,
 	.get_mode = fw1x_get_mode,
 	.get_stats = fw1x_get_stats,
 #if NSYSMON_ENVSYS > 0
 	.get_temperature = NULL
 #endif
 };
-static struct aq_firmware_ops aq_fw2x_ops = {
+static const struct aq_firmware_ops aq_fw2x_ops = {
 	.reset = fw2x_reset,
 	.set_mode = fw2x_set_mode,
 	.get_mode = fw2x_get_mode,
 	.get_stats = fw2x_get_stats,
 #if NSYSMON_ENVSYS > 0
 	.get_temperature = fw2x_get_temperature
 #endif
 };
 CFATTACH_DECL3_NEW(aq, sizeof(struct aq_softc),
     aq_match, aq_attach, aq_detach, NULL, NULL, NULL, DVF_DETACH_SHUTDOWN);
 static const struct aq_product {
 	pci_vendor_id_t aq_vendor;
 	pci_product_id_t aq_product;
 	const char *aq_name;
 	enum aq_media_type aq_media_type;
 	aq_link_speed_t aq_available_rates;
 } aq_products[] = {
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC100,
 	  "Aquantia AQC100 10 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_FIBRE, AQ_LINK_ALL
 	},
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC107,
 	  "Aquantia AQC107 10 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
 	},
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC108,
 	  "Aquantia AQC108 5 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
 	},
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC109,
 	  "Aquantia AQC109 2.5 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
 	},
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC111,
 	  "Aquantia AQC111 5 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
 	},
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC112,
 	  "Aquantia AQC112 2.5 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
 	},
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC100S,
 	  "Aquantia AQC100S 10 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_FIBRE, AQ_LINK_ALL
 	},
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC107S,
 	  "Aquantia AQC107S 10 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
 	},
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC108S,
 	  "Aquantia AQC108S 5 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
 	},
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC109S,
 	  "Aquantia AQC109S 2.5 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
 	},
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC111S,
 	  "Aquantia AQC111S 5 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
 	},
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC112S,
 	  "Aquantia AQC112S 2.5 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
 	},
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D100,
 	  "Aquantia D100 10 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_FIBRE, AQ_LINK_ALL
 	},
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D107,
 	  "Aquantia D107 10 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
 	},
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D108,
 	  "Aquantia D108 5 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
 	},
 	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D109,
 	  "Aquantia D109 2.5 Gigabit Network Adapter",
 	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
+	}
 };
 static const struct aq_product *
 aq_lookup(const struct pci_attach_args *pa)
+{
 	unsigned int i;
 	for (i = 0; i < __arraycount(aq_products); i++) {
 		if (PCI_VENDOR(pa->pa_id)  == aq_products[i].aq_vendor &&
 		    PCI_PRODUCT(pa->pa_id) == aq_products[i].aq_product)
 			return &aq_products[i];
+	}
 	return NULL;
+}
 static int
 aq_match(device_t parent, cfdata_t cf, void *aux)
+{
 	struct pci_attach_args *pa = aux;
 	if (aq_lookup(pa) != NULL)
 		return 1;
 	return 0;
+}
 static void
 aq_attach(device_t parent, device_t self, void *aux)
+{
 	struct aq_softc *sc = device_private(self);
 	struct pci_attach_args *pa = aux;
 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
 	pci_chipset_tag_t pc;
 	pcitag_t tag;
 	pcireg_t command, memtype, bar;
 	const struct aq_product *aqp;
 	int error;
 	sc->sc_dev = self;
 	mutex_init(&sc->sc_mutex, MUTEX_DEFAULT, IPL_NET);
 	mutex_init(&sc->sc_mpi_mutex, MUTEX_DEFAULT, IPL_NET);
 	sc->sc_pc = pc = pa->pa_pc;
 	sc->sc_pcitag = tag = pa->pa_tag;
 	sc->sc_dmat = pci_dma64_available(pa) ? pa->pa_dmat64 : pa->pa_dmat;
 	command = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
 	command |= PCI_COMMAND_MASTER_ENABLE;
 	pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, command);
 	sc->sc_product = PCI_PRODUCT(pa->pa_id);
 	sc->sc_revision = PCI_REVISION(pa->pa_class);
 	aqp = aq_lookup(pa);
 	KASSERT(aqp != NULL);
 	pci_aprint_devinfo_fancy(pa, "Ethernet controller", aqp->aq_name, 1);
 	bar = pci_conf_read(pc, tag, PCI_BAR(0));
 	if ((PCI_MAPREG_MEM_ADDR(bar) == 0) ||
 	    (PCI_MAPREG_TYPE(bar) != PCI_MAPREG_TYPE_MEM)) {
 		aprint_error_dev(sc->sc_dev, "wrong BAR type\n");
 		return;
+	}
 	memtype = pci_mapreg_type(pc, tag, PCI_BAR(0));
 	if (pci_mapreg_map(pa, PCI_BAR(0), memtype, 0, &sc->sc_iot, &sc->sc_ioh,
 	    NULL, &sc->sc_iosize) != 0) {
 		aprint_error_dev(sc->sc_dev, "unable to map register\n");
 		return;
+	}
 	sc->sc_nqueues = MIN(ncpu, AQ_RSSQUEUE_MAX);
 	/* max queue num is 8, and must be 2^n */
 	if (ncpu >= 8)
 		sc->sc_nqueues = 8;
 	else if (ncpu >= 4)
 		sc->sc_nqueues = 4;
 	else if (ncpu >= 2)
 		sc->sc_nqueues = 2;
 	else
 		sc->sc_nqueues = 1;
 	int msixcount = pci_msix_count(pa->pa_pc, pa->pa_tag);
 #ifndef CONFIG_NO_TXRX_INDEPENDENT
 	if (msixcount >= (sc->sc_nqueues * 2 + 1)) {
 		/* TX intrs + RX intrs + LINKSTAT intrs */
 		sc->sc_use_txrx_independent_intr = true;
 		sc->sc_poll_linkstat = false;
 		sc->sc_msix = true;
 	} else if (msixcount >= (sc->sc_nqueues * 2)) {
 		/* TX intrs + RX intrs */
 		sc->sc_use_txrx_independent_intr = true;
 		sc->sc_poll_linkstat = true;
 		sc->sc_msix = true;
 	} else
 #endif
 	if (msixcount >= (sc->sc_nqueues + 1)) {
 		/* TX/RX intrs LINKSTAT intrs */
 		sc->sc_use_txrx_independent_intr = false;
 		sc->sc_poll_linkstat = false;
 		sc->sc_msix = true;
 	} else if (msixcount >= sc->sc_nqueues) {
 		/* TX/RX intrs */
 		sc->sc_use_txrx_independent_intr = false;
 		sc->sc_poll_linkstat = true;
 		sc->sc_msix = true;
 	} else {
 		/* giving up using MSI-X */
 		sc->sc_msix = false;
+	}
 	/* XXX: on FIBRE, linkstat interrupt does not occur on boot? */
 	if (aqp->aq_media_type == AQ_MEDIA_TYPE_FIBRE)
 		sc->sc_poll_linkstat = true;
 #ifdef AQ_FORCE_POLL_LINKSTAT
 	sc->sc_poll_linkstat = true;
 #endif
 	aprint_debug_dev(sc->sc_dev,
 	    "ncpu=%d, pci_msix_count=%d."
 	    " allocate %d interrupts for %d%s queues%s\n",
 	    ncpu, msixcount,
 	    (sc->sc_use_txrx_independent_intr ?
 	    (sc->sc_nqueues * 2) : sc->sc_nqueues) +
 	    (sc->sc_poll_linkstat ? 0 : 1),
 	    sc->sc_nqueues,
 	    sc->sc_use_txrx_independent_intr ? "*2" : "",
 	    sc->sc_poll_linkstat ? "" : ", and link status");
 	if (sc->sc_msix)
 		error = aq_setup_msix(sc, pa, sc->sc_nqueues,
 		    sc->sc_use_txrx_independent_intr, !sc->sc_poll_linkstat);
 	else
 		error = ENODEV;
 	if (error != 0) {
 		/* if MSI-X failed, fallback to MSI with single queue */
 		sc->sc_use_txrx_independent_intr = false;
 		sc->sc_poll_linkstat = false;
 		sc->sc_msix = false;
 		sc->sc_nqueues = 1;
 		error = aq_setup_legacy(sc, pa, PCI_INTR_TYPE_MSI);
+	}
 	if (error != 0) {
 		/* if MSI failed, fallback to INTx */
 		error = aq_setup_legacy(sc, pa, PCI_INTR_TYPE_INTX);
+	}
 	if (error != 0)
 		return;
 	callout_init(&sc->sc_tick_ch, 0);
 	callout_setfunc(&sc->sc_tick_ch, aq_tick, sc);
 	sc->sc_intr_moderation_enable = CONFIG_INTR_MODERATION_ENABLE;
 	if (sc->sc_msix && (sc->sc_nqueues > 1))
 		sc->sc_rss_enable = true;
 	else
 		sc->sc_rss_enable = false;
 	error = aq_txrx_rings_alloc(sc);
 	if (error != 0)
 		goto attach_failure;
 	error = aq_fw_reset(sc);
 	if (error != 0)
 		goto attach_failure;
 	error = aq_fw_version_init(sc);
 	if (error != 0)
 		goto attach_failure;
 	error = aq_hw_init_ucp(sc);
 	if (error < 0)
 		goto attach_failure;
 	KASSERT(sc->sc_mbox_addr != 0);
 	error = aq_hw_reset(sc);
 	if (error != 0)
 		goto attach_failure;
 	aq_get_mac_addr(sc);
 	aq_init_rss(sc);
 	error = aq_hw_init(sc);	/* initialize and interrupts */
 	if (error != 0)
 		goto attach_failure;
 	sc->sc_media_type = aqp->aq_media_type;
 	sc->sc_available_rates = aqp->aq_available_rates;
 	sc->sc_ethercom.ec_ifmedia = &sc->sc_media;
 	ifmedia_init(&sc->sc_media, IFM_IMASK,
 	    aq_ifmedia_change, aq_ifmedia_status);
 	aq_initmedia(sc);
 	strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
 	ifp->if_softc = sc;
 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 	ifp->if_baudrate = IF_Gbps(10);
 	ifp->if_init = aq_init;
 	ifp->if_ioctl = aq_ioctl;
 	if (sc->sc_msix && (sc->sc_nqueues > 1))
 		ifp->if_transmit = aq_transmit;
 	ifp->if_start = aq_start;
 	ifp->if_stop = aq_stop;
 	ifp->if_watchdog = aq_watchdog;
 	IFQ_SET_READY(&ifp->if_snd);
 	/* initialize capabilities */
 	sc->sc_ethercom.ec_capabilities = 0;
 	sc->sc_ethercom.ec_capenable = 0;
 #if notyet
 	/* TODO */
 	sc->sc_ethercom.ec_capabilities |= ETHERCAP_EEE;
 #endif
 	sc->sc_ethercom.ec_capabilities |=
 	    ETHERCAP_JUMBO_MTU |
 	    ETHERCAP_VLAN_MTU |
 	    ETHERCAP_VLAN_HWTAGGING |
 	    ETHERCAP_VLAN_HWFILTER;
 	sc->sc_ethercom.ec_capenable |=
 	    ETHERCAP_VLAN_HWTAGGING |
 	    ETHERCAP_VLAN_HWFILTER;
 	ifp->if_capabilities = 0;
 	ifp->if_capenable = 0;
 #ifdef CONFIG_LRO_SUPPORT
 	ifp->if_capabilities |= IFCAP_LRO;
 	ifp->if_capenable |= IFCAP_LRO;
 #endif
 #if notyet
 	/* TSO */
 	ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;
 #endif
 #if notyet
 	/*
 	 * XXX:
 	 *   Rx L4 CSUM doesn't work well for fragment packet.
 	 *   aq marks 'CHEDKED' and 'BAD' for them.
 	 *   we need to ignore (clear) hw-csum flags if the packet is fragmented
+	 *
 	 *   TODO: test with LRO enabled
 	 */
 	ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv6_Rx;
 	ifp->if_capabilities |= IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv6_Rx;
 #endif
 	/* TX hardware checksum offloadding */
 	ifp->if_capabilities |= IFCAP_CSUM_IPv4_Tx;
 	ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv6_Tx;
 	ifp->if_capabilities |= IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv6_Tx;
 	/* RX hardware checksum offloadding */
 	ifp->if_capabilities |= IFCAP_CSUM_IPv4_Rx;
 	if_attach(ifp);
 	if_deferred_start_init(ifp, NULL);
 	ether_ifattach(ifp, sc->sc_enaddr.ether_addr_octet);
 	ether_set_vlan_cb(&sc->sc_ethercom, aq_vlan_cb);
 	ether_set_ifflags_cb(&sc->sc_ethercom, aq_ifflags_cb);
 	aq_enable_intr(sc, true, false);	/* only intr about link */
 	/* update media */
 	aq_ifmedia_change(ifp);
 #if NSYSMON_ENVSYS > 0
 	/* temperature monitoring */
 	if (sc->sc_fw_ops != NULL && sc->sc_fw_ops->get_temperature != NULL &&
 	    (sc->sc_fw_caps & FW2X_CTRL_TEMPERATURE) != 0) {
 		sc->sc_sme = sysmon_envsys_create();
 		sc->sc_sme->sme_name = device_xname(self);
 		sc->sc_sme->sme_cookie = sc;
 		sc->sc_sme->sme_flags = 0;
 		sc->sc_sme->sme_refresh = aq_temp_refresh;
 		sc->sc_sensor_temp.units = ENVSYS_STEMP;
 		sc->sc_sensor_temp.state = ENVSYS_SINVALID;
 		snprintf(sc->sc_sensor_temp.desc, ENVSYS_DESCLEN, "PHY");
 		sysmon_envsys_sensor_attach(sc->sc_sme, &sc->sc_sensor_temp);
 		sysmon_envsys_register(sc->sc_sme);
 		/*
 		 * for unknown reasons, the first call of fw2x_get_temperature()
 		 * will always fail (firmware matter?), so run once now.
 		 */
 		aq_temp_refresh(sc->sc_sme, &sc->sc_sensor_temp);
+	}
 #endif
 #ifdef AQ_EVENT_COUNTERS
 	/* get starting statistics values */
 	if (sc->sc_fw_ops != NULL && sc->sc_fw_ops->get_stats != NULL &&
 	    sc->sc_fw_ops->get_stats(sc, &sc->sc_statistics[0]) == 0) {
 		sc->sc_poll_statistics = true;
+	}
 	AQ_EVCNT_ATTACH_MISC(sc, uprc, "RX unicast packet");
 	AQ_EVCNT_ATTACH_MISC(sc, bprc, "RX broadcast packet");
 	AQ_EVCNT_ATTACH_MISC(sc, mprc, "RX multicast packet");
 	AQ_EVCNT_ATTACH_MISC(sc, erpr, "RX error packet");
 	AQ_EVCNT_ATTACH_MISC(sc, ubrc, "RX unicast bytes");
 	AQ_EVCNT_ATTACH_MISC(sc, bbrc, "RX broadcast bytes");
 	AQ_EVCNT_ATTACH_MISC(sc, mbrc, "RX multicast bytes");
 	AQ_EVCNT_ATTACH_MISC(sc, prc, "RX good packet");
 	AQ_EVCNT_ATTACH_MISC(sc, uptc, "TX unicast packet");
 	AQ_EVCNT_ATTACH_MISC(sc, bptc, "TX broadcast packet");
 	AQ_EVCNT_ATTACH_MISC(sc, mptc, "TX multicast packet");
 	AQ_EVCNT_ATTACH_MISC(sc, erpt, "TX error packet");
 	AQ_EVCNT_ATTACH_MISC(sc, ubtc, "TX unicast bytes");
 	AQ_EVCNT_ATTACH_MISC(sc, bbtc, "TX broadcast bytes");
 	AQ_EVCNT_ATTACH_MISC(sc, mbtc, "TX multicast bytes");
 	AQ_EVCNT_ATTACH_MISC(sc, ptc, "TX good packet");
 	AQ_EVCNT_ATTACH_MISC(sc, dpc, "DMA drop packet");
 	AQ_EVCNT_ATTACH_MISC(sc, cprc, "RX coalesced packet");
 #endif
 	return;
  attach_failure:
 	aq_detach(self, 0);
+}
 static int
 aq_detach(device_t self, int flags __unused)
+{
 	struct aq_softc *sc = device_private(self);
 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
 	int i, s;
 	if (sc->sc_iosize != 0) {
 		if (ifp->if_softc != NULL) {
 			s = splnet();
 			aq_stop(ifp, 0);
 			splx(s);
+		}
 		for (i = 0; i < AQ_NINTR_MAX; i++) {
 			if (sc->sc_ihs[i] != NULL) {
 				pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]);
 				sc->sc_ihs[i] = NULL;
+			}
+		}
 		if (sc->sc_nintrs > 0) {
 			pci_intr_release(sc->sc_pc, sc->sc_intrs,
 			    sc->sc_nintrs);
 			sc->sc_intrs = NULL;
 			sc->sc_nintrs = 0;
+		}
 		aq_txrx_rings_free(sc);
 		if (ifp->if_softc != NULL) {
 			ether_ifdetach(ifp);
 			if_detach(ifp);
+		}
 		aprint_debug_dev(sc->sc_dev, "%s: bus_space_unmap\n", __func__);
 		bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize);
 		sc->sc_iosize = 0;
+	}
 	callout_stop(&sc->sc_tick_ch);
 #if NSYSMON_ENVSYS > 0
 	if (sc->sc_sme != NULL) {
 		/* all sensors associated with this will also be detached */
 		sysmon_envsys_unregister(sc->sc_sme);
 		sc->sc_sme = NULL;
+	}
 #endif
 #ifdef AQ_EVENT_COUNTERS
 	AQ_EVCNT_DETACH(sc, uprc);
 	AQ_EVCNT_DETACH(sc, mprc);
 	AQ_EVCNT_DETACH(sc, bprc);
 	AQ_EVCNT_DETACH(sc, erpt);
 	AQ_EVCNT_DETACH(sc, uptc);
 	AQ_EVCNT_DETACH(sc, mptc);
 	AQ_EVCNT_DETACH(sc, bptc);
 	AQ_EVCNT_DETACH(sc, erpr);
 	AQ_EVCNT_DETACH(sc, mbtc);
 	AQ_EVCNT_DETACH(sc, bbtc);
 	AQ_EVCNT_DETACH(sc, mbrc);
 	AQ_EVCNT_DETACH(sc, bbrc);
 	AQ_EVCNT_DETACH(sc, ubrc);
 	AQ_EVCNT_DETACH(sc, ubtc);
 	AQ_EVCNT_DETACH(sc, ptc);
 	AQ_EVCNT_DETACH(sc, prc);
 	AQ_EVCNT_DETACH(sc, dpc);
 	AQ_EVCNT_DETACH(sc, cprc);
 #endif
 	mutex_destroy(&sc->sc_mpi_mutex);
 	mutex_destroy(&sc->sc_mutex);
 	return 0;
+}
 static int
 aq_establish_intr(struct aq_softc *sc, int intno, kcpuset_t *affinity,
     int (*func)(void *), void *arg, const char *xname)
+{
 	char intrbuf[PCI_INTRSTR_LEN];
 	pci_chipset_tag_t pc = sc->sc_pc;
 	void *vih;
 	const char *intrstr = NULL;
 	intrstr = pci_intr_string(pc, sc->sc_intrs[intno], intrbuf,
 	    sizeof(intrbuf));
 	pci_intr_setattr(pc, &sc->sc_intrs[intno], PCI_INTR_MPSAFE, true);
 	vih = pci_intr_establish_xname(pc, sc->sc_intrs[intno],
 	    IPL_NET, func, arg, xname);
 	if (vih == NULL) {
 		aprint_error_dev(sc->sc_dev,
 		    "unable to establish MSI-X%s%s for %s\n",
 		    intrstr ? " at " : "",
 		    intrstr ? intrstr : "", xname);
 		return EIO;
+	}
 	sc->sc_ihs[intno] = vih;
 	if (affinity != NULL) {
 		/* Round-robin affinity */
 		kcpuset_zero(affinity);
 		kcpuset_set(affinity, intno % ncpu);
 		interrupt_distribute(vih, affinity, NULL);
+	}
 	return 0;
+}
 static int
 aq_establish_msix_intr(struct aq_softc *sc, bool txrx_independent,
     bool linkintr)
+{
 	kcpuset_t *affinity;
 	int error, intno, i;
 	char intr_xname[INTRDEVNAMEBUF];
 	kcpuset_create(&affinity, false);
 	intno = 0;
 	if (txrx_independent) {
 		for (i = 0; i < sc->sc_nqueues; i++) {
 			snprintf(intr_xname, sizeof(intr_xname), "%s RX%d",
 			    device_xname(sc->sc_dev), i);
 			sc->sc_rx_irq[i] = intno;
 			error = aq_establish_intr(sc, intno++, affinity,
 			   aq_rx_intr, &sc->sc_queue[i].rxring, intr_xname);
 			if (error != 0)
 				goto fail;
+		}
 		for (i = 0; i < sc->sc_nqueues; i++) {
 			snprintf(intr_xname, sizeof(intr_xname), "%s TX%d",
 			    device_xname(sc->sc_dev), i);
 			sc->sc_tx_irq[i] = intno;
 			error = aq_establish_intr(sc, intno++, affinity,
 			    aq_tx_intr, &sc->sc_queue[i].txring, intr_xname);
 			if (error != 0)
 				goto fail;
+		}
 	} else {
 		for (i = 0; i < sc->sc_nqueues; i++) {
 			snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d",
 			    device_xname(sc->sc_dev), i);
 			sc->sc_rx_irq[i] = intno;
 			sc->sc_tx_irq[i] = intno;
 			error = aq_establish_intr(sc, intno++, affinity,
 			    aq_txrx_intr, &sc->sc_queue[i], intr_xname);
 			if (error != 0)
 				goto fail;
+		}
+	}
 	if (linkintr) {
 		snprintf(intr_xname, sizeof(intr_xname), "%s LINK",
 		    device_xname(sc->sc_dev));
 		sc->sc_linkstat_irq = intno;
 		error = aq_establish_intr(sc, intno++, affinity,
 		    aq_link_intr, sc, intr_xname);
 		if (error != 0)
 			goto fail;
+	}
 	kcpuset_destroy(affinity);
 	return 0;
  fail:
 	for (i = 0; i < AQ_NINTR_MAX; i++) {
 		if (sc->sc_ihs[i] != NULL) {
 			pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]);
 			sc->sc_ihs[i] = NULL;
+		}
+	}
 	kcpuset_destroy(affinity);
 	return ENOMEM;
+}
 static int
 aq_setup_msix(struct aq_softc *sc, struct pci_attach_args *pa, int nqueue,
     bool txrx_independent, bool linkintr)
+{
 	int error, nintr;
 	if (txrx_independent)
 		nintr = nqueue * 2;
 	else
 		nintr = nqueue;
 	if (linkintr)
 		nintr++;
 	error = pci_msix_alloc_exact(pa, &sc->sc_intrs, nintr);
 	if (error != 0) {
 		aprint_error_dev(sc->sc_dev,
 		    "failed to allocate MSI-X interrupts\n");
 		goto fail;
+	}
 	error = aq_establish_msix_intr(sc, txrx_independent, linkintr);
 	if (error == 0) {
 		sc->sc_nintrs = nintr;
 	} else {
 		pci_intr_release(sc->sc_pc, sc->sc_intrs, nintr);
 		sc->sc_nintrs = 0;
+	}
  fail:
 	return error;
+}
 static int
 aq_setup_legacy(struct aq_softc *sc, struct pci_attach_args *pa,
     pci_intr_type_t inttype)
+{
 	int counts[PCI_INTR_TYPE_SIZE];
 	int error, nintr;
 	nintr = 1;
 	memset(counts, 0, sizeof(counts));
 	counts[inttype] = nintr;
 	error = pci_intr_alloc(pa, &sc->sc_intrs, counts, inttype);
 	if (error != 0) {
 		aprint_error_dev(sc->sc_dev,
 		    "failed to allocate%s interrupts\n",
 		    (inttype == PCI_INTR_TYPE_MSI) ? " MSI" : "");
 		return error;
+	}
 	error = aq_establish_intr(sc, 0, NULL, aq_legacy_intr, sc,
 	    device_xname(sc->sc_dev));
 	if (error == 0) {
 		sc->sc_nintrs = nintr;
 	} else {
 		pci_intr_release(sc->sc_pc, sc->sc_intrs, nintr);
 		sc->sc_nintrs = 0;
+	}
 	return error;
+}
 static void
 global_software_reset(struct aq_softc *sc)
+{
 	uint32_t v;
 	AQ_WRITE_REG_BIT(sc, RX_SYSCONTROL_REG, RX_SYSCONTROL_RESET_DIS, 0);
 	AQ_WRITE_REG_BIT(sc, TX_SYSCONTROL_REG, TX_SYSCONTROL_RESET_DIS, 0);
 	AQ_WRITE_REG_BIT(sc, FW_MPI_RESETCTRL_REG,
 	    FW_MPI_RESETCTRL_RESET_DIS, 0);
 	v = AQ_READ_REG(sc, AQ_FW_SOFTRESET_REG);
 	v &= ~AQ_FW_SOFTRESET_DIS;
 	v |= AQ_FW_SOFTRESET_RESET;
 	AQ_WRITE_REG(sc, AQ_FW_SOFTRESET_REG, v);
+}
 static int
 mac_soft_reset_rbl(struct aq_softc *sc, aq_fw_bootloader_mode_t *mode)
+{
 	int timo;
 	aprint_debug_dev(sc->sc_dev, "RBL> MAC reset STARTED!\n");
 	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e1);
 	AQ_WRITE_REG(sc, AQ_FW_GLB_CPU_SEM_REG(0), 1);
 	AQ_WRITE_REG(sc, AQ_MBOXIF_POWER_GATING_CONTROL_REG, 0);
 	/* MAC FW will reload PHY FW if 1E.1000.3 was cleaned - #undone */
 	AQ_WRITE_REG(sc, FW_BOOT_EXIT_CODE_REG, RBL_STATUS_DEAD);
 	global_software_reset(sc);
 	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e0);
 	/* Wait for RBL to finish boot process. */
 #define RBL_TIMEOUT_MS	10000
 	uint16_t rbl_status;
 	for (timo = RBL_TIMEOUT_MS; timo > 0; timo--) {
 		rbl_status = AQ_READ_REG(sc, FW_BOOT_EXIT_CODE_REG) & 0xffff;
 		if (rbl_status != 0 && rbl_status != RBL_STATUS_DEAD)
 			break;
 		msec_delay(1);
+	}
 	if (timo <= 0) {
 		aprint_error_dev(sc->sc_dev,
 		    "RBL> RBL restart failed: timeout\n");
 		return EBUSY;
+	}
 	switch (rbl_status) {
 	case RBL_STATUS_SUCCESS:
 		if (mode != NULL)
 			*mode = FW_BOOT_MODE_RBL_FLASH;
 		aprint_debug_dev(sc->sc_dev, "RBL> reset complete! [Flash]\n");
 		break;
 	case RBL_STATUS_HOST_BOOT:
 		if (mode != NULL)
 			*mode = FW_BOOT_MODE_RBL_HOST_BOOTLOAD;
 		aprint_debug_dev(sc->sc_dev,
 		    "RBL> reset complete! [Host Bootload]\n");
 		break;
 	case RBL_STATUS_FAILURE:
 	default:
 		aprint_error_dev(sc->sc_dev,
 		    "unknown RBL status 0x%x\n", rbl_status);
 		return EBUSY;
+	}
 	return 0;
+}
 static int
 mac_soft_reset_flb(struct aq_softc *sc)
+{
 	uint32_t v;
 	int timo;
 	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e1);
 	/*
 	 * Let Felicity hardware to complete SMBUS transaction before
 	 * Global software reset.
 	 */
 	msec_delay(50);
 	/*
 	 * If SPI burst transaction was interrupted(before running the script),
 	 * global software reset may not clear SPI interface.
 	 * Clean it up manually before global reset.
 	 */
 	AQ_WRITE_REG(sc, AQ_GLB_NVR_PROVISIONING2_REG, 0x00a0);
 	AQ_WRITE_REG(sc, AQ_GLB_NVR_INTERFACE1_REG, 0x009f);
 	AQ_WRITE_REG(sc, AQ_GLB_NVR_INTERFACE1_REG, 0x809f);
 	msec_delay(50);
 	v = AQ_READ_REG(sc, AQ_FW_SOFTRESET_REG);
 	v &= ~AQ_FW_SOFTRESET_DIS;
 	v |= AQ_FW_SOFTRESET_RESET;
 	AQ_WRITE_REG(sc, AQ_FW_SOFTRESET_REG, v);
 	/* Kickstart. */
 	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x80e0);
 	AQ_WRITE_REG(sc, AQ_MBOXIF_POWER_GATING_CONTROL_REG, 0);
 	if (!sc->sc_fast_start_enabled)
 		AQ_WRITE_REG(sc, AQ_GLB_GENERAL_PROVISIONING9_REG, 1);
 	/*
 	 * For the case SPI burst transaction was interrupted (by MCP reset
 	 * above), wait until it is completed by hardware.
 	 */
 	msec_delay(50);
 	/* MAC Kickstart */
 	if (!sc->sc_fast_start_enabled) {
 		AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x180e0);
 		uint32_t flb_status;
 		for (timo = 0; timo < 1000; timo++) {
 			flb_status = AQ_READ_REG(sc,
 			    FW_MPI_DAISY_CHAIN_STATUS_REG) & 0x10;
 			if (flb_status != 0)
 				break;
 			msec_delay(1);
+		}
 		if (flb_status == 0) {
 			aprint_error_dev(sc->sc_dev,
 			    "FLB> MAC kickstart failed: timed out\n");
 			return ETIMEDOUT;
+		}
 		aprint_debug_dev(sc->sc_dev,
 		    "FLB> MAC kickstart done, %d ms\n", timo);
 		/* FW reset */
 		AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x80e0);
 		/*
 		 * Let Felicity hardware complete SMBUS transaction before
 		 * Global software reset.
 		 */
 		msec_delay(50);
 		sc->sc_fast_start_enabled = true;
+	}
 	AQ_WRITE_REG(sc, AQ_FW_GLB_CPU_SEM_REG(0), 1);
 	/* PHY Kickstart: #undone */
 	global_software_reset(sc);
 	for (timo = 0; timo < 1000; timo++) {
 		if (AQ_READ_REG(sc, AQ_FW_VERSION_REG) != 0)
 			break;
 		msec_delay(10);
+	}
 	if (timo >= 1000) {
 		aprint_error_dev(sc->sc_dev, "FLB> Global Soft Reset failed\n");
 		return ETIMEDOUT;
+	}
 	aprint_debug_dev(sc->sc_dev, "FLB> F/W restart: %d ms\n", timo * 10);
 	return 0;
+}
 static int
 mac_soft_reset(struct aq_softc *sc, aq_fw_bootloader_mode_t *mode)
+{
 	if (sc->sc_rbl_enabled)
 		return mac_soft_reset_rbl(sc, mode);
 	if (mode != NULL)
 		*mode = FW_BOOT_MODE_FLB;
 	return mac_soft_reset_flb(sc);
+}
 static int
 aq_fw_read_version(struct aq_softc *sc)
+{
 	int i, error = EBUSY;
 #define MAC_FW_START_TIMEOUT_MS	10000
 	for (i = 0; i < MAC_FW_START_TIMEOUT_MS; i++) {
 		sc->sc_fw_version = AQ_READ_REG(sc, AQ_FW_VERSION_REG);
 		if (sc->sc_fw_version != 0) {
 			error = 0;
 			break;
+		}
 		delay(1000);
+	}
 	return error;
+}
 static int
 aq_fw_reset(struct aq_softc *sc)
+{
 	uint32_t ver, v, bootExitCode;
 	int i, error;
 	ver = AQ_READ_REG(sc, AQ_FW_VERSION_REG);
 	for (i = 1000; i > 0; i--) {
 		v = AQ_READ_REG(sc, FW_MPI_DAISY_CHAIN_STATUS_REG);
 		bootExitCode = AQ_READ_REG(sc, FW_BOOT_EXIT_CODE_REG);
 		if (v != 0x06000000 || bootExitCode != 0)
 			break;
+	}
 	if (i <= 0) {
 		aprint_error_dev(sc->sc_dev,
 		    "F/W reset failed. Neither RBL nor FLB started\n");
 		return ETIMEDOUT;
+	}
 	sc->sc_rbl_enabled = (bootExitCode != 0);
 	/*
 	 * Having FW version 0 is an indicator that cold start
 	 * is in progress. This means two things:
 	 * 1) Driver have to wait for FW/HW to finish boot (500ms giveup)
 	 * 2) Driver may skip reset sequence and save time.
 	 */
 	if (sc->sc_fast_start_enabled && (ver != 0)) {
 		error = aq_fw_read_version(sc);
 		/* Skip reset as it just completed */
 		if (error == 0)
 			return 0;
+	}
 	aq_fw_bootloader_mode_t mode = FW_BOOT_MODE_UNKNOWN;
 	error = mac_soft_reset(sc, &mode);
 	if (error != 0) {
 		aprint_error_dev(sc->sc_dev, "MAC reset failed: %d\n", error);
 		return error;
+	}
 	switch (mode) {
 	case FW_BOOT_MODE_FLB:
 		aprint_debug_dev(sc->sc_dev,
 		    "FLB> F/W successfully loaded from flash.\n");
 		sc->sc_flash_present = true;
 		return aq_fw_read_version(sc);
 	case FW_BOOT_MODE_RBL_FLASH:
 		aprint_debug_dev(sc->sc_dev,
 		    "RBL> F/W loaded from flash. Host Bootload disabled.\n");
 		sc->sc_flash_present = true;
 		return aq_fw_read_version(sc);
 	case FW_BOOT_MODE_UNKNOWN:
 		aprint_error_dev(sc->sc_dev,
 		    "F/W bootload error: unknown bootloader type\n");
 		return ENOTSUP;
 	case FW_BOOT_MODE_RBL_HOST_BOOTLOAD:
 		aprint_debug_dev(sc->sc_dev, "RBL> Host Bootload mode\n");
 		break;
+	}
 	/*
 	 * XXX: TODO: add support Host Boot
 	 */
 	aprint_error_dev(sc->sc_dev,
 	    "RBL> F/W Host Bootload not implemented\n");
 	return ENOTSUP;
+}
 static int
 aq_hw_reset(struct aq_softc *sc)
+{
 	int error;
 	/* disable irq */
 	AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_DIS, 0);
 	/* apply */
 	AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_IRQ, 1);
 	/* wait ack 10 times by 1ms */
 	WAIT_FOR(
 	    (AQ_READ_REG(sc, AQ_INTR_CTRL_REG) & AQ_INTR_CTRL_RESET_IRQ) == 0,
 , 10, &error);
 	if (error != 0) {
 		aprint_error_dev(sc->sc_dev,
 		    "atlantic: IRQ reset failed: %d\n", error);
 		return error;
+	}
 	return sc->sc_fw_ops->reset(sc);
+}
 static int
 aq_hw_init_ucp(struct aq_softc *sc)
+{
 	int timo;
 	if (FW_VERSION_MAJOR(sc) == 1) {
 		if (AQ_READ_REG(sc, FW1X_MPI_INIT2_REG) == 0) {
 			uint32_t data;
 			cprng_fast(&data, sizeof(data));
 			data &= 0xfefefefe;
 			data |= 0x02020202;
 			AQ_WRITE_REG(sc, FW1X_MPI_INIT2_REG, data);
+		}
 		AQ_WRITE_REG(sc, FW1X_MPI_INIT1_REG, 0);
+	}
 	for (timo = 100; timo > 0; timo--) {
 		sc->sc_mbox_addr = AQ_READ_REG(sc, FW_MPI_MBOX_ADDR_REG);
 		if (sc->sc_mbox_addr != 0)
 			break;
 		delay(1000);
+	}
 #define AQ_FW_MIN_VERSION	0x01050006
 #define AQ_FW_MIN_VERSION_STR	"1.5.6"
 	if (sc->sc_fw_version < AQ_FW_MIN_VERSION) {
 		aprint_error_dev(sc->sc_dev,
 		    "atlantic: wrong FW version: " AQ_FW_MIN_VERSION_STR
 		    " or later required, this is %d.%d.%d\n",
 		    FW_VERSION_MAJOR(sc),
 		    FW_VERSION_MINOR(sc),
 		    FW_VERSION_BUILD(sc));
 		return ENOTSUP;
+	}
 	return 0;
+}
 static int
 aq_fw_version_init(struct aq_softc *sc)
+{
 	int error = 0;
 	char fw_vers[sizeof("F/W version xxxxx.xxxxx.xxxxx")];
 	if (FW_VERSION_MAJOR(sc) == 1) {
 		sc->sc_fw_ops = &aq_fw1x_ops;
 	} else if ((FW_VERSION_MAJOR(sc) == 2) || (FW_VERSION_MAJOR(sc) == 3)) {
 		sc->sc_fw_ops = &aq_fw2x_ops;
 	} else {
 		aprint_error_dev(sc->sc_dev,
 		    "Unsupported F/W version %d.%d.%d\n",
 		    FW_VERSION_MAJOR(sc), FW_VERSION_MINOR(sc),
 		    FW_VERSION_BUILD(sc));
 		return ENOTSUP;
+	}
 	snprintf(fw_vers, sizeof(fw_vers), "F/W version %d.%d.%d",
 	    FW_VERSION_MAJOR(sc), FW_VERSION_MINOR(sc), FW_VERSION_BUILD(sc));
 	/* detect revision */
 	uint32_t hwrev = AQ_READ_REG(sc, AQ_HW_REVISION_REG);
 	switch (hwrev & 0x0000000f) {
 	case 0x01:
 		aprint_normal_dev(sc->sc_dev, "Atlantic revision A0, %s\n",
 		    fw_vers);
 		sc->sc_features |= FEATURES_REV_A0 |
 		    FEATURES_MPI_AQ | FEATURES_MIPS;
 		break;
 	case 0x02:
 		aprint_normal_dev(sc->sc_dev, "Atlantic revision B0, %s\n",
 		    fw_vers);