Pull up following revision(s) (requested by jmcneill in ticket #980):

	sys/dev/pci/if_aq.c: revision 1.4
	sys/dev/pci/if_aq.c: revision 1.5
	sys/arch/amd64/conf/GENERIC: revision 1.553
	sys/dev/pci/files.pci: revision 1.419
	sys/arch/amd64/conf/XEN3_DOM0: revision 1.170
	sys/dev/pci/if_aq.c: revision 1.9
	share/man/man4/Makefile: revision 1.693
	sys/dev/pci/pcidevs: revision 1.1411
	share/man/man4/aq.4: revision 1.1
	share/man/man4/aq.4: revision 1.3
	sys/arch/i386/conf/ALL: revision 1.479
	share/man/man4/aq.4: revision 1.4
	sys/dev/pci/if_aq.c: revision 1.10
	sys/dev/pci/files.pci: revision 1.421
	sys/dev/pci/if_aq.c: revision 1.11
	sys/dev/pci/if_aq.c: revision 1.12
	sys/dev/pci/if_aq.c: revision 1.13
	sys/dev/pci/if_aq.c: revision 1.14
	sys/dev/pci/if_aq.c: revision 1.15
	sys/dev/pci/if_aq.c: revision 1.16
	sys/dev/pci/pcidevs: revision 1.1408
	sys/arch/amd64/conf/ALL: revision 1.135
	sys/net/ethertypes.h: revision 1.19
	sys/arch/i386/conf/GENERIC: revision 1.1218
	distrib/sets/lists/man/mi: revision 1.1668
	sys/dev/pci/if_aq.c: revision 1.1
	sys/dev/pci/if_aq.c: revision 1.2
	sys/dev/pci/pcidevs: revision 1.1395
	sys/dev/pci/if_aq.c: revision 1.3
	sys/arch/evbarm/conf/GENERIC64: revision 1.125

Add the ETHERTYPE_QINQ for 802.1ad VLAN stacking

add Aquantia AQC 10G network adapters
add support Aquantia AQC seriese 10G network adapters.

this driver is based on the FreeBSD version https://github.com/Aquantia/aqtion-freebsd ,
but drastically rewritten for NetBSD.

add aq(4)

 Add Aquantia AQC100, AQC100S and D100.

add support VLAN HW filter

set/clear IFF_OACTIVE flag only on txring 0

make counters per queue

support internal PHY temperature sensor

Found by kUBSan:
- Use unsigned to avoid undefined behavior in aq_hw_init().
- Cast to unsigned to avoid undefined behavior in aq_set_mac_addr().

fix descriptions of register map in comment

return the ifmedia active status correctly even while the link is not up after attach.
pointed out by msaitoh@. thanks.

On FIBRE devices, there are times when linkstat interrupt doesn't occur?
reported from Andrius V. thanks.
- use polling instead of linkstat interrupt when FIBRE
- add AQ_FORCE_POLL_LINKSTAT options (not by default)

sort product table, and tabify

add support AQC100S and D100.
not tested, but they are probably the same as the AQC100.


(martin)

.\"	$NetBSD: aq.4,v 1.4.2.2 2020/07/07 10:29:06 martin Exp $
.\"
.\" Copyright (c) 2020 Ryo Shimizu <ryo@nerv.org>
.\" All rights reserved.
.\"
.\" Redistribution and use in source and binary forms, with or without
.\" modification, are permitted provided that the following conditions
.\" are met:
.\" 1. Redistributions of source code must retain the above copyright
.\"    notice, this list of conditions and the following disclaimer.
.\" 2. Redistributions in binary form must reproduce the above copyright
.\"    notice, this list of conditions and the following disclaimer in the
.\"    documentation and/or other materials provided with the distribution.
.\"
.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
.\" DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
.\" INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
.\" (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
.\" SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
.\" POSSIBILITY OF SUCH DAMAGE.
.\"
.Dd April 24, 2020
.Dt AQ 4
.Os
.Sh NAME
.Nm aq
.Nd Aquantia AQC multigigabit Network driver
.Sh SYNOPSIS
.Cd "aq* at pci? dev ? function ?"
.Sh DESCRIPTION
The
.Nm
driver supports Aquantia AQC series controllers.
Supported controllers include:
.Pp
.Bl -bullet -compact
.It
AQC100 10 Gigabit Network Adapter
.It
AQC107 10 Gigabit Network Adapter
.It
AQC108 5 Gigabit Network Adapter
.It
AQC109 2.5 Gigabit Network Adapter
.It
AQC111 5 Gigabit Network Adapter
.It
AQC112 2.5 Gigabit Network Adapter
.It
AQC100S 10 Gigabit Network Adapter
.It
AQC107S 10 Gigabit Network Adapter
.It
AQC108S 5 Gigabit Network Adapter
.It
AQC109S 2.5 Gigabit Network Adapter
.It
AQC111S 5 Gigabit Network Adapter
.It
AQC112S 2.5 Gigabit Network Adapter
.It
D100 10 Gigabit Network Adapter
.It
D107 10 Gigabit Network Adapter
.It
D108 5 Gigabit Network Adapter
.It
D109 2.5 Gigabit Network Adapter
.El
.Sh SEE ALSO
.Xr arp 4 ,
.Xr ifmedia 4 ,
.Xr netintro 4 ,
.Xr pci 4 ,
.Xr vlan 4 ,
.Xr ifconfig 8
.Sh HISTORY
The
.Nm
driver first appeared in
.Nx 10.0 ,
and is based on the
.Fx
driver of the same name, but has been drastically rewritten by Ryo Shimizu.

/*	$NetBSD: if_aq.c,v 1.17.2.2 2020/07/07 10:29:05 martin Exp $	*/

/**
 * aQuantia Corporation Network Driver
 * Copyright (C) 2014-2017 aQuantia Corporation. All rights reserved
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *   (1) Redistributions of source code must retain the above
 *   copyright notice, this list of conditions and the following
 *   disclaimer.
 *
 *   (2) Redistributions in binary form must reproduce the above
 *   copyright notice, this list of conditions and the following
 *   disclaimer in the documentation and/or other materials provided
 *   with the distribution.
 *
 *   (3) The name of the author may not be used to endorse or promote
 *   products derived from this software without specific prior
 *   written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 */

/*-
 * Copyright (c) 2020 Ryo Shimizu <ryo@nerv.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: if_aq.c,v 1.17.2.2 2020/07/07 10:29:05 martin Exp $");

#ifdef _KERNEL_OPT
#include "opt_if_aq.h"
#include "sysmon_envsys.h"
#endif

#include <sys/param.h>
#include <sys/types.h>
#include <sys/bitops.h>
#include <sys/cprng.h>
#include <sys/cpu.h>
#include <sys/interrupt.h>
#include <sys/module.h>
#include <sys/pcq.h>

#include <net/bpf.h>
#include <net/if.h>
#include <net/if_dl.h>
#include <net/if_media.h>
#include <net/if_ether.h>
#include <net/rss_config.h>

#include <dev/pci/pcivar.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcidevs.h>
#include <dev/sysmon/sysmonvar.h>

/* driver configuration */
#define CONFIG_INTR_MODERATION_ENABLE	true	/* delayed interrupt */
#undef CONFIG_LRO_SUPPORT			/* no LRO not suppoted */
#undef CONFIG_NO_TXRX_INDEPENDENT		/* share TX/RX interrupts */

#define AQ_NINTR_MAX			(AQ_RSSQUEUE_MAX + AQ_RSSQUEUE_MAX + 1)
					/* TX + RX + LINK. must be <= 32 */
#define AQ_LINKSTAT_IRQ			31	/* for legacy mode */

#define AQ_TXD_NUM			2048	/* per ring. 8*n && 32~8184 */
#define AQ_RXD_NUM			2048	/* per ring. 8*n && 32~8184 */
/* minimum required to send a packet (vlan needs additional TX descriptor) */
#define AQ_TXD_MIN			(1 + 1)


/* hardware specification */
#define AQ_RINGS_NUM			32
#define AQ_RSSQUEUE_MAX			8
#define AQ_RX_DESCRIPTOR_MIN		32
#define AQ_TX_DESCRIPTOR_MIN		32
#define AQ_RX_DESCRIPTOR_MAX		8184
#define AQ_TX_DESCRIPTOR_MAX		8184
#define AQ_TRAFFICCLASS_NUM		8
#define AQ_RSS_HASHKEY_SIZE		40
#define AQ_RSS_INDIRECTION_TABLE_MAX	64

/*
 * TERMINOLOGY
 *	MPI = MAC PHY INTERFACE?
 *	RPO = RX Protocol Offloading
 *	TPO = TX Protocol Offloading
 *	RPF = RX Packet Filter
 *	TPB = TX Packet buffer
 *	RPB = RX Packet buffer
 */

/* registers */
#define AQ_FW_SOFTRESET_REG			0x0000
#define  AQ_FW_SOFTRESET_RESET			__BIT(15) /* soft reset bit */
#define  AQ_FW_SOFTRESET_DIS			__BIT(14) /* reset disable */

#define AQ_FW_VERSION_REG			0x0018
#define AQ_HW_REVISION_REG			0x001c
#define AQ_GLB_NVR_INTERFACE1_REG		0x0100

#define AQ_FW_MBOX_CMD_REG			0x0200
#define  AQ_FW_MBOX_CMD_EXECUTE			0x00008000
#define  AQ_FW_MBOX_CMD_BUSY			0x00000100
#define AQ_FW_MBOX_ADDR_REG			0x0208
#define AQ_FW_MBOX_VAL_REG			0x020c

#define FW2X_LED_MIN_VERSION			0x03010026	/* >= 3.1.38 */
#define FW2X_LED_REG				0x031c
#define  FW2X_LED_DEFAULT			0x00000000
#define  FW2X_LED_NONE				0x0000003f
#define  FW2X_LINKLED				__BITS(0,1)
#define   FW2X_LINKLED_ACTIVE			0
#define   FW2X_LINKLED_ON			1
#define   FW2X_LINKLED_BLINK			2
#define   FW2X_LINKLED_OFF			3
#define  FW2X_STATUSLED				__BITS(2,5)
#define   FW2X_STATUSLED_ORANGE			0
#define   FW2X_STATUSLED_ORANGE_BLINK		2
#define   FW2X_STATUSLED_OFF			3
#define   FW2X_STATUSLED_GREEN			4
#define   FW2X_STATUSLED_ORANGE_GREEN_BLINK	8
#define   FW2X_STATUSLED_GREEN_BLINK		10

#define FW_MPI_MBOX_ADDR_REG			0x0360
#define FW1X_MPI_INIT1_REG			0x0364
#define FW1X_MPI_CONTROL_REG			0x0368
#define FW1X_MPI_STATE_REG			0x036c
#define  FW1X_MPI_STATE_MODE			__BITS(7,0)
#define  FW1X_MPI_STATE_SPEED			__BITS(32,16)
#define  FW1X_MPI_STATE_DISABLE_DIRTYWAKE	__BITS(25)
#define  FW1X_MPI_STATE_DOWNSHIFT		__BITS(31,28)
#define FW1X_MPI_INIT2_REG			0x0370
#define FW1X_MPI_EFUSEADDR_REG			0x0374

#define FW2X_MPI_EFUSEADDR_REG			0x0364
#define FW2X_MPI_CONTROL_REG			0x0368	/* 64bit */
#define FW2X_MPI_STATE_REG			0x0370	/* 64bit */
#define FW_BOOT_EXIT_CODE_REG			0x0388
#define  RBL_STATUS_DEAD			0x0000dead
#define  RBL_STATUS_SUCCESS			0x0000abba
#define  RBL_STATUS_FAILURE			0x00000bad
#define  RBL_STATUS_HOST_BOOT			0x0000f1a7

#define AQ_FW_GLB_CPU_SEM_REG(i)		(0x03a0 + (i) * 4)
#define AQ_FW_SEM_RAM_REG			AQ_FW_GLB_CPU_SEM_REG(2)

#define AQ_FW_GLB_CTL2_REG			0x0404
#define  AQ_FW_GLB_CTL2_MCP_UP_FORCE_INTERRUPT	__BIT(1)

#define AQ_GLB_GENERAL_PROVISIONING9_REG	0x0520
#define AQ_GLB_NVR_PROVISIONING2_REG		0x0534

#define FW_MPI_DAISY_CHAIN_STATUS_REG		0x0704

#define AQ_PCI_REG_CONTROL_6_REG		0x1014

// msix bitmap */
#define AQ_INTR_STATUS_REG			0x2000	/* intr status */
#define AQ_INTR_STATUS_CLR_REG			0x2050	/* intr status clear */
#define AQ_INTR_MASK_REG			0x2060	/* intr mask set */
#define AQ_INTR_MASK_CLR_REG			0x2070	/* intr mask clear */
#define AQ_INTR_AUTOMASK_REG			0x2090

/* AQ_INTR_IRQ_MAP_TXRX_REG[AQ_RINGS_NUM] 0x2100-0x2140 */
#define AQ_INTR_IRQ_MAP_TXRX_REG(i)		(0x2100 + ((i) / 2) * 4)
#define AQ_INTR_IRQ_MAP_TX_REG(i)		AQ_INTR_IRQ_MAP_TXRX_REG(i)
#define  AQ_INTR_IRQ_MAP_TX_IRQMAP(i)		(__BITS(28,24) >> (((i) & 1)*8))
#define  AQ_INTR_IRQ_MAP_TX_EN(i)		(__BIT(31)     >> (((i) & 1)*8))
#define AQ_INTR_IRQ_MAP_RX_REG(i)		AQ_INTR_IRQ_MAP_TXRX_REG(i)
#define  AQ_INTR_IRQ_MAP_RX_IRQMAP(i)		(__BITS(12,8)  >> (((i) & 1)*8))
#define  AQ_INTR_IRQ_MAP_RX_EN(i)		(__BIT(15)     >> (((i) & 1)*8))

/* AQ_GEN_INTR_MAP_REG[AQ_RINGS_NUM] 0x2180-0x2200 */
#define AQ_GEN_INTR_MAP_REG(i)			(0x2180 + (i) * 4)
#define  AQ_B0_ERR_INT				8U

#define AQ_INTR_CTRL_REG			0x2300
#define  AQ_INTR_CTRL_IRQMODE			__BITS(1,0)
#define  AQ_INTR_CTRL_IRQMODE_LEGACY		0
#define  AQ_INTR_CTRL_IRQMODE_MSI		1
#define  AQ_INTR_CTRL_IRQMODE_MSIX		2
#define  AQ_INTR_CTRL_MULTIVEC			__BIT(2)
#define  AQ_INTR_CTRL_AUTO_MASK			__BIT(5)
#define  AQ_INTR_CTRL_CLR_ON_READ		__BIT(7)
#define  AQ_INTR_CTRL_RESET_DIS			__BIT(29)
#define  AQ_INTR_CTRL_RESET_IRQ			__BIT(31)

#define AQ_MBOXIF_POWER_GATING_CONTROL_REG	0x32a8

#define FW_MPI_RESETCTRL_REG			0x4000
#define  FW_MPI_RESETCTRL_RESET_DIS		__BIT(29)

#define RX_SYSCONTROL_REG			0x5000
#define  RX_SYSCONTROL_RPB_DMA_LOOPBACK		__BIT(6)
#define  RX_SYSCONTROL_RPF_TPO_LOOPBACK		__BIT(8)
#define  RX_SYSCONTROL_RESET_DIS		__BIT(29)

#define RX_TCP_RSS_HASH_REG			0x5040
#define  RX_TCP_RSS_HASH_RPF2			__BITS(19,16)
#define  RX_TCP_RSS_HASH_TYPE			__BITS(15,0)

/* for RPF_*_REG.ACTION */
#define RPF_ACTION_DISCARD			0
#define RPF_ACTION_HOST				1
#define RPF_ACTION_MANAGEMENT			2
#define RPF_ACTION_HOST_MANAGEMENT		3
#define RPF_ACTION_WOL				4

#define RPF_L2BC_REG				0x5100
#define  RPF_L2BC_EN				__BIT(0)
#define  RPF_L2BC_PROMISC			__BIT(3)
#define  RPF_L2BC_ACTION			__BITS(12,14)
#define  RPF_L2BC_THRESHOLD			__BITS(31,16)

/* RPF_L2UC_*_REG[34] (actual [38]?) */
#define RPF_L2UC_LSW_REG(i)			(0x5110 + (i) * 8)
#define RPF_L2UC_MSW_REG(i)			(0x5114 + (i) * 8)
#define  RPF_L2UC_MSW_MACADDR_HI		__BITS(15,0)
#define  RPF_L2UC_MSW_ACTION			__BITS(18,16)
#define  RPF_L2UC_MSW_EN			__BIT(31)
#define AQ_HW_MAC_OWN			0	/* index of own address */
#define AQ_HW_MAC_NUM			34

/* RPF_MCAST_FILTER_REG[8] 0x5250-0x5270 */
#define RPF_MCAST_FILTER_REG(i)			(0x5250 + (i) * 4)
#define  RPF_MCAST_FILTER_EN			__BIT(31)
#define RPF_MCAST_FILTER_MASK_REG		0x5270
#define  RPF_MCAST_FILTER_MASK_ALLMULTI		__BIT(14)

#define RPF_VLAN_MODE_REG			0x5280
#define  RPF_VLAN_MODE_PROMISC			__BIT(1)
#define  RPF_VLAN_MODE_ACCEPT_UNTAGGED		__BIT(2)
#define  RPF_VLAN_MODE_UNTAGGED_ACTION		__BITS(5,3)

#define RPF_VLAN_TPID_REG			0x5284
#define  RPF_VLAN_TPID_OUTER			__BITS(31,16)
#define  RPF_VLAN_TPID_INNER			__BITS(15,0)

/* RPF_VLAN_FILTER_REG[RPF_VLAN_MAX_FILTERS] 0x5290-0x52d0 */
#define RPF_VLAN_MAX_FILTERS			16
#define RPF_VLAN_FILTER_REG(i)			(0x5290 + (i) * 4)
#define  RPF_VLAN_FILTER_EN			__BIT(31)
#define  RPF_VLAN_FILTER_RXQ_EN			__BIT(28)
#define  RPF_VLAN_FILTER_RXQ			__BITS(24,20)
#define  RPF_VLAN_FILTER_ACTION			__BITS(18,16)
#define  RPF_VLAN_FILTER_ID			__BITS(11,0)

/* RPF_ETHERTYPE_FILTER_REG[AQ_RINGS_NUM] 0x5300-0x5380 */
#define RPF_ETHERTYPE_FILTER_REG(i)		(0x5300 + (i) * 4)
#define  RPF_ETHERTYPE_FILTER_EN		__BIT(31)
#define  RPF_ETHERTYPE_FILTER_PRIO_EN		__BIT(30)
#define  RPF_ETHERTYPE_FILTER_RXQF_EN		__BIT(29)
#define  RPF_ETHERTYPE_FILTER_PRIO		__BITS(28,26)
#define  RPF_ETHERTYPE_FILTER_RXQF		__BITS(24,20)
#define  RPF_ETHERTYPE_FILTER_MNG_RXQF		__BIT(19)
#define  RPF_ETHERTYPE_FILTER_ACTION		__BITS(18,16)
#define  RPF_ETHERTYPE_FILTER_VAL		__BITS(15,0)

/* RPF_L3_FILTER_REG[8] 0x5380-0x53a0 */
#define RPF_L3_FILTER_REG(i)			(0x5380 + (i) * 4)
#define  RPF_L3_FILTER_L4_EN			__BIT(31)
#define  RPF_L3_FILTER_IPV6_EN			__BIT(30)
#define  RPF_L3_FILTER_SRCADDR_EN		__BIT(29)
#define  RPF_L3_FILTER_DSTADDR_EN		__BIT(28)
#define  RPF_L3_FILTER_L4_SRCPORT_EN		__BIT(27)
#define  RPF_L3_FILTER_L4_DSTPORT_EN		__BIT(26)
#define  RPF_L3_FILTER_L4_PROTO_EN		__BIT(25)
#define  RPF_L3_FILTER_ARP_EN			__BIT(24)
#define  RPF_L3_FILTER_L4_RXQUEUE_EN		__BIT(23)
#define  RPF_L3_FILTER_L4_RXQUEUE_MANAGEMENT_EN	__BIT(22)
#define  RPF_L3_FILTER_L4_ACTION		__BITS(16,18)
#define  RPF_L3_FILTER_L4_RXQUEUE		__BITS(12,8)
#define  RPF_L3_FILTER_L4_PROTO			__BITS(2,0)
#define   RPF_L3_FILTER_L4_PROTO_TCP		0
#define   RPF_L3_FILTER_L4_PROTO_UDP		1
#define   RPF_L3_FILTER_L4_PROTO_SCTP		2
#define   RPF_L3_FILTER_L4_PROTO_ICMP		3
/* parameters of RPF_L3_FILTER_REG[8] */
#define RPF_L3_FILTER_SRCADDR_REG(i)		(0x53b0 + (i) * 4)
#define RPF_L3_FILTER_DSTADDR_REG(i)		(0x53d0 + (i) * 4)
#define RPF_L3_FILTER_L4_SRCPORT_REG(i)		(0x5400 + (i) * 4)
#define RPF_L3_FILTER_L4_DSTPORT_REG(i)		(0x5420 + (i) * 4)

#define RX_FLR_RSS_CONTROL1_REG			0x54c0
#define  RX_FLR_RSS_CONTROL1_EN			__BIT(31)

#define RPF_RPB_RX_TC_UPT_REG			0x54c4
#define  RPF_RPB_RX_TC_UPT_MASK(i)		(0x00000007 << ((i) * 4))

#define RPF_RSS_KEY_ADDR_REG			0x54d0
#define  RPF_RSS_KEY_ADDR			__BITS(4,0)
#define  RPF_RSS_KEY_WR_EN			__BIT(5)
#define RPF_RSS_KEY_WR_DATA_REG			0x54d4
#define RPF_RSS_KEY_RD_DATA_REG			0x54d8

#define RPF_RSS_REDIR_ADDR_REG			0x54e0
#define  RPF_RSS_REDIR_ADDR			__BITS(3,0)
#define  RPF_RSS_REDIR_WR_EN			__BIT(4)

#define RPF_RSS_REDIR_WR_DATA_REG		0x54e4
#define  RPF_RSS_REDIR_WR_DATA			__BITS(15,0)

#define RPO_HWCSUM_REG				0x5580
#define  RPO_HWCSUM_IP4CSUM_EN			__BIT(1)
#define  RPO_HWCSUM_L4CSUM_EN			__BIT(0) /* TCP/UDP/SCTP */

#define RPO_LRO_ENABLE_REG			0x5590

#define RPO_LRO_CONF_REG			0x5594
#define  RPO_LRO_CONF_QSESSION_LIMIT		__BITS(13,12)
#define  RPO_LRO_CONF_TOTAL_DESC_LIMIT		__BITS(6,5)
#define  RPO_LRO_CONF_PATCHOPTIMIZATION_EN	__BIT(15)
#define  RPO_LRO_CONF_MIN_PAYLOAD_OF_FIRST_PKT	__BITS(4,0)
#define RPO_LRO_RSC_MAX_REG			0x5598

/* RPO_LRO_LDES_MAX_REG[32/8] 0x55a0-0x55b0 */
#define RPO_LRO_LDES_MAX_REG(i)			(0x55a0 + (i / 8) * 4)
#define  RPO_LRO_LDES_MAX_MASK(i)		(0x00000003 << ((i & 7) * 4))
#define RPO_LRO_TB_DIV_REG			0x5620
#define  RPO_LRO_TB_DIV				__BITS(20,31)
#define RPO_LRO_INACTIVE_IVAL_REG		0x5620
#define  RPO_LRO_INACTIVE_IVAL			__BITS(10,19)
#define RPO_LRO_MAX_COALESCING_IVAL_REG		0x5620
#define  RPO_LRO_MAX_COALESCING_IVAL		__BITS(9,0)

#define RPB_RPF_RX_REG				0x5700
#define  RPB_RPF_RX_TC_MODE			__BIT(8)
#define  RPB_RPF_RX_FC_MODE			__BITS(5,4)
#define  RPB_RPF_RX_BUF_EN			__BIT(0)

/* RPB_RXB_BUFSIZE_REG[AQ_TRAFFICCLASS_NUM] 0x5710-0x5790 */
#define RPB_RXB_BUFSIZE_REG(i)			(0x5710 + (i) * 0x10)
#define  RPB_RXB_BUFSIZE			__BITS(8,0)
#define RPB_RXB_XOFF_REG(i)			(0x5714 + (i) * 0x10)
#define  RPB_RXB_XOFF_EN			__BIT(31)
#define  RPB_RXB_XOFF_THRESH_HI			__BITS(29,16)
#define  RPB_RXB_XOFF_THRESH_LO			__BITS(13,0)

#define RX_DMA_DESC_CACHE_INIT_REG		0x5a00
#define  RX_DMA_DESC_CACHE_INIT			__BIT(0)

#define RX_DMA_INT_DESC_WRWB_EN_REG		0x05a30
#define  RX_DMA_INT_DESC_WRWB_EN		__BIT(2)
#define  RX_DMA_INT_DESC_MODERATE_EN		__BIT(3)

/* RX_INTR_MODERATION_CTL_REG[AQ_RINGS_NUM] 0x5a40-0x5ac0 */
#define RX_INTR_MODERATION_CTL_REG(i)		(0x5a40 + (i) * 4)
#define  RX_INTR_MODERATION_CTL_EN		__BIT(1)
#define  RX_INTR_MODERATION_CTL_MIN		__BITS(15,8)
#define  RX_INTR_MODERATION_CTL_MAX		__BITS(24,16)

/* RX_DMA_DESC_*[AQ_RINGS_NUM] 0x5b00-0x5f00 */
#define RX_DMA_DESC_BASE_ADDRLSW_REG(i)		(0x5b00 + (i) * 0x20)
#define RX_DMA_DESC_BASE_ADDRMSW_REG(i)		(0x5b04 + (i) * 0x20)
#define RX_DMA_DESC_REG(i)			(0x5b08 + (i) * 0x20)
#define  RX_DMA_DESC_LEN			__BITS(12,3)	/* RXD_NUM/8 */
#define  RX_DMA_DESC_RESET			__BIT(25)
#define  RX_DMA_DESC_HEADER_SPLIT		__BIT(28)
#define  RX_DMA_DESC_VLAN_STRIP			__BIT(29)
#define  RX_DMA_DESC_EN				__BIT(31)
#define RX_DMA_DESC_HEAD_PTR_REG(i)		(0x5b0c + (i) * 0x20)
#define  RX_DMA_DESC_HEAD_PTR			__BITS(12,0)
#define RX_DMA_DESC_TAIL_PTR_REG(i)		(0x5b10 + (i) * 0x20)
#define RX_DMA_DESC_BUFSIZE_REG(i)		(0x5b18 + (i) * 0x20)
#define  RX_DMA_DESC_BUFSIZE_DATA		__BITS(4,0)
#define  RX_DMA_DESC_BUFSIZE_HDR		__BITS(12,8)

/* RX_DMA_DCAD_REG[AQ_RINGS_NUM] 0x6100-0x6180 */
#define RX_DMA_DCAD_REG(i)			(0x6100 + (i) * 4)
#define  RX_DMA_DCAD_CPUID			__BITS(7,0)
#define  RX_DMA_DCAD_PAYLOAD_EN			__BIT(29)
#define  RX_DMA_DCAD_HEADER_EN			__BIT(30)
#define  RX_DMA_DCAD_DESC_EN			__BIT(31)

#define RX_DMA_DCA_REG				0x6180
#define  RX_DMA_DCA_EN				__BIT(31)
#define  RX_DMA_DCA_MODE			__BITS(3,0)

/* counters */
#define RX_DMA_GOOD_PKT_COUNTERLSW		0x6800
#define RX_DMA_GOOD_OCTET_COUNTERLSW		0x6808
#define RX_DMA_DROP_PKT_CNT_REG			0x6818
#define RX_DMA_COALESCED_PKT_CNT_REG		0x6820

#define TX_SYSCONTROL_REG			0x7000
#define  TX_SYSCONTROL_TPB_DMA_LOOPBACK		__BIT(6)
#define  TX_SYSCONTROL_TPO_PKT_LOOPBACK		__BIT(7)
#define  TX_SYSCONTROL_RESET_DIS		__BIT(29)

#define TX_TPO2_REG				0x7040
#define  TX_TPO2_EN				__BIT(16)

#define TPS_DESC_VM_ARB_MODE_REG		0x7300
#define  TPS_DESC_VM_ARB_MODE			__BIT(0)
#define TPS_DESC_RATE_REG			0x7310
#define  TPS_DESC_RATE_TA_RST			__BIT(31)
#define  TPS_DESC_RATE_LIM			__BITS(10,0)
#define TPS_DESC_TC_ARB_MODE_REG		0x7200
#define  TPS_DESC_TC_ARB_MODE			__BITS(1,0)
#define TPS_DATA_TC_ARB_MODE_REG		0x7100
#define  TPS_DATA_TC_ARB_MODE			__BIT(0)

/* TPS_DATA_TCT_REG[AQ_TRAFFICCLASS_NUM] 0x7110-0x7130 */
#define TPS_DATA_TCT_REG(i)			(0x7110 + (i) * 4)
#define  TPS_DATA_TCT_CREDIT_MAX		__BITS(16,27)
#define  TPS_DATA_TCT_WEIGHT			__BITS(8,0)
/* TPS_DATA_TCT_REG[AQ_TRAFFICCLASS_NUM] 0x7210-0x7230 */
#define TPS_DESC_TCT_REG(i)			(0x7210 + (i) * 4)
#define  TPS_DESC_TCT_CREDIT_MAX		__BITS(16,27)
#define  TPS_DESC_TCT_WEIGHT			__BITS(8,0)

#define AQ_HW_TXBUF_MAX		160
#define AQ_HW_RXBUF_MAX		320

#define TPO_HWCSUM_REG				0x7800
#define  TPO_HWCSUM_IP4CSUM_EN			__BIT(1)
#define  TPO_HWCSUM_L4CSUM_EN			__BIT(0) /* TCP/UDP/SCTP */

#define TDM_LSO_EN_REG				0x7810

#define THM_LSO_TCP_FLAG1_REG			0x7820
#define  THM_LSO_TCP_FLAG1_FIRST		__BITS(11,0)
#define  THM_LSO_TCP_FLAG1_MID			__BITS(27,16)
#define THM_LSO_TCP_FLAG2_REG			0x7824
#define  THM_LSO_TCP_FLAG2_LAST			__BITS(11,0)

#define TPB_TX_BUF_REG				0x7900
#define  TPB_TX_BUF_EN				__BIT(0)
#define  TPB_TX_BUF_SCP_INS_EN			__BIT(2)
#define  TPB_TX_BUF_TC_MODE_EN			__BIT(8)

/* TPB_TXB_BUFSIZE_REG[AQ_TRAFFICCLASS_NUM] 0x7910-7990 */
#define TPB_TXB_BUFSIZE_REG(i)			(0x7910 + (i) * 0x10)
#define  TPB_TXB_BUFSIZE			__BITS(7,0)
#define TPB_TXB_THRESH_REG(i)			(0x7914 + (i) * 0x10)
#define  TPB_TXB_THRESH_HI			__BITS(16,28)
#define  TPB_TXB_THRESH_LO			__BITS(12,0)

#define AQ_HW_TX_DMA_TOTAL_REQ_LIMIT_REG	0x7b20
#define TX_DMA_INT_DESC_WRWB_EN_REG		0x7b40
#define  TX_DMA_INT_DESC_WRWB_EN		__BIT(1)
#define  TX_DMA_INT_DESC_MODERATE_EN		__BIT(4)

/* TX_DMA_DESC_*[AQ_RINGS_NUM] 0x7c00-0x8400 */
#define TX_DMA_DESC_BASE_ADDRLSW_REG(i)		(0x7c00 + (i) * 0x40)
#define TX_DMA_DESC_BASE_ADDRMSW_REG(i)		(0x7c04 + (i) * 0x40)
#define TX_DMA_DESC_REG(i)			(0x7c08 + (i) * 0x40)
#define  TX_DMA_DESC_LEN			__BITS(12, 3)	/* TXD_NUM/8 */
#define  TX_DMA_DESC_EN				__BIT(31)
#define TX_DMA_DESC_HEAD_PTR_REG(i)		(0x7c0c + (i) * 0x40)
#define  TX_DMA_DESC_HEAD_PTR			__BITS(12,0)
#define TX_DMA_DESC_TAIL_PTR_REG(i)		(0x7c10 + (i) * 0x40)
#define TX_DMA_DESC_WRWB_THRESH_REG(i)		(0x7c18 + (i) * 0x40)
#define  TX_DMA_DESC_WRWB_THRESH		__BITS(14,8)

/* TDM_DCAD_REG[AQ_RINGS_NUM] 0x8400-0x8480 */
#define TDM_DCAD_REG(i)				(0x8400 + (i) * 4)
#define  TDM_DCAD_CPUID				__BITS(7,0)
#define  TDM_DCAD_CPUID_EN			__BIT(31)

#define TDM_DCA_REG				0x8480
#define  TDM_DCA_EN				__BIT(31)
#define  TDM_DCA_MODE				__BITS(3,0)

/* TX_INTR_MODERATION_CTL_REG[AQ_RINGS_NUM] 0x8980-0x8a00 */
#define TX_INTR_MODERATION_CTL_REG(i)		(0x8980 + (i) * 4)
#define  TX_INTR_MODERATION_CTL_EN		__BIT(1)
#define  TX_INTR_MODERATION_CTL_MIN		__BITS(15,8)
#define  TX_INTR_MODERATION_CTL_MAX		__BITS(24,16)

#define FW1X_CTRL_10G				__BIT(0)
#define FW1X_CTRL_5G				__BIT(1)
#define FW1X_CTRL_5GSR				__BIT(2)
#define FW1X_CTRL_2G5				__BIT(3)
#define FW1X_CTRL_1G				__BIT(4)
#define FW1X_CTRL_100M				__BIT(5)

#define FW2X_CTRL_10BASET_HD			__BIT(0)
#define FW2X_CTRL_10BASET_FD			__BIT(1)
#define FW2X_CTRL_100BASETX_HD			__BIT(2)
#define FW2X_CTRL_100BASET4_HD			__BIT(3)
#define FW2X_CTRL_100BASET2_HD			__BIT(4)
#define FW2X_CTRL_100BASETX_FD			__BIT(5)
#define FW2X_CTRL_100BASET2_FD			__BIT(6)
#define FW2X_CTRL_1000BASET_HD			__BIT(7)
#define FW2X_CTRL_1000BASET_FD			__BIT(8)
#define FW2X_CTRL_2P5GBASET_FD			__BIT(9)
#define FW2X_CTRL_5GBASET_FD			__BIT(10)
#define FW2X_CTRL_10GBASET_FD			__BIT(11)
#define FW2X_CTRL_RESERVED1			__BIT(32)
#define FW2X_CTRL_10BASET_EEE			__BIT(33)
#define FW2X_CTRL_RESERVED2			__BIT(34)
#define FW2X_CTRL_PAUSE				__BIT(35)
#define FW2X_CTRL_ASYMMETRIC_PAUSE		__BIT(36)
#define FW2X_CTRL_100BASETX_EEE			__BIT(37)
#define FW2X_CTRL_RESERVED3			__BIT(38)
#define FW2X_CTRL_RESERVED4			__BIT(39)
#define FW2X_CTRL_1000BASET_FD_EEE		__BIT(40)
#define FW2X_CTRL_2P5GBASET_FD_EEE		__BIT(41)
#define FW2X_CTRL_5GBASET_FD_EEE		__BIT(42)
#define FW2X_CTRL_10GBASET_FD_EEE		__BIT(43)
#define FW2X_CTRL_RESERVED5			__BIT(44)
#define FW2X_CTRL_RESERVED6			__BIT(45)
#define FW2X_CTRL_RESERVED7			__BIT(46)
#define FW2X_CTRL_RESERVED8			__BIT(47)
#define FW2X_CTRL_RESERVED9			__BIT(48)
#define FW2X_CTRL_CABLE_DIAG			__BIT(49)
#define FW2X_CTRL_TEMPERATURE			__BIT(50)
#define FW2X_CTRL_DOWNSHIFT			__BIT(51)
#define FW2X_CTRL_PTP_AVB_EN			__BIT(52)
#define FW2X_CTRL_MEDIA_DETECT			__BIT(53)
#define FW2X_CTRL_LINK_DROP			__BIT(54)
#define FW2X_CTRL_SLEEP_PROXY			__BIT(55)
#define FW2X_CTRL_WOL				__BIT(56)
#define FW2X_CTRL_MAC_STOP			__BIT(57)
#define FW2X_CTRL_EXT_LOOPBACK			__BIT(58)
#define FW2X_CTRL_INT_LOOPBACK			__BIT(59)
#define FW2X_CTRL_EFUSE_AGENT			__BIT(60)
#define FW2X_CTRL_WOL_TIMER			__BIT(61)
#define FW2X_CTRL_STATISTICS			__BIT(62)
#define FW2X_CTRL_TRANSACTION_ID		__BIT(63)

#define FW2X_SNPRINTB			\
	"\177\020"			\
	"b\x23" "PAUSE\0"		\
	"b\x24" "ASYMMETRIC-PAUSE\0"	\
	"b\x31" "CABLE-DIAG\0"		\
	"b\x32" "TEMPERATURE\0"		\
	"b\x33" "DOWNSHIFT\0"		\
	"b\x34" "PTP-AVB\0"		\
	"b\x35" "MEDIA-DETECT\0"	\
	"b\x36" "LINK-DROP\0"		\
	"b\x37" "SLEEP-PROXY\0"		\
	"b\x38" "WOL\0"			\
	"b\x39" "MAC-STOP\0"		\
	"b\x3a" "EXT-LOOPBACK\0"	\
	"b\x3b" "INT-LOOPBACK\0"	\
	"b\x3c" "EFUSE-AGENT\0"		\
	"b\x3d" "WOL-TIMER\0"		\
	"b\x3e" "STATISTICS\0"		\
	"b\x3f" "TRANSACTION-ID\0"	\
	"\0"

#define FW2X_CTRL_RATE_100M			FW2X_CTRL_100BASETX_FD
#define FW2X_CTRL_RATE_1G			FW2X_CTRL_1000BASET_FD
#define FW2X_CTRL_RATE_2G5			FW2X_CTRL_2P5GBASET_FD
#define FW2X_CTRL_RATE_5G			FW2X_CTRL_5GBASET_FD
#define FW2X_CTRL_RATE_10G			FW2X_CTRL_10GBASET_FD
#define FW2X_CTRL_RATE_MASK		\
	(FW2X_CTRL_RATE_100M |		\
	 FW2X_CTRL_RATE_1G |		\
	 FW2X_CTRL_RATE_2G5 |		\
	 FW2X_CTRL_RATE_5G |		\
	 FW2X_CTRL_RATE_10G)
#define FW2X_CTRL_EEE_MASK		\
	(FW2X_CTRL_10BASET_EEE |	\
	 FW2X_CTRL_100BASETX_EEE |	\
	 FW2X_CTRL_1000BASET_FD_EEE |	\
	 FW2X_CTRL_2P5GBASET_FD_EEE |	\
	 FW2X_CTRL_5GBASET_FD_EEE |	\
	 FW2X_CTRL_10GBASET_FD_EEE)

typedef enum aq_fw_bootloader_mode {
	FW_BOOT_MODE_UNKNOWN = 0,
	FW_BOOT_MODE_FLB,
	FW_BOOT_MODE_RBL_FLASH,
	FW_BOOT_MODE_RBL_HOST_BOOTLOAD
} aq_fw_bootloader_mode_t;

#define AQ_WRITE_REG(sc, reg, val)				\
	bus_space_write_4((sc)->sc_iot, (sc)->sc_ioh, (reg), (val))

#define AQ_READ_REG(sc, reg)					\
	bus_space_read_4((sc)->sc_iot, (sc)->sc_ioh, (reg))

#define AQ_READ64_REG(sc, reg)					\
	((uint64_t)AQ_READ_REG(sc, reg) |			\
	(((uint64_t)AQ_READ_REG(sc, (reg) + 4)) << 32))

#define AQ_WRITE64_REG(sc, reg, val)				\
	do {							\
		AQ_WRITE_REG(sc, reg, (uint32_t)val);		\
		AQ_WRITE_REG(sc, reg + 4, (uint32_t)(val >> 32)); \
	} while (/* CONSTCOND */0)

#define AQ_READ_REG_BIT(sc, reg, mask)				\
	__SHIFTOUT(AQ_READ_REG(sc, reg), mask)

#define AQ_WRITE_REG_BIT(sc, reg, mask, val)			\
	do {							\
		uint32_t _v;					\
		_v = AQ_READ_REG((sc), (reg));			\
		_v &= ~(mask);					\
		if ((val) != 0)					\
			_v |= __SHIFTIN((val), (mask));		\
		AQ_WRITE_REG((sc), (reg), _v);			\
	} while (/* CONSTCOND */ 0)

#define WAIT_FOR(expr, us, n, errp)				\
	do {							\
		unsigned int _n;				\
		for (_n = n; (!(expr)) && _n != 0; --_n) {	\
			delay((us));				\
		}						\
		if ((errp != NULL)) {				\
			if (_n == 0)				\
				*(errp) = ETIMEDOUT;		\
			else					\
				*(errp) = 0;			\
		}						\
	} while (/* CONSTCOND */ 0)

#define msec_delay(x)	DELAY(1000 * (x))

typedef struct aq_mailbox_header {
	uint32_t version;
	uint32_t transaction_id;
	int32_t error;
} __packed aq_mailbox_header_t;

typedef struct aq_hw_stats_s {
	uint32_t uprc;
	uint32_t mprc;
	uint32_t bprc;
	uint32_t erpt;
	uint32_t uptc;
	uint32_t mptc;
	uint32_t bptc;
	uint32_t erpr;
	uint32_t mbtc;
	uint32_t bbtc;
	uint32_t mbrc;
	uint32_t bbrc;
	uint32_t ubrc;
	uint32_t ubtc;
	uint32_t ptc;
	uint32_t prc;
	uint32_t dpc;	/* not exists in fw2x_msm_statistics */
	uint32_t cprc;	/* not exists in fw2x_msm_statistics */
} __packed aq_hw_stats_s_t;

typedef struct fw1x_mailbox {
	aq_mailbox_header_t header;
	aq_hw_stats_s_t msm;
} __packed fw1x_mailbox_t;

typedef struct fw2x_msm_statistics {
	uint32_t uprc;
	uint32_t mprc;
	uint32_t bprc;
	uint32_t erpt;
	uint32_t uptc;
	uint32_t mptc;
	uint32_t bptc;
	uint32_t erpr;
	uint32_t mbtc;
	uint32_t bbtc;
	uint32_t mbrc;
	uint32_t bbrc;
	uint32_t ubrc;
	uint32_t ubtc;
	uint32_t ptc;
	uint32_t prc;
} __packed fw2x_msm_statistics_t;

typedef struct fw2x_phy_cable_diag_data {
	uint32_t lane_data[4];
} __packed fw2x_phy_cable_diag_data_t;

typedef struct fw2x_capabilities {
	uint32_t caps_lo;
	uint32_t caps_hi;
} __packed fw2x_capabilities_t;

typedef struct fw2x_mailbox {		/* struct fwHostInterface */
	aq_mailbox_header_t header;
	fw2x_msm_statistics_t msm;	/* msmStatistics_t msm; */

	uint32_t phy_info1;
#define PHYINFO1_FAULT_CODE	__BITS(31,16)
#define PHYINFO1_PHY_H_BIT	__BITS(0,15)
	uint32_t phy_info2;
#define PHYINFO2_TEMPERATURE	__BITS(15,0)
#define PHYINFO2_CABLE_LEN	__BITS(23,16)

	fw2x_phy_cable_diag_data_t diag_data;
	uint32_t reserved[8];

	fw2x_capabilities_t caps;

	/* ... */
} __packed fw2x_mailbox_t;

typedef enum aq_link_speed {
	AQ_LINK_NONE	= 0,
	AQ_LINK_100M	= (1 << 0),
	AQ_LINK_1G	= (1 << 1),
	AQ_LINK_2G5	= (1 << 2),
	AQ_LINK_5G	= (1 << 3),
	AQ_LINK_10G	= (1 << 4)
} aq_link_speed_t;
#define AQ_LINK_ALL	(AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | \
			 AQ_LINK_5G | AQ_LINK_10G )
#define AQ_LINK_AUTO	AQ_LINK_ALL

typedef enum aq_link_fc {
	AQ_FC_NONE = 0,
	AQ_FC_RX = __BIT(0),
	AQ_FC_TX = __BIT(1),
	AQ_FC_ALL = (AQ_FC_RX | AQ_FC_TX)
} aq_link_fc_t;

typedef enum aq_link_eee {
	AQ_EEE_DISABLE = 0,
	AQ_EEE_ENABLE = 1
} aq_link_eee_t;

typedef enum aq_hw_fw_mpi_state {
	MPI_DEINIT	= 0,
	MPI_RESET	= 1,
	MPI_INIT	= 2,
	MPI_POWER	= 4
} aq_hw_fw_mpi_state_t;

enum aq_media_type {
	AQ_MEDIA_TYPE_UNKNOWN = 0,
	AQ_MEDIA_TYPE_FIBRE,
	AQ_MEDIA_TYPE_TP
};

struct aq_rx_desc_read {
	uint64_t buf_addr;
	uint64_t hdr_addr;
} __packed;

struct aq_rx_desc_wb {
	uint32_t type;
#define RXDESC_TYPE_RSSTYPE		__BITS(3,0)
#define  RXDESC_TYPE_RSSTYPE_NONE		0
#define  RXDESC_TYPE_RSSTYPE_IPV4		2
#define  RXDESC_TYPE_RSSTYPE_IPV6		3
#define  RXDESC_TYPE_RSSTYPE_IPV4_TCP		4
#define  RXDESC_TYPE_RSSTYPE_IPV6_TCP		5
#define  RXDESC_TYPE_RSSTYPE_IPV4_UDP		6
#define  RXDESC_TYPE_RSSTYPE_IPV6_UDP		7
#define RXDESC_TYPE_PKTTYPE_ETHER	__BITS(5,4)
#define  RXDESC_TYPE_PKTTYPE_ETHER_IPV4		0
#define  RXDESC_TYPE_PKTTYPE_ETHER_IPV6		1
#define  RXDESC_TYPE_PKTTYPE_ETHER_OTHERS	2
#define  RXDESC_TYPE_PKTTYPE_ETHER_ARP		3
#define RXDESC_TYPE_PKTTYPE_PROTO	__BITS(8,6)
#define  RXDESC_TYPE_PKTTYPE_PROTO_TCP		0
#define  RXDESC_TYPE_PKTTYPE_PROTO_UDP		1
#define  RXDESC_TYPE_PKTTYPE_PROTO_SCTP		2
#define  RXDESC_TYPE_PKTTYPE_PROTO_ICMP		3
#define  RXDESC_TYPE_PKTTYPE_PROTO_OTHERS	4
#define RXDESC_TYPE_PKTTYPE_VLAN	__BIT(9)
#define RXDESC_TYPE_PKTTYPE_VLAN_DOUBLE	__BIT(10)
#define RXDESC_TYPE_MAC_DMA_ERR		__BIT(12)
#define RXDESC_TYPE_RESERVED		__BITS(18,13)
#define RXDESC_TYPE_IPV4_CSUM_CHECKED	__BIT(19)	/* PKTTYPE_ETHER_IPV4 */
#define RXDESC_TYPE_TCPUDP_CSUM_CHECKED	__BIT(20)
#define RXDESC_TYPE_SPH			__BIT(21)
#define RXDESC_TYPE_HDR_LEN		__BITS(31,22)
	uint32_t rss_hash;
	uint16_t status;
#define RXDESC_STATUS_DD		__BIT(0)
#define RXDESC_STATUS_EOP		__BIT(1)
#define RXDESC_STATUS_MACERR		__BIT(2)
#define RXDESC_STATUS_IPV4_CSUM_NG	__BIT(3)
#define RXDESC_STATUS_TCPUDP_CSUM_ERROR	__BIT(4)
#define RXDESC_STATUS_TCPUDP_CSUM_OK	__BIT(5)

#define RXDESC_STATUS_STAT		__BITS(2,5)
#define RXDESC_STATUS_ESTAT		__BITS(6,11)
#define RXDESC_STATUS_RSC_CNT		__BITS(12,15)
	uint16_t pkt_len;
	uint16_t next_desc_ptr;
	uint16_t vlan;
} __packed;

typedef union aq_rx_desc {
	struct aq_rx_desc_read read;
	struct aq_rx_desc_wb wb;
} __packed aq_rx_desc_t;

typedef struct aq_tx_desc {
	uint64_t buf_addr;
	uint32_t ctl1;
#define AQ_TXDESC_CTL1_TYPE_MASK	0x00000003
#define AQ_TXDESC_CTL1_TYPE_TXD		0x00000001
#define AQ_TXDESC_CTL1_TYPE_TXC		0x00000002
#define AQ_TXDESC_CTL1_BLEN		__BITS(19,4)	/* TXD */
#define AQ_TXDESC_CTL1_DD		__BIT(20)	/* TXD */
#define AQ_TXDESC_CTL1_EOP		__BIT(21)	/* TXD */
#define AQ_TXDESC_CTL1_CMD_VLAN		__BIT(22)	/* TXD */
#define AQ_TXDESC_CTL1_CMD_FCS		__BIT(23)	/* TXD */
#define AQ_TXDESC_CTL1_CMD_IP4CSUM	__BIT(24)	/* TXD */
#define AQ_TXDESC_CTL1_CMD_L4CSUM	__BIT(25)	/* TXD */
#define AQ_TXDESC_CTL1_CMD_LSO		__BIT(26)	/* TXD */
#define AQ_TXDESC_CTL1_CMD_WB		__BIT(27)	/* TXD */
#define AQ_TXDESC_CTL1_CMD_VXLAN	__BIT(28)	/* TXD */
#define AQ_TXDESC_CTL1_VID		__BITS(15,4)	/* TXC */
#define AQ_TXDESC_CTL1_LSO_IPV6		__BIT(21)	/* TXC */
#define AQ_TXDESC_CTL1_LSO_TCP		__BIT(22)	/* TXC */
	uint32_t ctl2;
#define AQ_TXDESC_CTL2_LEN		__BITS(31,14)
#define AQ_TXDESC_CTL2_CTX_EN		__BIT(13)
#define AQ_TXDESC_CTL2_CTX_IDX		__BIT(12)
} __packed aq_tx_desc_t;

struct aq_txring {
	struct aq_softc *txr_sc;
	int txr_index;
	kmutex_t txr_mutex;
	bool txr_active;

	pcq_t *txr_pcq;
	void *txr_softint;

	aq_tx_desc_t *txr_txdesc;	/* aq_tx_desc_t[AQ_TXD_NUM] */
	bus_dmamap_t txr_txdesc_dmamap;
	bus_dma_segment_t txr_txdesc_seg[1];
	bus_size_t txr_txdesc_size;

	struct {
		struct mbuf *m;
		bus_dmamap_t dmamap;
	} txr_mbufs[AQ_TXD_NUM];
	unsigned int txr_prodidx;
	unsigned int txr_considx;
	int txr_nfree;

	/* counters */
	uint64_t txr_opackets;
	uint64_t txr_obytes;
	uint64_t txr_omcasts;
	uint64_t txr_oerrors;
};

struct aq_rxring {
	struct aq_softc *rxr_sc;
	int rxr_index;
	kmutex_t rxr_mutex;
	bool rxr_active;

	aq_rx_desc_t *rxr_rxdesc;	/* aq_rx_desc_t[AQ_RXD_NUM] */
	bus_dmamap_t rxr_rxdesc_dmamap;
	bus_dma_segment_t rxr_rxdesc_seg[1];
	bus_size_t rxr_rxdesc_size;
	struct {
		struct mbuf *m;
		bus_dmamap_t dmamap;
	} rxr_mbufs[AQ_RXD_NUM];
	unsigned int rxr_readidx;

	/* counters */
	uint64_t rxr_ipackets;
	uint64_t rxr_ibytes;
	uint64_t rxr_ierrors;
	uint64_t rxr_iqdrops;
};

struct aq_queue {
	struct aq_softc *sc;
	struct aq_txring txring;
	struct aq_rxring rxring;
};

struct aq_softc;
struct aq_firmware_ops {
	int (*reset)(struct aq_softc *);
	int (*set_mode)(struct aq_softc *, aq_hw_fw_mpi_state_t,
	    aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
	int (*get_mode)(struct aq_softc *, aq_hw_fw_mpi_state_t *,
	    aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
	int (*get_stats)(struct aq_softc *, aq_hw_stats_s_t *);
#if NSYSMON_ENVSYS > 0
	int (*get_temperature)(struct aq_softc *, uint32_t *);
#endif
};

#ifdef AQ_EVENT_COUNTERS
#define AQ_EVCNT_DECL(name)						\
	char sc_evcount_##name##_name[32];				\
	struct evcnt sc_evcount_##name##_ev;
#define AQ_EVCNT_ATTACH(sc, name, desc, evtype)				\
	do {								\
		snprintf((sc)->sc_evcount_##name##_name,		\
		    sizeof((sc)->sc_evcount_##name##_name),		\
		    "%s", desc);					\
		evcnt_attach_dynamic(&(sc)->sc_evcount_##name##_ev,	\
		    (evtype), NULL, device_xname((sc)->sc_dev),		\
		    (sc)->sc_evcount_##name##_name);			\
	} while (/*CONSTCOND*/0)
#define AQ_EVCNT_ATTACH_MISC(sc, name, desc)				\
	AQ_EVCNT_ATTACH(sc, name, desc, EVCNT_TYPE_MISC)
#define AQ_EVCNT_DETACH(sc, name)					\
	evcnt_detach(&(sc)->sc_evcount_##name##_ev)
#define AQ_EVCNT_ADD(sc, name, val)					\
	((sc)->sc_evcount_##name##_ev.ev_count += (val))
#endif /* AQ_EVENT_COUNTERS */

#define AQ_LOCK(sc)		mutex_enter(&(sc)->sc_mutex);
#define AQ_UNLOCK(sc)		mutex_exit(&(sc)->sc_mutex);

/* lock for FW2X_MPI_{CONTROL,STATE]_REG read-modify-write */
#define AQ_MPI_LOCK(sc)		mutex_enter(&(sc)->sc_mpi_mutex);
#define AQ_MPI_UNLOCK(sc)	mutex_exit(&(sc)->sc_mpi_mutex);


struct aq_softc {
	device_t sc_dev;

	bus_space_tag_t sc_iot;
	bus_space_handle_t sc_ioh;
	bus_size_t sc_iosize;
	bus_dma_tag_t sc_dmat;;

	void *sc_ihs[AQ_NINTR_MAX];
	pci_intr_handle_t *sc_intrs;

	int sc_tx_irq[AQ_RSSQUEUE_MAX];
	int sc_rx_irq[AQ_RSSQUEUE_MAX];
	int sc_linkstat_irq;
	bool sc_use_txrx_independent_intr;
	bool sc_poll_linkstat;
	bool sc_detect_linkstat;

#if NSYSMON_ENVSYS > 0
	struct sysmon_envsys *sc_sme;
	envsys_data_t sc_sensor_temp;
#endif

	callout_t sc_tick_ch;

	int sc_nintrs;
	bool sc_msix;

	struct aq_queue sc_queue[AQ_RSSQUEUE_MAX];
	int sc_nqueues;

	pci_chipset_tag_t sc_pc;
	pcitag_t sc_pcitag;
	uint16_t sc_product;
	uint16_t sc_revision;

	kmutex_t sc_mutex;
	kmutex_t sc_mpi_mutex;

	struct aq_firmware_ops *sc_fw_ops;
	uint64_t sc_fw_caps;
	enum aq_media_type sc_media_type;
	aq_link_speed_t sc_available_rates;

	aq_link_speed_t sc_link_rate;
	aq_link_fc_t sc_link_fc;
	aq_link_eee_t sc_link_eee;

	uint32_t sc_fw_version;
#define FW_VERSION_MAJOR(sc)	(((sc)->sc_fw_version >> 24) & 0xff)
#define FW_VERSION_MINOR(sc)	(((sc)->sc_fw_version >> 16) & 0xff)
#define FW_VERSION_BUILD(sc)	((sc)->sc_fw_version & 0xffff)
	uint32_t sc_features;
#define FEATURES_MIPS		0x00000001
#define FEATURES_TPO2		0x00000002
#define FEATURES_RPF2		0x00000004
#define FEATURES_MPI_AQ		0x00000008
#define FEATURES_REV_A0		0x10000000
#define FEATURES_REV_A		(FEATURES_REV_A0)
#define FEATURES_REV_B0		0x20000000
#define FEATURES_REV_B1		0x40000000
#define FEATURES_REV_B		(FEATURES_REV_B0|FEATURES_REV_B1)
	uint32_t sc_mbox_addr;

	bool sc_rbl_enabled;
	bool sc_fast_start_enabled;
	bool sc_flash_present;

	bool sc_intr_moderation_enable;
	bool sc_rss_enable;

	struct ethercom sc_ethercom;
	struct ether_addr sc_enaddr;
	struct ifmedia sc_media;
	int sc_ec_capenable;		/* last ec_capenable */
	unsigned short sc_if_flags;	/* last if_flags */

#ifdef AQ_EVENT_COUNTERS
	aq_hw_stats_s_t sc_statistics[2];
	int sc_statistics_idx;
	bool sc_poll_statistics;

	AQ_EVCNT_DECL(uprc);
	AQ_EVCNT_DECL(mprc);
	AQ_EVCNT_DECL(bprc);
	AQ_EVCNT_DECL(erpt);
	AQ_EVCNT_DECL(uptc);
	AQ_EVCNT_DECL(mptc);
	AQ_EVCNT_DECL(bptc);
	AQ_EVCNT_DECL(erpr);
	AQ_EVCNT_DECL(mbtc);
	AQ_EVCNT_DECL(bbtc);
	AQ_EVCNT_DECL(mbrc);
	AQ_EVCNT_DECL(bbrc);
	AQ_EVCNT_DECL(ubrc);
	AQ_EVCNT_DECL(ubtc);
	AQ_EVCNT_DECL(ptc);
	AQ_EVCNT_DECL(prc);
	AQ_EVCNT_DECL(dpc);
	AQ_EVCNT_DECL(cprc);
#endif
};

static int aq_match(device_t, cfdata_t, void *);
static void aq_attach(device_t, device_t, void *);
static int aq_detach(device_t, int);

static int aq_setup_msix(struct aq_softc *, struct pci_attach_args *, int,
    bool, bool);
static int aq_setup_legacy(struct aq_softc *, struct pci_attach_args *,
    pci_intr_type_t);
static int aq_establish_msix_intr(struct aq_softc *, bool, bool);

static int aq_ifmedia_change(struct ifnet * const);
static void aq_ifmedia_status(struct ifnet * const, struct ifmediareq *);
static int aq_vlan_cb(struct ethercom *ec, uint16_t vid, bool set);
static int aq_ifflags_cb(struct ethercom *);
static int aq_init(struct ifnet *);
static void aq_send_common_locked(struct ifnet *, struct aq_softc *,
    struct aq_txring *, bool);
static int aq_transmit(struct ifnet *, struct mbuf *);
static void aq_deferred_transmit(void *);
static void aq_start(struct ifnet *);
static void aq_stop(struct ifnet *, int);
static void aq_watchdog(struct ifnet *);
static int aq_ioctl(struct ifnet *, unsigned long, void *);

static int aq_txrx_rings_alloc(struct aq_softc *);
static void aq_txrx_rings_free(struct aq_softc *);
static int aq_tx_pcq_alloc(struct aq_softc *, struct aq_txring *);
static void aq_tx_pcq_free(struct aq_softc *, struct aq_txring *);

static void aq_initmedia(struct aq_softc *);
static void aq_enable_intr(struct aq_softc *, bool, bool);

#if NSYSMON_ENVSYS > 0
static void aq_temp_refresh(struct sysmon_envsys *, envsys_data_t *);
#endif
static void aq_tick(void *);
static int aq_legacy_intr(void *);
static int aq_link_intr(void *);
static int aq_txrx_intr(void *);
static int aq_tx_intr(void *);
static int aq_rx_intr(void *);

static int aq_set_linkmode(struct aq_softc *, aq_link_speed_t, aq_link_fc_t,
    aq_link_eee_t);
static int aq_get_linkmode(struct aq_softc *, aq_link_speed_t *, aq_link_fc_t *,
    aq_link_eee_t *);

static int aq_fw_reset(struct aq_softc *);
static int aq_fw_version_init(struct aq_softc *);
static int aq_hw_init(struct aq_softc *);
static int aq_hw_init_ucp(struct aq_softc *);
static int aq_hw_reset(struct aq_softc *);
static int aq_fw_downld_dwords(struct aq_softc *, uint32_t, uint32_t *,
    uint32_t);
static int aq_get_mac_addr(struct aq_softc *);
static int aq_init_rss(struct aq_softc *);
static int aq_set_capability(struct aq_softc *);

static int fw1x_reset(struct aq_softc *);
static int fw1x_set_mode(struct aq_softc *, aq_hw_fw_mpi_state_t,
    aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
static int fw1x_get_mode(struct aq_softc *, aq_hw_fw_mpi_state_t *,
    aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
static int fw1x_get_stats(struct aq_softc *, aq_hw_stats_s_t *);

static int fw2x_reset(struct aq_softc *);
static int fw2x_set_mode(struct aq_softc *, aq_hw_fw_mpi_state_t,
    aq_link_speed_t, aq_link_fc_t, aq_link_eee_t);
static int fw2x_get_mode(struct aq_softc *, aq_hw_fw_mpi_state_t *,
    aq_link_speed_t *, aq_link_fc_t *, aq_link_eee_t *);
static int fw2x_get_stats(struct aq_softc *, aq_hw_stats_s_t *);
#if NSYSMON_ENVSYS > 0
static int fw2x_get_temperature(struct aq_softc *, uint32_t *);
#endif

static struct aq_firmware_ops aq_fw1x_ops = {
	.reset = fw1x_reset,
	.set_mode = fw1x_set_mode,
	.get_mode = fw1x_get_mode,
	.get_stats = fw1x_get_stats,
#if NSYSMON_ENVSYS > 0
	.get_temperature = NULL
#endif
};

static struct aq_firmware_ops aq_fw2x_ops = {
	.reset = fw2x_reset,
	.set_mode = fw2x_set_mode,
	.get_mode = fw2x_get_mode,
	.get_stats = fw2x_get_stats,
#if NSYSMON_ENVSYS > 0
	.get_temperature = fw2x_get_temperature
#endif
};

CFATTACH_DECL3_NEW(aq, sizeof(struct aq_softc),
    aq_match, aq_attach, aq_detach, NULL, NULL, NULL, DVF_DETACH_SHUTDOWN);

static const struct aq_product {
	pci_vendor_id_t aq_vendor;
	pci_product_id_t aq_product;
	const char *aq_name;
	enum aq_media_type aq_media_type;
	aq_link_speed_t aq_available_rates;
} aq_products[] = {
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC100,
	  "Aquantia AQC100 10 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_FIBRE, AQ_LINK_ALL
	},
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC107,
	  "Aquantia AQC107 10 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
	},
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC108,
	  "Aquantia AQC108 5 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
	},
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC109,
	  "Aquantia AQC109 2.5 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
	},
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC111,
	  "Aquantia AQC111 5 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
	},
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC112,
	  "Aquantia AQC112 2.5 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
	},
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC100S,
	  "Aquantia AQC100S 10 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_FIBRE, AQ_LINK_ALL
	},
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC107S,
	  "Aquantia AQC107S 10 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
	},
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC108S,
	  "Aquantia AQC108S 5 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
	},
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC109S,
	  "Aquantia AQC109S 2.5 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
	},
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC111S,
	  "Aquantia AQC111S 5 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
	},
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_AQC112S,
	  "Aquantia AQC112S 2.5 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
	},
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D100,
	  "Aquantia D100 10 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_FIBRE, AQ_LINK_ALL
	},
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D107,
	  "Aquantia D107 10 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_TP, AQ_LINK_ALL
	},
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D108,
	  "Aquantia D108 5 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5 | AQ_LINK_5G
	},
	{ PCI_VENDOR_AQUANTIA, PCI_PRODUCT_AQUANTIA_D109,
	  "Aquantia D109 2.5 Gigabit Network Adapter",
	  AQ_MEDIA_TYPE_TP, AQ_LINK_100M | AQ_LINK_1G | AQ_LINK_2G5
	}
};

static const struct aq_product *
aq_lookup(const struct pci_attach_args *pa)
{
	unsigned int i;

	for (i = 0; i < __arraycount(aq_products); i++) {
		if (PCI_VENDOR(pa->pa_id)  == aq_products[i].aq_vendor &&
		    PCI_PRODUCT(pa->pa_id) == aq_products[i].aq_product)
			return &aq_products[i];
	}
	return NULL;
}

static int
aq_match(device_t parent, cfdata_t cf, void *aux)
{
	struct pci_attach_args *pa = aux;

	if (aq_lookup(pa) != NULL)
		return 1;

	return 0;
}

static void
aq_attach(device_t parent, device_t self, void *aux)
{
	struct aq_softc *sc = device_private(self);
	struct pci_attach_args *pa = aux;
	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
	pci_chipset_tag_t pc;
	pcitag_t tag;
	pcireg_t command, memtype, bar;
	const struct aq_product *aqp;
	int error;

	sc->sc_dev = self;
	mutex_init(&sc->sc_mutex, MUTEX_DEFAULT, IPL_NET);
	mutex_init(&sc->sc_mpi_mutex, MUTEX_DEFAULT, IPL_NET);

	sc->sc_pc = pc = pa->pa_pc;
	sc->sc_pcitag = tag = pa->pa_tag;
	sc->sc_dmat = pci_dma64_available(pa) ? pa->pa_dmat64 : pa->pa_dmat;

	command = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
	command |= PCI_COMMAND_MASTER_ENABLE;
	pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG, command);

	sc->sc_product = PCI_PRODUCT(pa->pa_id);
	sc->sc_revision = PCI_REVISION(pa->pa_class);

	aqp = aq_lookup(pa);
	KASSERT(aqp != NULL);

	pci_aprint_devinfo_fancy(pa, "Ethernet controller", aqp->aq_name, 1);

	bar = pci_conf_read(pc, tag, PCI_BAR(0));
	if ((PCI_MAPREG_MEM_ADDR(bar) == 0) ||
	    (PCI_MAPREG_TYPE(bar) != PCI_MAPREG_TYPE_MEM)) {
		aprint_error_dev(sc->sc_dev, "wrong BAR type\n");
		return;
	}
	memtype = pci_mapreg_type(pc, tag, PCI_BAR(0));
	if (pci_mapreg_map(pa, PCI_BAR(0), memtype, 0, &sc->sc_iot, &sc->sc_ioh,
	    NULL, &sc->sc_iosize) != 0) {
		aprint_error_dev(sc->sc_dev, "unable to map register\n");
		return;
	}

	sc->sc_nqueues = MIN(ncpu, AQ_RSSQUEUE_MAX);

	/* max queue num is 8, and must be 2^n */
	if (ncpu >= 8)
		sc->sc_nqueues = 8;
	else if (ncpu >= 4)
		sc->sc_nqueues = 4;
	else if (ncpu >= 2)
		sc->sc_nqueues = 2;
	else
		sc->sc_nqueues = 1;

	int msixcount = pci_msix_count(pa->pa_pc, pa->pa_tag);
#ifndef CONFIG_NO_TXRX_INDEPENDENT
	if (msixcount >= (sc->sc_nqueues * 2 + 1)) {
		/* TX intrs + RX intrs + LINKSTAT intrs */
		sc->sc_use_txrx_independent_intr = true;
		sc->sc_poll_linkstat = false;
		sc->sc_msix = true;
	} else if (msixcount >= (sc->sc_nqueues * 2)) {
		/* TX intrs + RX intrs */
		sc->sc_use_txrx_independent_intr = true;
		sc->sc_poll_linkstat = true;
		sc->sc_msix = true;
	} else
#endif
	if (msixcount >= (sc->sc_nqueues + 1)) {
		/* TX/RX intrs LINKSTAT intrs */
		sc->sc_use_txrx_independent_intr = false;
		sc->sc_poll_linkstat = false;
		sc->sc_msix = true;
	} else if (msixcount >= sc->sc_nqueues) {
		/* TX/RX intrs */
		sc->sc_use_txrx_independent_intr = false;
		sc->sc_poll_linkstat = true;
		sc->sc_msix = true;
	} else {
		/* giving up using MSI-X */
		sc->sc_msix = false;
	}

	/* XXX: on FIBRE, linkstat interrupt does not occur on boot? */
	if (aqp->aq_media_type == AQ_MEDIA_TYPE_FIBRE)
		sc->sc_poll_linkstat = true;

#ifdef AQ_FORCE_POLL_LINKSTAT
	sc->sc_poll_linkstat = true;
#endif

	aprint_debug_dev(sc->sc_dev,
	    "ncpu=%d, pci_msix_count=%d."
	    " allocate %d interrupts for %d%s queues%s\n",
	    ncpu, msixcount,
	    (sc->sc_use_txrx_independent_intr ?
	    (sc->sc_nqueues * 2) : sc->sc_nqueues) +
	    (sc->sc_poll_linkstat ? 0 : 1),
	    sc->sc_nqueues,
	    sc->sc_use_txrx_independent_intr ? "*2" : "",
	    sc->sc_poll_linkstat ? "" : ", and link status");

	if (sc->sc_msix)
		error = aq_setup_msix(sc, pa, sc->sc_nqueues,
		    sc->sc_use_txrx_independent_intr, !sc->sc_poll_linkstat);
	else
		error = ENODEV;

	if (error != 0) {
		/* if MSI-X failed, fallback to MSI with single queue */
		sc->sc_use_txrx_independent_intr = false;
		sc->sc_poll_linkstat = false;
		sc->sc_msix = false;
		sc->sc_nqueues = 1;
		error = aq_setup_legacy(sc, pa, PCI_INTR_TYPE_MSI);
	}
	if (error != 0) {
		/* if MSI failed, fallback to INTx */
		error = aq_setup_legacy(sc, pa, PCI_INTR_TYPE_INTX);
	}
	if (error != 0)
		return;

	callout_init(&sc->sc_tick_ch, 0);
	callout_setfunc(&sc->sc_tick_ch, aq_tick, sc);

	sc->sc_intr_moderation_enable = CONFIG_INTR_MODERATION_ENABLE;

	if (sc->sc_msix && (sc->sc_nqueues > 1))
		sc->sc_rss_enable = true;
	else
		sc->sc_rss_enable = false;

	error = aq_txrx_rings_alloc(sc);
	if (error != 0)
		goto attach_failure;

	error = aq_fw_reset(sc);
	if (error != 0)
		goto attach_failure;

	error = aq_fw_version_init(sc);
	if (error != 0)
		goto attach_failure;

	error = aq_hw_init_ucp(sc);
	if (error < 0)
		goto attach_failure;

	KASSERT(sc->sc_mbox_addr != 0);
	error = aq_hw_reset(sc);
	if (error != 0)
		goto attach_failure;

	aq_get_mac_addr(sc);
	aq_init_rss(sc);

	error = aq_hw_init(sc);	/* initialize and interrupts */
	if (error != 0)
		goto attach_failure;

	sc->sc_media_type = aqp->aq_media_type;
	sc->sc_available_rates = aqp->aq_available_rates;

	sc->sc_ethercom.ec_ifmedia = &sc->sc_media;
	ifmedia_init(&sc->sc_media, IFM_IMASK,
	    aq_ifmedia_change, aq_ifmedia_status);
	aq_initmedia(sc);

	strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
	ifp->if_softc = sc;
	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
	ifp->if_baudrate = IF_Gbps(10);
	ifp->if_init = aq_init;
	ifp->if_ioctl = aq_ioctl;
	if (sc->sc_msix && (sc->sc_nqueues > 1))
		ifp->if_transmit = aq_transmit;
	ifp->if_start = aq_start;
	ifp->if_stop = aq_stop;
	ifp->if_watchdog = aq_watchdog;
	IFQ_SET_READY(&ifp->if_snd);

	/* initialize capabilities */
	sc->sc_ethercom.ec_capabilities = 0;
	sc->sc_ethercom.ec_capenable = 0;
#if notyet
	/* TODO */
	sc->sc_ethercom.ec_capabilities |= ETHERCAP_EEE;
#endif
	sc->sc_ethercom.ec_capabilities |=
	    ETHERCAP_JUMBO_MTU |
	    ETHERCAP_VLAN_MTU |
	    ETHERCAP_VLAN_HWTAGGING |
	    ETHERCAP_VLAN_HWFILTER;
	sc->sc_ethercom.ec_capenable |=
	    ETHERCAP_VLAN_HWTAGGING |
	    ETHERCAP_VLAN_HWFILTER;

	ifp->if_capabilities = 0;
	ifp->if_capenable = 0;
#ifdef CONFIG_LRO_SUPPORT
	ifp->if_capabilities |= IFCAP_LRO;
	ifp->if_capenable |= IFCAP_LRO;
#endif
#if notyet
	/* TSO */
	ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;
#endif

#if notyet
	/*
	 * XXX:
	 *   Rx L4 CSUM doesn't work well for fragment packet.
	 *   aq marks 'CHEDKED' and 'BAD' for them.
	 *   we need to ignore (clear) hw-csum flags if the packet is fragmented
	 *
	 *   TODO: test with LRO enabled
	 */
	ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_TCPv6_Rx;
	ifp->if_capabilities |= IFCAP_CSUM_UDPv4_Rx | IFCAP_CSUM_UDPv6_Rx;
#endif
	/* TX hardware checksum offloadding */
	ifp->if_capabilities |= IFCAP_CSUM_IPv4_Tx;
	ifp->if_capabilities |= IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv6_Tx;
	ifp->if_capabilities |= IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv6_Tx;
	/* RX hardware checksum offloadding */
	ifp->if_capabilities |= IFCAP_CSUM_IPv4_Rx;

	if_attach(ifp);
	if_deferred_start_init(ifp, NULL);
	ether_ifattach(ifp, sc->sc_enaddr.ether_addr_octet);
	ether_set_vlan_cb(&sc->sc_ethercom, aq_vlan_cb);
	ether_set_ifflags_cb(&sc->sc_ethercom, aq_ifflags_cb);

	aq_enable_intr(sc, true, false);	/* only intr about link */

	/* update media */
	aq_ifmedia_change(ifp);

#if NSYSMON_ENVSYS > 0
	/* temperature monitoring */
	if (sc->sc_fw_ops != NULL && sc->sc_fw_ops->get_temperature != NULL &&
	    (sc->sc_fw_caps & FW2X_CTRL_TEMPERATURE) != 0) {

		sc->sc_sme = sysmon_envsys_create();
		sc->sc_sme->sme_name = device_xname(self);
		sc->sc_sme->sme_cookie = sc;
		sc->sc_sme->sme_flags = 0;
		sc->sc_sme->sme_refresh = aq_temp_refresh;
		sc->sc_sensor_temp.units = ENVSYS_STEMP;
		sc->sc_sensor_temp.state = ENVSYS_SINVALID;
		snprintf(sc->sc_sensor_temp.desc, ENVSYS_DESCLEN, "PHY");

		sysmon_envsys_sensor_attach(sc->sc_sme, &sc->sc_sensor_temp);
		sysmon_envsys_register(sc->sc_sme);

		/*
		 * for unknown reasons, the first call of fw2x_get_temperature()
		 * will always fail (firmware matter?), so run once now.
		 */
		aq_temp_refresh(sc->sc_sme, &sc->sc_sensor_temp);
	}
#endif

#ifdef AQ_EVENT_COUNTERS
	/* get starting statistics values */
	if (sc->sc_fw_ops != NULL && sc->sc_fw_ops->get_stats != NULL &&
	    sc->sc_fw_ops->get_stats(sc, &sc->sc_statistics[0]) == 0) {
		sc->sc_poll_statistics = true;
	}

	AQ_EVCNT_ATTACH_MISC(sc, uprc, "RX unicast packet");
	AQ_EVCNT_ATTACH_MISC(sc, bprc, "RX broadcast packet");
	AQ_EVCNT_ATTACH_MISC(sc, mprc, "RX multicast packet");
	AQ_EVCNT_ATTACH_MISC(sc, erpr, "RX error packet");
	AQ_EVCNT_ATTACH_MISC(sc, ubrc, "RX unicast bytes");
	AQ_EVCNT_ATTACH_MISC(sc, bbrc, "RX broadcast bytes");
	AQ_EVCNT_ATTACH_MISC(sc, mbrc, "RX multicast bytes");
	AQ_EVCNT_ATTACH_MISC(sc, prc, "RX good packet");
	AQ_EVCNT_ATTACH_MISC(sc, uptc, "TX unicast packet");
	AQ_EVCNT_ATTACH_MISC(sc, bptc, "TX broadcast packet");
	AQ_EVCNT_ATTACH_MISC(sc, mptc, "TX multicast packet");
	AQ_EVCNT_ATTACH_MISC(sc, erpt, "TX error packet");
	AQ_EVCNT_ATTACH_MISC(sc, ubtc, "TX unicast bytes");
	AQ_EVCNT_ATTACH_MISC(sc, bbtc, "TX broadcast bytes");
	AQ_EVCNT_ATTACH_MISC(sc, mbtc, "TX multicast bytes");
	AQ_EVCNT_ATTACH_MISC(sc, ptc, "TX good packet");
	AQ_EVCNT_ATTACH_MISC(sc, dpc, "DMA drop packet");
	AQ_EVCNT_ATTACH_MISC(sc, cprc, "RX coalesced packet");
#endif

	return;

 attach_failure:
	aq_detach(self, 0);
}

static int
aq_detach(device_t self, int flags __unused)
{
	struct aq_softc *sc = device_private(self);
	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
	int i, s;

	if (sc->sc_iosize != 0) {
		if (ifp->if_softc != NULL) {
			s = splnet();
			aq_stop(ifp, 0);
			splx(s);
		}

		for (i = 0; i < AQ_NINTR_MAX; i++) {
			if (sc->sc_ihs[i] != NULL) {
				pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]);
				sc->sc_ihs[i] = NULL;
			}
		}
		if (sc->sc_nintrs > 0) {
			pci_intr_release(sc->sc_pc, sc->sc_intrs,
			    sc->sc_nintrs);
			sc->sc_intrs = NULL;
			sc->sc_nintrs = 0;
		}

		aq_txrx_rings_free(sc);

		if (ifp->if_softc != NULL) {
			ether_ifdetach(ifp);
			if_detach(ifp);
		}

		aprint_debug_dev(sc->sc_dev, "%s: bus_space_unmap\n", __func__);
		bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize);
		sc->sc_iosize = 0;
	}

	callout_stop(&sc->sc_tick_ch);

#if NSYSMON_ENVSYS > 0
	if (sc->sc_sme != NULL) {
		/* all sensors associated with this will also be detached */
		sysmon_envsys_unregister(sc->sc_sme);
		sc->sc_sme = NULL;
	}
#endif

#ifdef AQ_EVENT_COUNTERS
	AQ_EVCNT_DETACH(sc, uprc);
	AQ_EVCNT_DETACH(sc, mprc);
	AQ_EVCNT_DETACH(sc, bprc);
	AQ_EVCNT_DETACH(sc, erpt);
	AQ_EVCNT_DETACH(sc, uptc);
	AQ_EVCNT_DETACH(sc, mptc);
	AQ_EVCNT_DETACH(sc, bptc);
	AQ_EVCNT_DETACH(sc, erpr);
	AQ_EVCNT_DETACH(sc, mbtc);
	AQ_EVCNT_DETACH(sc, bbtc);
	AQ_EVCNT_DETACH(sc, mbrc);
	AQ_EVCNT_DETACH(sc, bbrc);
	AQ_EVCNT_DETACH(sc, ubrc);
	AQ_EVCNT_DETACH(sc, ubtc);
	AQ_EVCNT_DETACH(sc, ptc);
	AQ_EVCNT_DETACH(sc, prc);
	AQ_EVCNT_DETACH(sc, dpc);
	AQ_EVCNT_DETACH(sc, cprc);
#endif

	mutex_destroy(&sc->sc_mpi_mutex);
	mutex_destroy(&sc->sc_mutex);

	return 0;
}

static int
aq_establish_intr(struct aq_softc *sc, int intno, kcpuset_t *affinity,
    int (*func)(void *), void *arg, const char *xname)
{
	char intrbuf[PCI_INTRSTR_LEN];
	pci_chipset_tag_t pc = sc->sc_pc;
	void *vih;
	const char *intrstr = NULL;

	intrstr = pci_intr_string(pc, sc->sc_intrs[intno], intrbuf,
	    sizeof(intrbuf));

	pci_intr_setattr(pc, &sc->sc_intrs[intno], PCI_INTR_MPSAFE, true);

	vih = pci_intr_establish_xname(pc, sc->sc_intrs[intno],
	    IPL_NET, func, arg, xname);
	if (vih == NULL) {
		aprint_error_dev(sc->sc_dev,
		    "unable to establish MSI-X%s%s for %s\n",
		    intrstr ? " at " : "",
		    intrstr ? intrstr : "", xname);
		return EIO;
	}
	sc->sc_ihs[intno] = vih;

	if (affinity != NULL) {
		/* Round-robin affinity */
		kcpuset_zero(affinity);
		kcpuset_set(affinity, intno % ncpu);
		interrupt_distribute(vih, affinity, NULL);
	}

	return 0;
}

static int
aq_establish_msix_intr(struct aq_softc *sc, bool txrx_independent,
    bool linkintr)
{
	kcpuset_t *affinity;
	int error, intno, i;
	char intr_xname[INTRDEVNAMEBUF];

	kcpuset_create(&affinity, false);

	intno = 0;

	if (txrx_independent) {
		for (i = 0; i < sc->sc_nqueues; i++) {
			snprintf(intr_xname, sizeof(intr_xname), "%s RX%d",
			    device_xname(sc->sc_dev), i);
			sc->sc_rx_irq[i] = intno;
			error = aq_establish_intr(sc, intno++, affinity,
			   aq_rx_intr, &sc->sc_queue[i].rxring, intr_xname);
			if (error != 0)
				goto fail;
		}
		for (i = 0; i < sc->sc_nqueues; i++) {
			snprintf(intr_xname, sizeof(intr_xname), "%s TX%d",
			    device_xname(sc->sc_dev), i);
			sc->sc_tx_irq[i] = intno;
			error = aq_establish_intr(sc, intno++, affinity,
			    aq_tx_intr, &sc->sc_queue[i].txring, intr_xname);
			if (error != 0)
				goto fail;
		}
	} else {
		for (i = 0; i < sc->sc_nqueues; i++) {
			snprintf(intr_xname, sizeof(intr_xname), "%s TXRX%d",
			    device_xname(sc->sc_dev), i);
			sc->sc_rx_irq[i] = intno;
			sc->sc_tx_irq[i] = intno;
			error = aq_establish_intr(sc, intno++, affinity,
			    aq_txrx_intr, &sc->sc_queue[i], intr_xname);
			if (error != 0)
				goto fail;
		}
	}

	if (linkintr) {
		snprintf(intr_xname, sizeof(intr_xname), "%s LINK",
		    device_xname(sc->sc_dev));
		sc->sc_linkstat_irq = intno;
		error = aq_establish_intr(sc, intno++, affinity,
		    aq_link_intr, sc, intr_xname);
		if (error != 0)
			goto fail;
	}

	kcpuset_destroy(affinity);
	return 0;

 fail:
	for (i = 0; i < AQ_NINTR_MAX; i++) {
		if (sc->sc_ihs[i] != NULL) {
			pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]);
			sc->sc_ihs[i] = NULL;
		}
	}

	kcpuset_destroy(affinity);
	return ENOMEM;
}

static int
aq_setup_msix(struct aq_softc *sc, struct pci_attach_args *pa, int nqueue,
    bool txrx_independent, bool linkintr)
{
	int error, nintr;

	if (txrx_independent)
		nintr = nqueue * 2;
	else
		nintr = nqueue;

	if (linkintr)
		nintr++;

	error = pci_msix_alloc_exact(pa, &sc->sc_intrs, nintr);
	if (error != 0) {
		aprint_error_dev(sc->sc_dev,
		    "failed to allocate MSI-X interrupts\n");
		goto fail;
	}

	error = aq_establish_msix_intr(sc, txrx_independent, linkintr);
	if (error == 0) {
		sc->sc_nintrs = nintr;
	} else {
		pci_intr_release(sc->sc_pc, sc->sc_intrs, nintr);
		sc->sc_nintrs = 0;
	}
 fail:
	return error;

}

static int
aq_setup_legacy(struct aq_softc *sc, struct pci_attach_args *pa,
    pci_intr_type_t inttype)
{
	int counts[PCI_INTR_TYPE_SIZE];
	int error, nintr;

	nintr = 1;

	memset(counts, 0, sizeof(counts));
	counts[inttype] = nintr;

	error = pci_intr_alloc(pa, &sc->sc_intrs, counts, inttype);
	if (error != 0) {
		aprint_error_dev(sc->sc_dev,
		    "failed to allocate%s interrupts\n",
		    (inttype == PCI_INTR_TYPE_MSI) ? " MSI" : "");
		return error;
	}
	error = aq_establish_intr(sc, 0, NULL, aq_legacy_intr, sc,
	    device_xname(sc->sc_dev));
	if (error == 0) {
		sc->sc_nintrs = nintr;
	} else {
		pci_intr_release(sc->sc_pc, sc->sc_intrs, nintr);
		sc->sc_nintrs = 0;
	}
	return error;
}

static void
global_software_reset(struct aq_softc *sc)
{
	uint32_t v;

	AQ_WRITE_REG_BIT(sc, RX_SYSCONTROL_REG, RX_SYSCONTROL_RESET_DIS, 0);
	AQ_WRITE_REG_BIT(sc, TX_SYSCONTROL_REG, TX_SYSCONTROL_RESET_DIS, 0);
	AQ_WRITE_REG_BIT(sc, FW_MPI_RESETCTRL_REG,
	    FW_MPI_RESETCTRL_RESET_DIS, 0);

	v = AQ_READ_REG(sc, AQ_FW_SOFTRESET_REG);
	v &= ~AQ_FW_SOFTRESET_DIS;
	v |= AQ_FW_SOFTRESET_RESET;
	AQ_WRITE_REG(sc, AQ_FW_SOFTRESET_REG, v);
}

static int
mac_soft_reset_rbl(struct aq_softc *sc, aq_fw_bootloader_mode_t *mode)
{
	int timo;

	aprint_debug_dev(sc->sc_dev, "RBL> MAC reset STARTED!\n");

	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e1);
	AQ_WRITE_REG(sc, AQ_FW_GLB_CPU_SEM_REG(0), 1);
	AQ_WRITE_REG(sc, AQ_MBOXIF_POWER_GATING_CONTROL_REG, 0);

	/* MAC FW will reload PHY FW if 1E.1000.3 was cleaned - #undone */
	AQ_WRITE_REG(sc, FW_BOOT_EXIT_CODE_REG, RBL_STATUS_DEAD);

	global_software_reset(sc);

	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e0);

	/* Wait for RBL to finish boot process. */
#define RBL_TIMEOUT_MS	10000
	uint16_t rbl_status;
	for (timo = RBL_TIMEOUT_MS; timo > 0; timo--) {
		rbl_status = AQ_READ_REG(sc, FW_BOOT_EXIT_CODE_REG) & 0xffff;
		if (rbl_status != 0 && rbl_status != RBL_STATUS_DEAD)
			break;
		msec_delay(1);
	}
	if (timo <= 0) {
		aprint_error_dev(sc->sc_dev,
		    "RBL> RBL restart failed: timeout\n");
		return EBUSY;
	}
	switch (rbl_status) {
	case RBL_STATUS_SUCCESS:
		if (mode != NULL)
			*mode = FW_BOOT_MODE_RBL_FLASH;
		aprint_debug_dev(sc->sc_dev, "RBL> reset complete! [Flash]\n");
		break;
	case RBL_STATUS_HOST_BOOT:
		if (mode != NULL)
			*mode = FW_BOOT_MODE_RBL_HOST_BOOTLOAD;
		aprint_debug_dev(sc->sc_dev,
		    "RBL> reset complete! [Host Bootload]\n");
		break;
	case RBL_STATUS_FAILURE:
	default:
		aprint_error_dev(sc->sc_dev,
		    "unknown RBL status 0x%x\n", rbl_status);
		return EBUSY;
	}

	return 0;
}

static int
mac_soft_reset_flb(struct aq_softc *sc)
{
	uint32_t v;
	int timo;

	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x40e1);
	/*
	 * Let Felicity hardware to complete SMBUS transaction before
	 * Global software reset.
	 */
	msec_delay(50);

	/*
	 * If SPI burst transaction was interrupted(before running the script),
	 * global software reset may not clear SPI interface.
	 * Clean it up manually before global reset.
	 */
	AQ_WRITE_REG(sc, AQ_GLB_NVR_PROVISIONING2_REG, 0x00a0);
	AQ_WRITE_REG(sc, AQ_GLB_NVR_INTERFACE1_REG, 0x009f);
	AQ_WRITE_REG(sc, AQ_GLB_NVR_INTERFACE1_REG, 0x809f);
	msec_delay(50);

	v = AQ_READ_REG(sc, AQ_FW_SOFTRESET_REG);
	v &= ~AQ_FW_SOFTRESET_DIS;
	v |= AQ_FW_SOFTRESET_RESET;
	AQ_WRITE_REG(sc, AQ_FW_SOFTRESET_REG, v);

	/* Kickstart. */
	AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x80e0);
	AQ_WRITE_REG(sc, AQ_MBOXIF_POWER_GATING_CONTROL_REG, 0);
	if (!sc->sc_fast_start_enabled)
		AQ_WRITE_REG(sc, AQ_GLB_GENERAL_PROVISIONING9_REG, 1);

	/*
	 * For the case SPI burst transaction was interrupted (by MCP reset
	 * above), wait until it is completed by hardware.
	 */
	msec_delay(50);

	/* MAC Kickstart */
	if (!sc->sc_fast_start_enabled) {
		AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x180e0);

		uint32_t flb_status;
		for (timo = 0; timo < 1000; timo++) {
			flb_status = AQ_READ_REG(sc,
			    FW_MPI_DAISY_CHAIN_STATUS_REG) & 0x10;
			if (flb_status != 0)
				break;
			msec_delay(1);
		}
		if (flb_status == 0) {
			aprint_error_dev(sc->sc_dev,
			    "FLB> MAC kickstart failed: timed out\n");
			return ETIMEDOUT;
		}
		aprint_debug_dev(sc->sc_dev,
		    "FLB> MAC kickstart done, %d ms\n", timo);
		/* FW reset */
		AQ_WRITE_REG(sc, AQ_FW_GLB_CTL2_REG, 0x80e0);
		/*
		 * Let Felicity hardware complete SMBUS transaction before
		 * Global software reset.
		 */
		msec_delay(50);
		sc->sc_fast_start_enabled = true;
	}
	AQ_WRITE_REG(sc, AQ_FW_GLB_CPU_SEM_REG(0), 1);

	/* PHY Kickstart: #undone */
	global_software_reset(sc);

	for (timo = 0; timo < 1000; timo++) {
		if (AQ_READ_REG(sc, AQ_FW_VERSION_REG) != 0)
			break;
		msec_delay(10);
	}
	if (timo >= 1000) {
		aprint_error_dev(sc->sc_dev, "FLB> Global Soft Reset failed\n");
		return ETIMEDOUT;
	}
	aprint_debug_dev(sc->sc_dev, "FLB> F/W restart: %d ms\n", timo * 10);
	return 0;

}

static int
mac_soft_reset(struct aq_softc *sc, aq_fw_bootloader_mode_t *mode)
{
	if (sc->sc_rbl_enabled)
		return mac_soft_reset_rbl(sc, mode);

	if (mode != NULL)
		*mode = FW_BOOT_MODE_FLB;
	return mac_soft_reset_flb(sc);
}

static int
aq_fw_read_version(struct aq_softc *sc)
{
	int i, error = EBUSY;
#define MAC_FW_START_TIMEOUT_MS	10000
	for (i = 0; i < MAC_FW_START_TIMEOUT_MS; i++) {
		sc->sc_fw_version = AQ_READ_REG(sc, AQ_FW_VERSION_REG);
		if (sc->sc_fw_version != 0) {
			error = 0;
			break;
		}
		delay(1000);
	}
	return error;
}

static int
aq_fw_reset(struct aq_softc *sc)
{
	uint32_t ver, v, bootExitCode;
	int i, error;

	ver = AQ_READ_REG(sc, AQ_FW_VERSION_REG);

	for (i = 1000; i > 0; i--) {
		v = AQ_READ_REG(sc, FW_MPI_DAISY_CHAIN_STATUS_REG);
		bootExitCode = AQ_READ_REG(sc, FW_BOOT_EXIT_CODE_REG);
		if (v != 0x06000000 || bootExitCode != 0)
			break;
	}
	if (i <= 0) {
		aprint_error_dev(sc->sc_dev,
		    "F/W reset failed. Neither RBL nor FLB started\n");
		return ETIMEDOUT;
	}
	sc->sc_rbl_enabled = (bootExitCode != 0);

	/*
	 * Having FW version 0 is an indicator that cold start
	 * is in progress. This means two things:
	 * 1) Driver have to wait for FW/HW to finish boot (500ms giveup)
	 * 2) Driver may skip reset sequence and save time.
	 */
	if (sc->sc_fast_start_enabled && (ver != 0)) {
		error = aq_fw_read_version(sc);
		/* Skip reset as it just completed */
		if (error == 0)
			return 0;
	}

	aq_fw_bootloader_mode_t mode = FW_BOOT_MODE_UNKNOWN;
	error = mac_soft_reset(sc, &mode);
	if (error != 0) {
		aprint_error_dev(sc->sc_dev, "MAC reset failed: %d\n", error);
		return error;
	}

	switch (mode) {
	case FW_BOOT_MODE_FLB:
		aprint_debug_dev(sc->sc_dev,
		    "FLB> F/W successfully loaded from flash.\n");
		sc->sc_flash_present = true;
		return aq_fw_read_version(sc);
	case FW_BOOT_MODE_RBL_FLASH:
		aprint_debug_dev(sc->sc_dev,
		    "RBL> F/W loaded from flash. Host Bootload disabled.\n");
		sc->sc_flash_present = true;
		return aq_fw_read_version(sc);
	case FW_BOOT_MODE_UNKNOWN:
		aprint_error_dev(sc->sc_dev,
		    "F/W bootload error: unknown bootloader type\n");
		return ENOTSUP;
	case FW_BOOT_MODE_RBL_HOST_BOOTLOAD:
		aprint_debug_dev(sc->sc_dev, "RBL> Host Bootload mode\n");
		break;
	}

	/*
	 * XXX: TODO: add support Host Boot
	 */
	aprint_error_dev(sc->sc_dev,
	    "RBL> F/W Host Bootload not implemented\n");
	return ENOTSUP;
}

static int
aq_hw_reset(struct aq_softc *sc)
{
	int error;

	/* disable irq */
	AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_DIS, 0);

	/* apply */
	AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_IRQ, 1);

	/* wait ack 10 times by 1ms */
	WAIT_FOR(
	    (AQ_READ_REG(sc, AQ_INTR_CTRL_REG) & AQ_INTR_CTRL_RESET_IRQ) == 0,
	    1000, 10, &error);
	if (error != 0) {
		aprint_error_dev(sc->sc_dev,
		    "atlantic: IRQ reset failed: %d\n", error);
		return error;
	}

	return sc->sc_fw_ops->reset(sc);
}

static int
aq_hw_init_ucp(struct aq_softc *sc)
{
	int timo;

	if (FW_VERSION_MAJOR(sc) == 1) {
		if (AQ_READ_REG(sc, FW1X_MPI_INIT2_REG) == 0) {
			uint32_t data;
			cprng_fast(&data, sizeof(data));
			data &= 0xfefefefe;
			data |= 0x02020202;
			AQ_WRITE_REG(sc, FW1X_MPI_INIT2_REG, data);
		}
		AQ_WRITE_REG(sc, FW1X_MPI_INIT1_REG, 0);
	}

	for (timo = 100; timo > 0; timo--) {
		sc->sc_mbox_addr = AQ_READ_REG(sc, FW_MPI_MBOX_ADDR_REG);
		if (sc->sc_mbox_addr != 0)
			break;
		delay(1000);
	}

#define AQ_FW_MIN_VERSION	0x01050006
#define AQ_FW_MIN_VERSION_STR	"1.5.6"
	if (sc->sc_fw_version < AQ_FW_MIN_VERSION) {
		aprint_error_dev(sc->sc_dev,
		    "atlantic: wrong FW version: " AQ_FW_MIN_VERSION_STR
		    " or later required, this is %d.%d.%d\n",
		    FW_VERSION_MAJOR(sc),
		    FW_VERSION_MINOR(sc),
		    FW_VERSION_BUILD(sc));
		return ENOTSUP;
	}

	return 0;
}

static int
aq_fw_version_init(struct aq_softc *sc)
{
	int error = 0;
	char fw_vers[sizeof("F/W version xxxxx.xxxxx.xxxxx")];

	if (FW_VERSION_MAJOR(sc) == 1) {
		sc->sc_fw_ops = &aq_fw1x_ops;
	} else if ((FW_VERSION_MAJOR(sc) == 2) || (FW_VERSION_MAJOR(sc) == 3)) {
		sc->sc_fw_ops = &aq_fw2x_ops;
	} else {
		aprint_error_dev(sc->sc_dev,
		    "Unsupported F/W version %d.%d.%d\n",
		    FW_VERSION_MAJOR(sc), FW_VERSION_MINOR(sc),
		    FW_VERSION_BUILD(sc));
		return ENOTSUP;
	}
	snprintf(fw_vers, sizeof(fw_vers), "F/W version %d.%d.%d",
	    FW_VERSION_MAJOR(sc), FW_VERSION_MINOR(sc), FW_VERSION_BUILD(sc));

	/* detect revision */
	uint32_t hwrev = AQ_READ_REG(sc, AQ_HW_REVISION_REG);
	switch (hwrev & 0x0000000f) {
	case 0x01:
		aprint_normal_dev(sc->sc_dev, "Atlantic revision A0, %s\n",
		    fw_vers);
		sc->sc_features |= FEATURES_REV_A0 |
		    FEATURES_MPI_AQ | FEATURES_MIPS;
		break;
	case 0x02:
		aprint_normal_dev(sc->sc_dev, "Atlantic revision B0, %s\n",
		    fw_vers);
		sc->sc_features |= FEATURES_REV_B0 |
		    FEATURES_MPI_AQ | FEATURES_MIPS |
		    FEATURES_TPO2 | FEATURES_RPF2;
		break;
	case 0x0A:
		aprint_normal_dev(sc->sc_dev, "Atlantic revision B1, %s\n",
		    fw_vers);
		sc->sc_features |= FEATURES_REV_B1 |
		    FEATURES_MPI_AQ | FEATURES_MIPS |
		    FEATURES_TPO2 | FEATURES_RPF2;
		break;
	default:
		aprint_error_dev(sc->sc_dev,
		    "Unknown revision (0x%08x)\n", hwrev);
		error = ENOTSUP;
		break;
	}
	return error;
}

static int
fw1x_reset(struct aq_softc *sc)
{
	struct aq_mailbox_header mbox;
	const int retryCount = 1000;
	uint32_t tid0;
	int i;

	tid0 = ~0;	/*< Initial value of MBOX transactionId. */

	for (i = 0; i < retryCount; ++i) {
		/*
		 * Read the beginning of Statistics structure to capture
		 * the Transaction ID.
		 */
		aq_fw_downld_dwords(sc, sc->sc_mbox_addr,
		    (uint32_t *)&mbox, sizeof(mbox) / sizeof(uint32_t));

		/* Successfully read the stats. */
		if (tid0 == ~0U) {
			/* We have read the initial value. */
			tid0 = mbox.transaction_id;
			continue;
		} else if (mbox.transaction_id != tid0) {
			/*
			 * Compare transaction ID to initial value.
			 * If it's different means f/w is alive.
			 * We're done.
			 */
			return 0;
		}

		/*
		 * Transaction ID value haven't changed since last time.
		 * Try reading the stats again.
		 */
		delay(10);
	}
	aprint_error_dev(sc->sc_dev, "F/W 1.x reset finalize timeout\n");
	return EBUSY;
}

static int
fw1x_set_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t mode,
    aq_link_speed_t speed, aq_link_fc_t fc, aq_link_eee_t eee)
{
	uint32_t mpictrl = 0;
	uint32_t mpispeed = 0;

	if (speed & AQ_LINK_10G)
		mpispeed |= FW1X_CTRL_10G;
	if (speed & AQ_LINK_5G)
		mpispeed |= (FW1X_CTRL_5G | FW1X_CTRL_5GSR);
	if (speed & AQ_LINK_2G5)
		mpispeed |= FW1X_CTRL_2G5;
	if (speed & AQ_LINK_1G)
		mpispeed |= FW1X_CTRL_1G;
	if (speed & AQ_LINK_100M)
		mpispeed |= FW1X_CTRL_100M;

	mpictrl |= __SHIFTIN(mode, FW1X_MPI_STATE_MODE);
	mpictrl |= __SHIFTIN(mpispeed, FW1X_MPI_STATE_SPEED);
	AQ_WRITE_REG(sc, FW1X_MPI_CONTROL_REG, mpictrl);
	return 0;
}

static int
fw1x_get_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t *modep,
    aq_link_speed_t *speedp, aq_link_fc_t *fcp, aq_link_eee_t *eeep)
{
	uint32_t mpistate, mpi_speed;
	aq_link_speed_t speed = AQ_LINK_NONE;

	mpistate = AQ_READ_REG(sc, FW1X_MPI_STATE_REG);

	if (modep != NULL)
		*modep = __SHIFTOUT(mpistate, FW1X_MPI_STATE_MODE);

	mpi_speed = __SHIFTOUT(mpistate, FW1X_MPI_STATE_SPEED);
	if (mpi_speed & FW1X_CTRL_10G)
		speed = AQ_LINK_10G;
	else if (mpi_speed & (FW1X_CTRL_5G|FW1X_CTRL_5GSR))
		speed = AQ_LINK_5G;
	else if (mpi_speed & FW1X_CTRL_2G5)
		speed = AQ_LINK_2G5;
	else if (mpi_speed & FW1X_CTRL_1G)
		speed = AQ_LINK_1G;
	else if (mpi_speed & FW1X_CTRL_100M)
		speed = AQ_LINK_100M;

	if (speedp != NULL)
		*speedp = speed;

	if (fcp != NULL)
		*fcp = AQ_FC_NONE;

	if (eeep != NULL)
		*eeep = AQ_EEE_DISABLE;

	return 0;
}

static int
fw1x_get_stats(struct aq_softc *sc, aq_hw_stats_s_t *stats)
{
	int error;

	error = aq_fw_downld_dwords(sc,
	    sc->sc_mbox_addr + offsetof(fw1x_mailbox_t, msm), (uint32_t *)stats,
	    sizeof(aq_hw_stats_s_t) / sizeof(uint32_t));
	if (error < 0) {
		device_printf(sc->sc_dev,
		    "fw1x> download statistics data FAILED, error %d", error);
		return error;
	}

	stats->dpc = AQ_READ_REG(sc, RX_DMA_DROP_PKT_CNT_REG);
	stats->cprc = AQ_READ_REG(sc, RX_DMA_COALESCED_PKT_CNT_REG);
	return 0;
}

static int
fw2x_reset(struct aq_softc *sc)
{
	fw2x_capabilities_t caps = { 0 };
	int error;

	error = aq_fw_downld_dwords(sc,
	    sc->sc_mbox_addr + offsetof(fw2x_mailbox_t, caps),
	    (uint32_t *)&caps, sizeof caps / sizeof(uint32_t));
	if (error != 0) {
		aprint_error_dev(sc->sc_dev,
		    "fw2x> can't get F/W capabilities mask, error %d\n",
		    error);
		return error;
	}
	sc->sc_fw_caps = caps.caps_lo | ((uint64_t)caps.caps_hi << 32);

	char buf[256];
	snprintb(buf, sizeof(buf), FW2X_SNPRINTB, sc->sc_fw_caps);
	aprint_verbose_dev(sc->sc_dev, "fw2x> F/W capabilities=%s\n", buf);

	return 0;
}

static int
fw2x_set_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t mode,
    aq_link_speed_t speed, aq_link_fc_t fc, aq_link_eee_t eee)
{
	uint64_t mpi_ctrl;
	int error = 0;

	AQ_MPI_LOCK(sc);

	mpi_ctrl = AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG);

	switch (mode) {
	case MPI_INIT:
		mpi_ctrl &= ~FW2X_CTRL_RATE_MASK;
		if (speed & AQ_LINK_10G)
			mpi_ctrl |= FW2X_CTRL_RATE_10G;
		if (speed & AQ_LINK_5G)
			mpi_ctrl |= FW2X_CTRL_RATE_5G;
		if (speed & AQ_LINK_2G5)
			mpi_ctrl |= FW2X_CTRL_RATE_2G5;
		if (speed & AQ_LINK_1G)
			mpi_ctrl |= FW2X_CTRL_RATE_1G;
		if (speed & AQ_LINK_100M)
			mpi_ctrl |= FW2X_CTRL_RATE_100M;

		mpi_ctrl &= ~FW2X_CTRL_LINK_DROP;

		mpi_ctrl &= ~FW2X_CTRL_EEE_MASK;
		if (eee == AQ_EEE_ENABLE)
			mpi_ctrl |= FW2X_CTRL_EEE_MASK;

		mpi_ctrl &= ~(FW2X_CTRL_PAUSE | FW2X_CTRL_ASYMMETRIC_PAUSE);
		if (fc & AQ_FC_RX)
			mpi_ctrl |= FW2X_CTRL_PAUSE;
		if (fc & AQ_FC_TX)
			mpi_ctrl |= FW2X_CTRL_ASYMMETRIC_PAUSE;
		break;
	case MPI_DEINIT:
		mpi_ctrl &= ~(FW2X_CTRL_RATE_MASK | FW2X_CTRL_EEE_MASK);
		mpi_ctrl &= ~(FW2X_CTRL_PAUSE | FW2X_CTRL_ASYMMETRIC_PAUSE);
		break;
	default:
		device_printf(sc->sc_dev, "fw2x> unknown MPI state %d\n", mode);
		error =  EINVAL;
		goto failure;
	}
	AQ_WRITE64_REG(sc, FW2X_MPI_CONTROL_REG, mpi_ctrl);

 failure:
	AQ_MPI_UNLOCK(sc);
	return error;
}

static int
fw2x_get_mode(struct aq_softc *sc, aq_hw_fw_mpi_state_t *modep,
    aq_link_speed_t *speedp, aq_link_fc_t *fcp, aq_link_eee_t *eeep)
{
	uint64_t mpi_state = AQ_READ64_REG(sc, FW2X_MPI_STATE_REG);

	if (modep != NULL) {
		uint64_t mpi_ctrl = AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG);
		if (mpi_ctrl & FW2X_CTRL_RATE_MASK)
			*modep = MPI_INIT;
		else
			*modep = MPI_DEINIT;
	}

	aq_link_speed_t speed = AQ_LINK_NONE;
	if (mpi_state & FW2X_CTRL_RATE_10G)
		speed = AQ_LINK_10G;
	else if (mpi_state & FW2X_CTRL_RATE_5G)
		speed = AQ_LINK_5G;
	else if (mpi_state & FW2X_CTRL_RATE_2G5)
		speed = AQ_LINK_2G5;
	else if (mpi_state & FW2X_CTRL_RATE_1G)
		speed = AQ_LINK_1G;
	else if (mpi_state & FW2X_CTRL_RATE_100M)
		speed = AQ_LINK_100M;

	if (speedp != NULL)
		*speedp = speed;

	aq_link_fc_t fc = AQ_FC_NONE;
	if (mpi_state & FW2X_CTRL_PAUSE)
		fc |= AQ_FC_RX;
	if (mpi_state & FW2X_CTRL_ASYMMETRIC_PAUSE)
		fc |= AQ_FC_TX;
	if (fcp != NULL)
		*fcp = fc;

	/* XXX: TODO: EEE */
	if (eeep != NULL)
		*eeep = AQ_EEE_DISABLE;

	return 0;
}

static int
toggle_mpi_ctrl_and_wait(struct aq_softc *sc, uint64_t mask,
    uint32_t timeout_ms, uint32_t try_count)
{
	uint64_t mpi_ctrl = AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG);
	uint64_t mpi_state = AQ_READ64_REG(sc, FW2X_MPI_STATE_REG);
	int error;

	/* First, check that control and state values are consistent */
	if ((mpi_ctrl & mask) != (mpi_state & mask)) {
		device_printf(sc->sc_dev,
		    "fw2x> MPI control (%#llx) and state (%#llx)"
		    " are not consistent for mask %#llx!\n",
		    (unsigned long long)mpi_ctrl, (unsigned long long)mpi_state,
		    (unsigned long long)mask);
		return EINVAL;
	}

	/* Invert bits (toggle) in control register */
	mpi_ctrl ^= mask;
	AQ_WRITE64_REG(sc, FW2X_MPI_CONTROL_REG, mpi_ctrl);

	/* Clear all bits except masked */
	mpi_ctrl &= mask;

	/* Wait for FW reflecting change in state register */
	WAIT_FOR((AQ_READ64_REG(sc, FW2X_MPI_CONTROL_REG) & mask) == mpi_ctrl,
	    1000 * timeout_ms, try_count, &error);
	if (error != 0) {
		device_printf(sc->sc_dev,
		    "f/w2x> timeout while waiting for response"
		    " in state register for bit %#llx!",
		    (unsigned long long)mask);
		return error;
	}
	return 0;
}

static int
fw2x_get_stats(struct aq_softc *sc, aq_hw_stats_s_t *stats)
{
	int error;

	AQ_MPI_LOCK(sc);
	/* Say to F/W to update the statistics */
	error = toggle_mpi_ctrl_and_wait(sc, FW2X_CTRL_STATISTICS, 1, 25);
	if (error != 0) {
		device_printf(sc->sc_dev,
		    "fw2x> statistics update error %d\n", error);
		goto failure;
	}

	CTASSERT(sizeof(fw2x_msm_statistics_t) <= sizeof(struct aq_hw_stats_s));
	error = aq_fw_downld_dwords(sc,
	    sc->sc_mbox_addr + offsetof(fw2x_mailbox_t, msm), (uint32_t *)stats,
	    sizeof(fw2x_msm_statistics_t) / sizeof(uint32_t));
	if (error != 0) {
		device_printf(sc->sc_dev,
		    "fw2x> download statistics data FAILED, error %d", error);
		goto failure;
	}
	stats->dpc = AQ_READ_REG(sc, RX_DMA_DROP_PKT_CNT_REG);
	stats->cprc = AQ_READ_REG(sc, RX_DMA_COALESCED_PKT_CNT_REG);

 failure:
	AQ_MPI_UNLOCK(sc);
	return error;
}

#if NSYSMON_ENVSYS > 0
static int
fw2x_get_temperature(struct aq_softc *sc, uint32_t *temp)
{
	int error;
	uint32_t value, celsius;

	AQ_MPI_LOCK(sc);

	/* Say to F/W to update the temperature */
	error = toggle_mpi_ctrl_and_wait(sc, FW2X_CTRL_TEMPERATURE, 1, 25);
	if (error != 0)
		goto failure;

	error = aq_fw_downld_dwords(sc,
	    sc->sc_mbox_addr + offsetof(fw2x_mailbox_t, phy_info2),
	    &value, sizeof(value) / sizeof(uint32_t));
	if (error != 0)
		goto failure;

	/* 1/256 decrees C to microkelvin */
	celsius = __SHIFTOUT(value, PHYINFO2_TEMPERATURE);
	if (celsius == 0) {
		error = EIO;
		goto failure;
	}
	*temp = celsius * (1000000 / 256) + 273150000;

 failure:
	AQ_MPI_UNLOCK(sc);
	return 0;
}
#endif

static int
aq_fw_downld_dwords(struct aq_softc *sc, uint32_t addr, uint32_t *p,
    uint32_t cnt)
{
	uint32_t v;
	int error = 0;

	WAIT_FOR(AQ_READ_REG(sc, AQ_FW_SEM_RAM_REG) == 1, 1, 10000, &error);
	if (error != 0) {
		AQ_WRITE_REG(sc, AQ_FW_SEM_RAM_REG, 1);
		v = AQ_READ_REG(sc, AQ_FW_SEM_RAM_REG);
		if (v == 0) {
			device_printf(sc->sc_dev,
			    "%s:%d: timeout\n", __func__, __LINE__);
			return ETIMEDOUT;
		}
	}

	AQ_WRITE_REG(sc, AQ_FW_MBOX_ADDR_REG, addr);

	error = 0;
	for (; cnt > 0 && error == 0; cnt--) {
		/* execute mailbox interface */
		AQ_WRITE_REG_BIT(sc, AQ_FW_MBOX_CMD_REG,
		    AQ_FW_MBOX_CMD_EXECUTE, 1);
		if (sc->sc_features & FEATURES_REV_B1) {
			WAIT_FOR(AQ_READ_REG(sc, AQ_FW_MBOX_ADDR_REG) != addr,
			    1, 1000, &error);
		} else {
			WAIT_FOR((AQ_READ_REG(sc, AQ_FW_MBOX_CMD_REG) &
			    AQ_FW_MBOX_CMD_BUSY) == 0,
			    1, 1000, &error);
		}
		*p++ = AQ_READ_REG(sc, AQ_FW_MBOX_VAL_REG);
		addr += sizeof(uint32_t);
	}
	AQ_WRITE_REG(sc, AQ_FW_SEM_RAM_REG, 1);

	if (error != 0)
		device_printf(sc->sc_dev,
		    "%s:%d: timeout\n", __func__, __LINE__);

	return error;
}

/* read my mac address */
static int
aq_get_mac_addr(struct aq_softc *sc)
{
	uint32_t mac_addr[2];
	uint32_t efuse_shadow_addr;
	int err;

	efuse_shadow_addr = 0;
	if (FW_VERSION_MAJOR(sc) >= 2)
		efuse_shadow_addr = AQ_READ_REG(sc, FW2X_MPI_EFUSEADDR_REG);
	else
		efuse_shadow_addr = AQ_READ_REG(sc, FW1X_MPI_EFUSEADDR_REG);

	if (efuse_shadow_addr == 0) {
		aprint_error_dev(sc->sc_dev, "cannot get efuse addr\n");
		return ENXIO;
	}

	memset(mac_addr, 0, sizeof(mac_addr));
	err = aq_fw_downld_dwords(sc, efuse_shadow_addr + (40 * 4),
	    mac_addr, __arraycount(mac_addr));
	if (err < 0)
		return err;

	if (mac_addr[0] == 0 && mac_addr[1] == 0) {
		aprint_error_dev(sc->sc_dev, "mac address not found\n");
		return ENXIO;
	}

	mac_addr[0] = bswap32(mac_addr[0]);
	mac_addr[1] = bswap32(mac_addr[1]);

	memcpy(sc->sc_enaddr.ether_addr_octet,
	    (uint8_t *)mac_addr, ETHER_ADDR_LEN);
	aprint_normal_dev(sc->sc_dev, "Etheraddr: %s\n",
	    ether_sprintf(sc->sc_enaddr.ether_addr_octet));

	return 0;
}

/* set multicast filter. index 0 for own address */
static int
aq_set_mac_addr(struct aq_softc *sc, int index, uint8_t *enaddr)
{
	uint32_t h, l;

	if (index >= AQ_HW_MAC_NUM)
		return EINVAL;

	if (enaddr == NULL) {
		/* disable */
		AQ_WRITE_REG_BIT(sc,
		    RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_EN, 0);
		return 0;
	}

	h = (enaddr[0] <<  8) | (enaddr[1]);
	l = ((uint32_t)enaddr[2] << 24) | (enaddr[3] << 16) |
	    (enaddr[4] <<  8) | (enaddr[5]);

	/* disable, set, and enable */
	AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_EN, 0);
	AQ_WRITE_REG(sc, RPF_L2UC_LSW_REG(index), l);
	AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index),
	    RPF_L2UC_MSW_MACADDR_HI, h);
	AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_ACTION, 1);
	AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(index), RPF_L2UC_MSW_EN, 1);

	return 0;
}

static int
aq_set_capability(struct aq_softc *sc)
{
	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
	int ip4csum_tx =
	    ((ifp->if_capenable & IFCAP_CSUM_IPv4_Tx) == 0) ? 0 : 1;
	int ip4csum_rx =
	    ((ifp->if_capenable & IFCAP_CSUM_IPv4_Rx) == 0) ? 0 : 1;
	int l4csum_tx = ((ifp->if_capenable &
	   (IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_UDPv4_Tx |
	   IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_UDPv6_Tx)) == 0) ? 0 : 1;
	int l4csum_rx =
	   ((ifp->if_capenable & (IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
	   IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx)) == 0) ? 0 : 1;
	uint32_t lso =
	   ((ifp->if_capenable & (IFCAP_TSOv4 | IFCAP_TSOv6)) == 0) ?
	   0 : 0xffffffff;
	uint32_t lro = ((ifp->if_capenable & IFCAP_LRO) == 0) ?
	    0 : 0xffffffff;
	uint32_t i, v;

	/* TX checksums offloads*/
	AQ_WRITE_REG_BIT(sc, TPO_HWCSUM_REG, TPO_HWCSUM_IP4CSUM_EN, ip4csum_tx);
	AQ_WRITE_REG_BIT(sc, TPO_HWCSUM_REG, TPO_HWCSUM_L4CSUM_EN, l4csum_tx);

	/* RX checksums offloads*/
	AQ_WRITE_REG_BIT(sc, RPO_HWCSUM_REG, RPO_HWCSUM_IP4CSUM_EN, ip4csum_rx);
	AQ_WRITE_REG_BIT(sc, RPO_HWCSUM_REG, RPO_HWCSUM_L4CSUM_EN, l4csum_rx);

	/* LSO offloads*/
	AQ_WRITE_REG(sc, TDM_LSO_EN_REG, lso);

#define AQ_B0_LRO_RXD_MAX	16
	v = (8 < AQ_B0_LRO_RXD_MAX) ? 3 :
	    (4 < AQ_B0_LRO_RXD_MAX) ? 2 :
	    (2 < AQ_B0_LRO_RXD_MAX) ? 1 : 0;
	for (i = 0; i < AQ_RINGS_NUM; i++) {
		AQ_WRITE_REG_BIT(sc, RPO_LRO_LDES_MAX_REG(i),
		    RPO_LRO_LDES_MAX_MASK(i), v);
	}

	AQ_WRITE_REG_BIT(sc, RPO_LRO_TB_DIV_REG, RPO_LRO_TB_DIV, 0x61a);
	AQ_WRITE_REG_BIT(sc, RPO_LRO_INACTIVE_IVAL_REG,
	    RPO_LRO_INACTIVE_IVAL, 0);
	/*
	 * the LRO timebase divider is 5 uS (0x61a),
	 * to get a maximum coalescing interval of 250 uS,
	 * we need to multiply by 50(0x32) to get
	 * the default value 250 uS
	 */
	AQ_WRITE_REG_BIT(sc, RPO_LRO_MAX_COALESCING_IVAL_REG,
	    RPO_LRO_MAX_COALESCING_IVAL, 50);
	AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
	    RPO_LRO_CONF_QSESSION_LIMIT, 1);
	AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
	    RPO_LRO_CONF_TOTAL_DESC_LIMIT, 2);
	AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
	    RPO_LRO_CONF_PATCHOPTIMIZATION_EN, 0);
	AQ_WRITE_REG_BIT(sc, RPO_LRO_CONF_REG,
	    RPO_LRO_CONF_MIN_PAYLOAD_OF_FIRST_PKT, 10);
	AQ_WRITE_REG(sc, RPO_LRO_RSC_MAX_REG, 1);
	AQ_WRITE_REG(sc, RPO_LRO_ENABLE_REG, lro);

	return 0;
}

static int
aq_set_filter(struct aq_softc *sc)
{
	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
	struct ethercom *ec = &sc->sc_ethercom;
	struct ether_multi *enm;
	struct ether_multistep step;
	int idx, error = 0;

	if (ifp->if_flags & IFF_PROMISC) {
		AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_PROMISC,
		    (ifp->if_flags & IFF_PROMISC) ? 1 : 0);
		ec->ec_flags |= ETHER_F_ALLMULTI;
		goto done;
	}

	/* clear all table */
	for (idx = 0; idx < AQ_HW_MAC_NUM; idx++) {
		if (idx == AQ_HW_MAC_OWN)	/* already used for own */
			continue;
		aq_set_mac_addr(sc, idx, NULL);
	}

	/* don't accept all multicast */
	AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_MASK_REG,
	    RPF_MCAST_FILTER_MASK_ALLMULTI, 0);
	AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_REG(0),
	    RPF_MCAST_FILTER_EN, 0);

	idx = 0;
	ETHER_LOCK(ec);
	ETHER_FIRST_MULTI(step, ec, enm);
	while (enm != NULL) {
		if (idx == AQ_HW_MAC_OWN)
			idx++;

		if ((idx >= AQ_HW_MAC_NUM) ||
		    memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
			/*
			 * too many filters.
			 * fallback to accept all multicast addresses.
			 */
			AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_MASK_REG,
			    RPF_MCAST_FILTER_MASK_ALLMULTI, 1);
			AQ_WRITE_REG_BIT(sc, RPF_MCAST_FILTER_REG(0),
			    RPF_MCAST_FILTER_EN, 1);
			ec->ec_flags |= ETHER_F_ALLMULTI;
			ETHER_UNLOCK(ec);
			goto done;
		}

		/* add a filter */
		aq_set_mac_addr(sc, idx++, enm->enm_addrlo);

		ETHER_NEXT_MULTI(step, enm);
	}
	ec->ec_flags &= ~ETHER_F_ALLMULTI;
	ETHER_UNLOCK(ec);

 done:
	return error;
}

static int
aq_ifmedia_change(struct ifnet * const ifp)
{
	struct aq_softc *sc = ifp->if_softc;
	aq_link_speed_t rate = AQ_LINK_NONE;
	aq_link_fc_t fc = AQ_FC_NONE;
	aq_link_eee_t eee = AQ_EEE_DISABLE;

	if (IFM_TYPE(sc->sc_media.ifm_media) != IFM_ETHER)
		return EINVAL;

	switch (IFM_SUBTYPE(sc->sc_media.ifm_media)) {
	case IFM_AUTO:
		rate = AQ_LINK_AUTO;
		break;
	case IFM_NONE:
		rate = AQ_LINK_NONE;
		break;
	case IFM_100_TX:
		rate = AQ_LINK_100M;
		break;
	case IFM_1000_T:
		rate = AQ_LINK_1G;
		break;
	case IFM_2500_T:
		rate = AQ_LINK_2G5;
		break;
	case IFM_5000_T:
		rate = AQ_LINK_5G;
		break;
	case IFM_10G_T:
		rate = AQ_LINK_10G;
		break;
	default:
		device_printf(sc->sc_dev, "unknown media: 0x%X\n",
		    IFM_SUBTYPE(sc->sc_media.ifm_media));
		return ENODEV;
	}

	if (sc->sc_media.ifm_media & IFM_FLOW)
		fc = AQ_FC_ALL;

	/* XXX: todo EEE */

	/* re-initialize hardware with new parameters */
	aq_set_linkmode(sc, rate, fc, eee);

	return 0;
}

static void
aq_ifmedia_status(struct ifnet * const ifp, struct ifmediareq *ifmr)
{
	struct aq_softc *sc = ifp->if_softc;

	/* update ifm_active */
	ifmr->ifm_active = IFM_ETHER;
	if (sc->sc_link_fc & AQ_FC_RX)
		ifmr->ifm_active |= IFM_ETH_RXPAUSE;
	if (sc->sc_link_fc & AQ_FC_TX)
		ifmr->ifm_active |= IFM_ETH_TXPAUSE;

	switch (sc->sc_link_rate) {
	case AQ_LINK_100M:
		/* XXX: need to detect fulldup or halfdup */
		ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
		break;
	case AQ_LINK_1G:
		ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
		break;
	case AQ_LINK_2G5:
		ifmr->ifm_active |= IFM_2500_T | IFM_FDX;
		break;
	case AQ_LINK_5G:
		ifmr->ifm_active |= IFM_5000_T | IFM_FDX;
		break;
	case AQ_LINK_10G:
		ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
		break;
	default:
		ifmr->ifm_active |= IFM_NONE;
		break;
	}

	/* update ifm_status */
	ifmr->ifm_status = IFM_AVALID;
	if (sc->sc_link_rate != AQ_LINK_NONE)
		ifmr->ifm_status |= IFM_ACTIVE;
}

static void
aq_initmedia(struct aq_softc *sc)
{
#define IFMEDIA_ETHER_ADD(sc, media)	\
	ifmedia_add(&(sc)->sc_media, IFM_ETHER | media, 0, NULL);

	IFMEDIA_ETHER_ADD(sc, IFM_NONE);
	if (sc->sc_available_rates & AQ_LINK_100M) {
		IFMEDIA_ETHER_ADD(sc, IFM_100_TX);
		IFMEDIA_ETHER_ADD(sc, IFM_100_TX | IFM_FLOW);
		IFMEDIA_ETHER_ADD(sc, IFM_100_TX | IFM_FDX | IFM_FLOW);
	}
	if (sc->sc_available_rates & AQ_LINK_1G) {
		IFMEDIA_ETHER_ADD(sc, IFM_1000_T | IFM_FDX);
		IFMEDIA_ETHER_ADD(sc, IFM_1000_T | IFM_FDX | IFM_FLOW);
	}
	if (sc->sc_available_rates & AQ_LINK_2G5) {
		IFMEDIA_ETHER_ADD(sc, IFM_2500_T | IFM_FDX);
		IFMEDIA_ETHER_ADD(sc, IFM_2500_T | IFM_FDX | IFM_FLOW);
	}
	if (sc->sc_available_rates & AQ_LINK_5G) {
		IFMEDIA_ETHER_ADD(sc, IFM_5000_T | IFM_FDX);
		IFMEDIA_ETHER_ADD(sc, IFM_5000_T | IFM_FDX | IFM_FLOW);
	}
	if (sc->sc_available_rates & AQ_LINK_10G) {
		IFMEDIA_ETHER_ADD(sc, IFM_10G_T | IFM_FDX);
		IFMEDIA_ETHER_ADD(sc, IFM_10G_T | IFM_FDX | IFM_FLOW);
	}
	IFMEDIA_ETHER_ADD(sc, IFM_AUTO);
	IFMEDIA_ETHER_ADD(sc, IFM_AUTO | IFM_FLOW);

	/* default: auto without flowcontrol */
	ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);
	aq_set_linkmode(sc, AQ_LINK_AUTO, AQ_FC_NONE, AQ_EEE_DISABLE);
}

static int
aq_set_linkmode(struct aq_softc *sc, aq_link_speed_t speed, aq_link_fc_t fc,
    aq_link_eee_t eee)
{
	return sc->sc_fw_ops->set_mode(sc, MPI_INIT, speed, fc, eee);
}

static int
aq_get_linkmode(struct aq_softc *sc, aq_link_speed_t *speed, aq_link_fc_t *fc,
   aq_link_eee_t *eee)
{
	aq_hw_fw_mpi_state_t mode;
	int error;

	error = sc->sc_fw_ops->get_mode(sc, &mode, speed, fc, eee);
	if (error != 0)
		return error;
	if (mode != MPI_INIT)
		return ENXIO;

	return 0;
}

static void
aq_hw_init_tx_path(struct aq_softc *sc)
{
	/* Tx TC/RSS number config */
	AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_TC_MODE_EN, 1);

	AQ_WRITE_REG_BIT(sc, THM_LSO_TCP_FLAG1_REG,
	    THM_LSO_TCP_FLAG1_FIRST, 0x0ff6);
	AQ_WRITE_REG_BIT(sc, THM_LSO_TCP_FLAG1_REG,
	    THM_LSO_TCP_FLAG1_MID,   0x0ff6);
	AQ_WRITE_REG_BIT(sc, THM_LSO_TCP_FLAG2_REG,
	   THM_LSO_TCP_FLAG2_LAST,  0x0f7f);

	/* misc */
	AQ_WRITE_REG(sc, TX_TPO2_REG,
	   (sc->sc_features & FEATURES_TPO2) ? TX_TPO2_EN : 0);
	AQ_WRITE_REG_BIT(sc, TDM_DCA_REG, TDM_DCA_EN, 0);
	AQ_WRITE_REG_BIT(sc, TDM_DCA_REG, TDM_DCA_MODE, 0);

	AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_SCP_INS_EN, 1);
}

static void
aq_hw_init_rx_path(struct aq_softc *sc)
{
	int i;

	/* clear setting */
	AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_TC_MODE, 0);
	AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_FC_MODE, 0);
	AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG, 0);
	for (i = 0; i < 32; i++) {
		AQ_WRITE_REG_BIT(sc, RPF_ETHERTYPE_FILTER_REG(i),
		   RPF_ETHERTYPE_FILTER_EN, 0);
	}

	if (sc->sc_rss_enable) {
		/* Rx TC/RSS number config */
		AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_TC_MODE, 1);

		/* Rx flow control */
		AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_FC_MODE, 1);

		/* RSS Ring selection */
		switch (sc->sc_nqueues) {
		case 2:
			AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG,
			    RX_FLR_RSS_CONTROL1_EN | 0x11111111);
			break;
		case 4:
			AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG,
			    RX_FLR_RSS_CONTROL1_EN | 0x22222222);
			break;
		case 8:
			AQ_WRITE_REG(sc, RX_FLR_RSS_CONTROL1_REG,
			    RX_FLR_RSS_CONTROL1_EN | 0x33333333);
			break;
		}
	}

	/* L2 and Multicast filters */
	for (i = 0; i < AQ_HW_MAC_NUM; i++) {
		AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(i), RPF_L2UC_MSW_EN, 0);
		AQ_WRITE_REG_BIT(sc, RPF_L2UC_MSW_REG(i), RPF_L2UC_MSW_ACTION,
		    RPF_ACTION_HOST);
	}
	AQ_WRITE_REG(sc, RPF_MCAST_FILTER_MASK_REG, 0);
	AQ_WRITE_REG(sc, RPF_MCAST_FILTER_REG(0), 0x00010fff);

	/* Vlan filters */
	AQ_WRITE_REG_BIT(sc, RPF_VLAN_TPID_REG, RPF_VLAN_TPID_OUTER,
	    ETHERTYPE_QINQ);
	AQ_WRITE_REG_BIT(sc, RPF_VLAN_TPID_REG, RPF_VLAN_TPID_INNER,
	    ETHERTYPE_VLAN);
	AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG, RPF_VLAN_MODE_PROMISC, 0);

	if (sc->sc_features & FEATURES_REV_B) {
		AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG,
		    RPF_VLAN_MODE_ACCEPT_UNTAGGED, 1);
		AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG,
		    RPF_VLAN_MODE_UNTAGGED_ACTION, RPF_ACTION_HOST);
	}

	/* misc */
	if (sc->sc_features & FEATURES_RPF2)
		AQ_WRITE_REG(sc, RX_TCP_RSS_HASH_REG, RX_TCP_RSS_HASH_RPF2);
	else
		AQ_WRITE_REG(sc, RX_TCP_RSS_HASH_REG, 0);

	/*
	 * XXX: RX_TCP_RSS_HASH_REG:
	 *  linux   set 0x000f0000
	 *  freebsd set 0x000f001e
	 */
	/* RSS hash type set for IP/TCP */
	AQ_WRITE_REG_BIT(sc, RX_TCP_RSS_HASH_REG,
	    RX_TCP_RSS_HASH_TYPE, 0x001e);

	AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_EN, 1);
	AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_ACTION, RPF_ACTION_HOST);
	AQ_WRITE_REG_BIT(sc, RPF_L2BC_REG, RPF_L2BC_THRESHOLD, 0xffff);

	AQ_WRITE_REG_BIT(sc, RX_DMA_DCA_REG, RX_DMA_DCA_EN, 0);
	AQ_WRITE_REG_BIT(sc, RX_DMA_DCA_REG, RX_DMA_DCA_MODE, 0);
}

static void
aq_hw_interrupt_moderation_set(struct aq_softc *sc)
{
	int i;

	if (sc->sc_intr_moderation_enable) {
		unsigned int tx_min, rx_min;	/* 0-255 */
		unsigned int tx_max, rx_max;	/* 0-511? */

		switch (sc->sc_link_rate) {
		case AQ_LINK_100M:
			tx_min = 0x4f;
			tx_max = 0xff;
			rx_min = 0x04;
			rx_max = 0x50;
			break;
		case AQ_LINK_1G:
		default:
			tx_min = 0x4f;
			tx_max = 0xff;
			rx_min = 0x30;
			rx_max = 0x80;
			break;
		case AQ_LINK_2G5:
			tx_min = 0x4f;
			tx_max = 0xff;
			rx_min = 0x18;
			rx_max = 0xe0;
			break;
		case AQ_LINK_5G:
			tx_min = 0x4f;
			tx_max = 0xff;
			rx_min = 0x0c;
			rx_max = 0x70;
			break;
		case AQ_LINK_10G:
			tx_min = 0x4f;
			tx_max = 0x1ff;
			rx_min = 0x06;	/* freebsd use 80 */
			rx_max = 0x38;	/* freebsd use 120 */
			break;
		}

		AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
		    TX_DMA_INT_DESC_WRWB_EN, 0);
		AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
		    TX_DMA_INT_DESC_MODERATE_EN, 1);
		AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
		    RX_DMA_INT_DESC_WRWB_EN, 0);
		AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
		    RX_DMA_INT_DESC_MODERATE_EN, 1);

		for (i = 0; i < AQ_RINGS_NUM; i++) {
			AQ_WRITE_REG(sc, TX_INTR_MODERATION_CTL_REG(i),
			    __SHIFTIN(tx_min, TX_INTR_MODERATION_CTL_MIN) |
			    __SHIFTIN(tx_max, TX_INTR_MODERATION_CTL_MAX) |
			    TX_INTR_MODERATION_CTL_EN);
		}
		for (i = 0; i < AQ_RINGS_NUM; i++) {
			AQ_WRITE_REG(sc, RX_INTR_MODERATION_CTL_REG(i),
			    __SHIFTIN(rx_min, RX_INTR_MODERATION_CTL_MIN) |
			    __SHIFTIN(rx_max, RX_INTR_MODERATION_CTL_MAX) |
			    RX_INTR_MODERATION_CTL_EN);
		}

	} else {
		AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
		    TX_DMA_INT_DESC_WRWB_EN, 1);
		AQ_WRITE_REG_BIT(sc, TX_DMA_INT_DESC_WRWB_EN_REG,
		    TX_DMA_INT_DESC_MODERATE_EN, 0);
		AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
		    RX_DMA_INT_DESC_WRWB_EN, 1);
		AQ_WRITE_REG_BIT(sc, RX_DMA_INT_DESC_WRWB_EN_REG,
		    RX_DMA_INT_DESC_MODERATE_EN, 0);

		for (i = 0; i < AQ_RINGS_NUM; i++) {
			AQ_WRITE_REG(sc, TX_INTR_MODERATION_CTL_REG(i), 0);
		}
		for (i = 0; i < AQ_RINGS_NUM; i++) {
			AQ_WRITE_REG(sc, RX_INTR_MODERATION_CTL_REG(i), 0);
		}
	}
}

static void
aq_hw_qos_set(struct aq_softc *sc)
{
	uint32_t tc = 0;
	uint32_t buff_size;

	/* TPS Descriptor rate init */
	AQ_WRITE_REG_BIT(sc, TPS_DESC_RATE_REG, TPS_DESC_RATE_TA_RST, 0);
	AQ_WRITE_REG_BIT(sc, TPS_DESC_RATE_REG, TPS_DESC_RATE_LIM, 0xa);

	/* TPS VM init */
	AQ_WRITE_REG_BIT(sc, TPS_DESC_VM_ARB_MODE_REG, TPS_DESC_VM_ARB_MODE, 0);

	/* TPS TC credits init */
	AQ_WRITE_REG_BIT(sc, TPS_DESC_TC_ARB_MODE_REG, TPS_DESC_TC_ARB_MODE, 0);
	AQ_WRITE_REG_BIT(sc, TPS_DATA_TC_ARB_MODE_REG, TPS_DATA_TC_ARB_MODE, 0);

	AQ_WRITE_REG_BIT(sc, TPS_DATA_TCT_REG(tc),
	    TPS_DATA_TCT_CREDIT_MAX, 0xfff);
	AQ_WRITE_REG_BIT(sc, TPS_DATA_TCT_REG(tc),
	    TPS_DATA_TCT_WEIGHT, 0x64);
	AQ_WRITE_REG_BIT(sc, TPS_DESC_TCT_REG(tc),
	    TPS_DESC_TCT_CREDIT_MAX, 0x50);
	AQ_WRITE_REG_BIT(sc, TPS_DESC_TCT_REG(tc),
	    TPS_DESC_TCT_WEIGHT, 0x1e);

	/* Tx buf size */
	tc = 0;
	buff_size = AQ_HW_TXBUF_MAX;
	AQ_WRITE_REG_BIT(sc, TPB_TXB_BUFSIZE_REG(tc), TPB_TXB_BUFSIZE,
	    buff_size);
	AQ_WRITE_REG_BIT(sc, TPB_TXB_THRESH_REG(tc), TPB_TXB_THRESH_HI,
	    (buff_size * (1024 / 32) * 66) / 100);
	AQ_WRITE_REG_BIT(sc, TPB_TXB_THRESH_REG(tc), TPB_TXB_THRESH_LO,
	    (buff_size * (1024 / 32) * 50) / 100);

	/* QoS Rx buf size per TC */
	tc = 0;
	buff_size = AQ_HW_RXBUF_MAX;
	AQ_WRITE_REG_BIT(sc, RPB_RXB_BUFSIZE_REG(tc), RPB_RXB_BUFSIZE,
	    buff_size);
	AQ_WRITE_REG_BIT(sc, RPB_RXB_XOFF_REG(tc), RPB_RXB_XOFF_EN, 0);
	AQ_WRITE_REG_BIT(sc, RPB_RXB_XOFF_REG(tc), RPB_RXB_XOFF_THRESH_HI,
	    (buff_size * (1024 / 32) * 66) / 100);
	AQ_WRITE_REG_BIT(sc, RPB_RXB_XOFF_REG(tc), RPB_RXB_XOFF_THRESH_LO,
	    (buff_size * (1024 / 32) * 50) / 100);

	/* QoS 802.1p priority -> TC mapping */
	int i_priority;
	for (i_priority = 0; i_priority < 8; i_priority++) {
		AQ_WRITE_REG_BIT(sc, RPF_RPB_RX_TC_UPT_REG,
		    RPF_RPB_RX_TC_UPT_MASK(i_priority), 0);
	}
}

/* called once from aq_attach */
static int
aq_init_rss(struct aq_softc *sc)
{
	CTASSERT(AQ_RSS_HASHKEY_SIZE == RSS_KEYSIZE);
	uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)];
	uint8_t rss_table[AQ_RSS_INDIRECTION_TABLE_MAX];
	unsigned int i;
	int error;

	/* initialize rss key */
	rss_getkey((uint8_t *)rss_key);

	/* hash to ring table */
	for (i = 0; i < AQ_RSS_INDIRECTION_TABLE_MAX; i++) {
		rss_table[i] = i % sc->sc_nqueues;
	}

	/*
	 * set rss key
	 */
	for (i = 0; i < __arraycount(rss_key); i++) {
		uint32_t key_data = sc->sc_rss_enable ? ntohl(rss_key[i]) : 0;
		AQ_WRITE_REG(sc, RPF_RSS_KEY_WR_DATA_REG, key_data);
		AQ_WRITE_REG_BIT(sc, RPF_RSS_KEY_ADDR_REG,
		    RPF_RSS_KEY_ADDR, __arraycount(rss_key) - 1 - i);
		AQ_WRITE_REG_BIT(sc, RPF_RSS_KEY_ADDR_REG,
		    RPF_RSS_KEY_WR_EN, 1);
		WAIT_FOR(AQ_READ_REG_BIT(sc, RPF_RSS_KEY_ADDR_REG,
		    RPF_RSS_KEY_WR_EN) == 0, 1000, 10, &error);
		if (error != 0) {
			device_printf(sc->sc_dev, "%s: rss key write timeout\n",
			    __func__);
			goto rss_set_timeout;
		}
	}

	/*
	 * set rss indirection table
	 *
	 * AQ's rss redirect table is consist of 3bit*64 (192bit) packed array.
	 * we'll make it by __BITMAP(3) macros.
	 */
	__BITMAP_TYPE(, uint16_t, 3 * AQ_RSS_INDIRECTION_TABLE_MAX) bit3x64;
	__BITMAP_ZERO(&bit3x64);

#define AQ_3BIT_PACKED_ARRAY_SET(bitmap, idx, val)		\
	do {							\
		if (val & 1) {					\
			__BITMAP_SET((idx) * 3, (bitmap));	\
		} else {					\
			__BITMAP_CLR((idx) * 3, (bitmap));	\
		}						\
		if (val & 2) {					\
			__BITMAP_SET((idx) * 3 + 1, (bitmap));	\
		} else {					\
			__BITMAP_CLR((idx) * 3 + 1, (bitmap));	\
		}						\
		if (val & 4) {					\
			__BITMAP_SET((idx) * 3 + 2, (bitmap));	\
		} else {					\
			__BITMAP_CLR((idx) * 3 + 2, (bitmap));	\
		}						\
	} while (0 /* CONSTCOND */)

	for (i = 0; i < AQ_RSS_INDIRECTION_TABLE_MAX; i++) {
		AQ_3BIT_PACKED_ARRAY_SET(&bit3x64, i, rss_table[i]);
	}

	/* write 192bit data in steps of 16bit */
	for (i = 0; i < (int)__arraycount(bit3x64._b); i++) {
		AQ_WRITE_REG_BIT(sc, RPF_RSS_REDIR_WR_DATA_REG,
		    RPF_RSS_REDIR_WR_DATA, bit3x64._b[i]);
		AQ_WRITE_REG_BIT(sc, RPF_RSS_REDIR_ADDR_REG,
		    RPF_RSS_REDIR_ADDR, i);
		AQ_WRITE_REG_BIT(sc, RPF_RSS_REDIR_ADDR_REG,
		    RPF_RSS_REDIR_WR_EN, 1);

		WAIT_FOR(AQ_READ_REG_BIT(sc, RPF_RSS_REDIR_ADDR_REG,
		    RPF_RSS_REDIR_WR_EN) == 0, 1000, 10, &error);
		if (error != 0)
			break;
	}

 rss_set_timeout:
	return error;
}

static void
aq_hw_l3_filter_set(struct aq_softc *sc)
{
	int i;

	/* clear all filter */
	for (i = 0; i < 8; i++) {
		AQ_WRITE_REG_BIT(sc, RPF_L3_FILTER_REG(i),
		    RPF_L3_FILTER_L4_EN, 0);
	}
}

static void
aq_set_vlan_filters(struct aq_softc *sc)
{
	struct ethercom *ec = &sc->sc_ethercom;
	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
	struct vlanid_list *vlanidp;
	int i;

	ETHER_LOCK(ec);

	/* disable all vlan filters */
	for (i = 0; i < RPF_VLAN_MAX_FILTERS; i++)
		AQ_WRITE_REG(sc, RPF_VLAN_FILTER_REG(i), 0);

	/* count VID */
	i = 0;
	SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list)
		i++;

	if (((sc->sc_ethercom.ec_capenable & ETHERCAP_VLAN_HWFILTER) == 0) ||
	    (ifp->if_flags & IFF_PROMISC) ||
	    (i > RPF_VLAN_MAX_FILTERS)) {
		/*
		 * no vlan hwfilter, in promiscuous mode, or too many VID?
		 * must receive all VID
		 */
		AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG,
		    RPF_VLAN_MODE_PROMISC, 1);
		goto done;
	}

	/* receive only selected VID */
	AQ_WRITE_REG_BIT(sc, RPF_VLAN_MODE_REG, RPF_VLAN_MODE_PROMISC, 0);
	i = 0;
	SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) {
		AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
		    RPF_VLAN_FILTER_EN, 1);
		AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
		    RPF_VLAN_FILTER_RXQ_EN, 0);
		AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
		    RPF_VLAN_FILTER_RXQ, 0);
		AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
		    RPF_VLAN_FILTER_ACTION, RPF_ACTION_HOST);
		AQ_WRITE_REG_BIT(sc, RPF_VLAN_FILTER_REG(i),
		    RPF_VLAN_FILTER_ID, vlanidp->vid);
		i++;
	}

 done:
	ETHER_UNLOCK(ec);
}

static int
aq_hw_init(struct aq_softc *sc)
{
	uint32_t v;

	/* Force limit MRRS on RDM/TDM to 2K */
	v = AQ_READ_REG(sc, AQ_PCI_REG_CONTROL_6_REG);
	AQ_WRITE_REG(sc, AQ_PCI_REG_CONTROL_6_REG, (v & ~0x0707) | 0x0404);

	/*
	 * TX DMA total request limit. B0 hardware is not capable to
	 * handle more than (8K-MRRS) incoming DMA data.
	 * Value 24 in 256byte units
	 */
	AQ_WRITE_REG(sc, AQ_HW_TX_DMA_TOTAL_REQ_LIMIT_REG, 24);

	aq_hw_init_tx_path(sc);
	aq_hw_init_rx_path(sc);

	aq_hw_interrupt_moderation_set(sc);

	aq_set_mac_addr(sc, AQ_HW_MAC_OWN, sc->sc_enaddr.ether_addr_octet);
	aq_set_linkmode(sc, AQ_LINK_NONE, AQ_FC_NONE, AQ_EEE_DISABLE);

	aq_hw_qos_set(sc);

	/* Enable interrupt */
	int irqmode;
	if (sc->sc_msix)
		irqmode =  AQ_INTR_CTRL_IRQMODE_MSIX;
	else
		irqmode =  AQ_INTR_CTRL_IRQMODE_MSI;

	AQ_WRITE_REG(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_RESET_DIS);
	AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_MULTIVEC,
	    sc->sc_msix ? 1 : 0);
	AQ_WRITE_REG_BIT(sc, AQ_INTR_CTRL_REG, AQ_INTR_CTRL_IRQMODE, irqmode);

	AQ_WRITE_REG(sc, AQ_INTR_AUTOMASK_REG, 0xffffffff);

	AQ_WRITE_REG(sc, AQ_GEN_INTR_MAP_REG(0),
	    ((AQ_B0_ERR_INT << 24) | (1U << 31)) |
	    ((AQ_B0_ERR_INT << 16) | (1 << 23))
	);

	/* link interrupt */
	if (!sc->sc_msix)
		sc->sc_linkstat_irq = AQ_LINKSTAT_IRQ;
	AQ_WRITE_REG(sc, AQ_GEN_INTR_MAP_REG(3),
	    __BIT(7) | sc->sc_linkstat_irq);

	return 0;
}

static int
aq_update_link_status(struct aq_softc *sc)
{
	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
	aq_link_speed_t rate = AQ_LINK_NONE;
	aq_link_fc_t fc = AQ_FC_NONE;
	aq_link_eee_t eee = AQ_EEE_DISABLE;
	unsigned int speed;
	int changed = 0;

	aq_get_linkmode(sc, &rate, &fc, &eee);

	if (sc->sc_link_rate != rate)
		changed = 1;
	if (sc->sc_link_fc != fc)
		changed = 1;
	if (sc->sc_link_eee != eee)
		changed = 1;

	if (changed) {
		switch (rate) {
		case AQ_LINK_100M:
			speed = 100;
			break;
		case AQ_LINK_1G:
			speed = 1000;
			break;
		case AQ_LINK_2G5:
			speed = 2500;
			break;
		case AQ_LINK_5G:
			speed = 5000;
			break;
		case AQ_LINK_10G:
			speed = 10000;
			break;
		case AQ_LINK_NONE:
		default:
			speed = 0;
			break;
		}

		if (sc->sc_link_rate == AQ_LINK_NONE) {
			/* link DOWN -> UP */
			device_printf(sc->sc_dev, "link is UP: speed=%u\n",
			    speed);
			if_link_state_change(ifp, LINK_STATE_UP);
		} else if (rate == AQ_LINK_NONE) {
			/* link UP -> DOWN */
			device_printf(sc->sc_dev, "link is DOWN\n");
			if_link_state_change(ifp, LINK_STATE_DOWN);
		} else {
			device_printf(sc->sc_dev,
			    "link mode changed: speed=%u, fc=0x%x, eee=%x\n",
			    speed, fc, eee);
		}

		sc->sc_link_rate = rate;
		sc->sc_link_fc = fc;
		sc->sc_link_eee = eee;

		/* update interrupt timing according to new link speed */
		aq_hw_interrupt_moderation_set(sc);
	}

	return changed;
}

#ifdef AQ_EVENT_COUNTERS
static void
aq_update_statistics(struct aq_softc *sc)
{
	int prev = sc->sc_statistics_idx;
	int cur = prev ^ 1;

	sc->sc_fw_ops->get_stats(sc, &sc->sc_statistics[cur]);

	/*
	 * aq's internal statistics counter is 32bit.
	 * cauculate delta, and add to evcount
	 */
#define ADD_DELTA(cur, prev, name)				\
	do {							\
		uint32_t n;					\
		n = (uint32_t)(sc->sc_statistics[cur].name -	\
		    sc->sc_statistics[prev].name);		\
		if (n != 0) {					\
			AQ_EVCNT_ADD(sc, name, n);		\
		}						\
	} while (/*CONSTCOND*/0);

	ADD_DELTA(cur, prev, uprc);
	ADD_DELTA(cur, prev, mprc);
	ADD_DELTA(cur, prev, bprc);
	ADD_DELTA(cur, prev, prc);
	ADD_DELTA(cur, prev, erpr);
	ADD_DELTA(cur, prev, uptc);
	ADD_DELTA(cur, prev, mptc);
	ADD_DELTA(cur, prev, bptc);
	ADD_DELTA(cur, prev, ptc);
	ADD_DELTA(cur, prev, erpt);
	ADD_DELTA(cur, prev, mbtc);
	ADD_DELTA(cur, prev, bbtc);
	ADD_DELTA(cur, prev, mbrc);
	ADD_DELTA(cur, prev, bbrc);
	ADD_DELTA(cur, prev, ubrc);
	ADD_DELTA(cur, prev, ubtc);
	ADD_DELTA(cur, prev, dpc);
	ADD_DELTA(cur, prev, cprc);

	sc->sc_statistics_idx = cur;
}
#endif /* AQ_EVENT_COUNTERS */

/* allocate and map one DMA block */
static int
_alloc_dma(struct aq_softc *sc, bus_size_t size, bus_size_t *sizep,
    void **addrp, bus_dmamap_t *mapp, bus_dma_segment_t *seg)
{
	int nsegs, error;

	if ((error = bus_dmamem_alloc(sc->sc_dmat, size, PAGE_SIZE, 0, seg,
	    1, &nsegs, 0)) != 0) {
		aprint_error_dev(sc->sc_dev,
		    "unable to allocate DMA buffer, error=%d\n", error);
		goto fail_alloc;
	}

	if ((error = bus_dmamem_map(sc->sc_dmat, seg, 1, size, addrp,
	    BUS_DMA_COHERENT)) != 0) {
		aprint_error_dev(sc->sc_dev,
		    "unable to map DMA buffer, error=%d\n", error);
		goto fail_map;
	}

	if ((error = bus_dmamap_create(sc->sc_dmat, size, 1, size, 0,
	    0, mapp)) != 0) {
		aprint_error_dev(sc->sc_dev,
		    "unable to create DMA map, error=%d\n", error);
		goto fail_create;
	}

	if ((error = bus_dmamap_load(sc->sc_dmat, *mapp, *addrp, size, NULL,
	    0)) != 0) {
		aprint_error_dev(sc->sc_dev,
		    "unable to load DMA map, error=%d\n", error);
		goto fail_load;
	}

	*sizep = size;
	return 0;

 fail_load:
	bus_dmamap_destroy(sc->sc_dmat, *mapp);
	*mapp = NULL;
 fail_create:
	bus_dmamem_unmap(sc->sc_dmat, *addrp, size);
	*addrp = NULL;
 fail_map:
	bus_dmamem_free(sc->sc_dmat, seg, 1);
	memset(seg, 0, sizeof(*seg));
 fail_alloc:
	*sizep = 0;
	return error;
}

static void
_free_dma(struct aq_softc *sc, bus_size_t *sizep, void **addrp,
    bus_dmamap_t *mapp, bus_dma_segment_t *seg)
{
	if (*mapp != NULL) {
		bus_dmamap_destroy(sc->sc_dmat, *mapp);
		*mapp = NULL;
	}
	if (*addrp != NULL) {
		bus_dmamem_unmap(sc->sc_dmat, *addrp, *sizep);
		*addrp = NULL;
	}
	if (*sizep != 0) {
		bus_dmamem_free(sc->sc_dmat, seg, 1);
		memset(seg, 0, sizeof(*seg));
		*sizep = 0;
	}
}

static int
aq_txring_alloc(struct aq_softc *sc, struct aq_txring *txring)
{
	int i, error;

	/* allocate tx descriptors */
	error = _alloc_dma(sc, sizeof(aq_tx_desc_t) * AQ_TXD_NUM,
	    &txring->txr_txdesc_size, (void **)&txring->txr_txdesc,
	    &txring->txr_txdesc_dmamap, txring->txr_txdesc_seg);
	if (error != 0)
		return error;

	memset(txring->txr_txdesc, 0, sizeof(aq_tx_desc_t) * AQ_TXD_NUM);

	/* fill tx ring with dmamap */
	for (i = 0; i < AQ_TXD_NUM; i++) {
#define AQ_MAXDMASIZE	(16 * 1024)
#define AQ_NTXSEGS	32
		/* XXX: TODO: error check */
		bus_dmamap_create(sc->sc_dmat, AQ_MAXDMASIZE, AQ_NTXSEGS,
		    AQ_MAXDMASIZE, 0, 0, &txring->txr_mbufs[i].dmamap);
	}
	return 0;
}

static void
aq_txring_free(struct aq_softc *sc, struct aq_txring *txring)
{
	int i;

	_free_dma(sc, &txring->txr_txdesc_size, (void **)&txring->txr_txdesc,
	    &txring->txr_txdesc_dmamap, txring->txr_txdesc_seg);

	for (i = 0; i < AQ_TXD_NUM; i++) {
		if (txring->txr_mbufs[i].dmamap != NULL) {
			if (txring->txr_mbufs[i].m != NULL) {
				bus_dmamap_unload(sc->sc_dmat,
				    txring->txr_mbufs[i].dmamap);
				m_freem(txring->txr_mbufs[i].m);
				txring->txr_mbufs[i].m = NULL;
			}
			bus_dmamap_destroy(sc->sc_dmat,
			    txring->txr_mbufs[i].dmamap);
			txring->txr_mbufs[i].dmamap = NULL;
		}
	}
}

static int
aq_rxring_alloc(struct aq_softc *sc, struct aq_rxring *rxring)
{
	int i, error;

	/* allocate rx descriptors */
	error = _alloc_dma(sc, sizeof(aq_rx_desc_t) * AQ_RXD_NUM,
	    &rxring->rxr_rxdesc_size, (void **)&rxring->rxr_rxdesc,
	    &rxring->rxr_rxdesc_dmamap, rxring->rxr_rxdesc_seg);
	if (error != 0)
		return error;

	memset(rxring->rxr_rxdesc, 0, sizeof(aq_rx_desc_t) * AQ_RXD_NUM);

	/* fill rxring with dmamaps */
	for (i = 0; i < AQ_RXD_NUM; i++) {
		rxring->rxr_mbufs[i].m = NULL;
		/* XXX: TODO: error check */
		bus_dmamap_create(sc->sc_dmat, MCLBYTES, 1, MCLBYTES, 0, 0,
		    &rxring->rxr_mbufs[i].dmamap);
	}
	return 0;
}

static void
aq_rxdrain(struct aq_softc *sc, struct aq_rxring *rxring)
{
	int i;

	/* free all mbufs allocated for RX */
	for (i = 0; i < AQ_RXD_NUM; i++) {
		if (rxring->rxr_mbufs[i].m != NULL) {
			bus_dmamap_unload(sc->sc_dmat,
			    rxring->rxr_mbufs[i].dmamap);
			m_freem(rxring->rxr_mbufs[i].m);
			rxring->rxr_mbufs[i].m = NULL;
		}
	}
}

static void
aq_rxring_free(struct aq_softc *sc, struct aq_rxring *rxring)
{
	int i;

	/* free all mbufs and dmamaps */
	aq_rxdrain(sc, rxring);
	for (i = 0; i < AQ_RXD_NUM; i++) {
		if (rxring->rxr_mbufs[i].dmamap != NULL) {
			bus_dmamap_destroy(sc->sc_dmat,
			    rxring->rxr_mbufs[i].dmamap);
			rxring->rxr_mbufs[i].dmamap = NULL;
		}
	}

	/* free RX descriptor */
	_free_dma(sc, &rxring->rxr_rxdesc_size, (void **)&rxring->rxr_rxdesc,
	    &rxring->rxr_rxdesc_dmamap, rxring->rxr_rxdesc_seg);
}

static void
aq_rxring_setmbuf(struct aq_softc *sc, struct aq_rxring *rxring, int idx,
    struct mbuf *m)
{
	int error;

	/* if mbuf already exists, unload and free */
	if (rxring->rxr_mbufs[idx].m != NULL) {
		bus_dmamap_unload(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap);
		m_freem(rxring->rxr_mbufs[idx].m);
		rxring->rxr_mbufs[idx].m = NULL;
	}

	rxring->rxr_mbufs[idx].m = m;

	m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
	error = bus_dmamap_load_mbuf(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap,
	    m, BUS_DMA_READ | BUS_DMA_NOWAIT);
	if (error) {
		device_printf(sc->sc_dev,
		    "unable to load rx DMA map %d, error = %d\n", idx, error);
		panic("%s: unable to load rx DMA map. error=%d",
		    __func__, error);
	}
	bus_dmamap_sync(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap, 0,
	    rxring->rxr_mbufs[idx].dmamap->dm_mapsize, BUS_DMASYNC_PREREAD);
}

static inline void
aq_rxring_reset_desc(struct aq_softc *sc, struct aq_rxring *rxring, int idx)
{
	/* refill rxdesc, and sync */
	rxring->rxr_rxdesc[idx].read.buf_addr =
	   htole64(rxring->rxr_mbufs[idx].dmamap->dm_segs[0].ds_addr);
	rxring->rxr_rxdesc[idx].read.hdr_addr = 0;
	bus_dmamap_sync(sc->sc_dmat, rxring->rxr_rxdesc_dmamap,
	    sizeof(aq_rx_desc_t) * idx, sizeof(aq_rx_desc_t),
	    BUS_DMASYNC_PREWRITE);
}

static struct mbuf *
aq_alloc_mbuf(void)
{
	struct mbuf *m;

	MGETHDR(m, M_DONTWAIT, MT_DATA);
	if (m == NULL)
		return NULL;

	MCLGET(m, M_DONTWAIT);
	if ((m->m_flags & M_EXT) == 0) {
		m_freem(m);
		return NULL;
	}

	return m;
}

/* allocate mbuf and unload dmamap */
static int
aq_rxring_add(struct aq_softc *sc, struct aq_rxring *rxring, int idx)
{
	struct mbuf *m;

	m = aq_alloc_mbuf();
	if (m == NULL)
		return ENOBUFS;

	aq_rxring_setmbuf(sc, rxring, idx, m);
	return 0;
}

static int
aq_txrx_rings_alloc(struct aq_softc *sc)
{
	int n, error;

	for (n = 0; n < sc->sc_nqueues; n++) {
		sc->sc_queue[n].sc = sc;
		sc->sc_queue[n].txring.txr_sc = sc;
		sc->sc_queue[n].txring.txr_index = n;
		mutex_init(&sc->sc_queue[n].txring.txr_mutex, MUTEX_DEFAULT,
		    IPL_NET);
		error = aq_txring_alloc(sc, &sc->sc_queue[n].txring);
		if (error != 0)
			goto failure;

		error = aq_tx_pcq_alloc(sc, &sc->sc_queue[n].txring);
		if (error != 0)
			goto failure;

		sc->sc_queue[n].rxring.rxr_sc = sc;
		sc->sc_queue[n].rxring.rxr_index = n;
		mutex_init(&sc->sc_queue[n].rxring.rxr_mutex, MUTEX_DEFAULT,
		   IPL_NET);
		error = aq_rxring_alloc(sc, &sc->sc_queue[n].rxring);
		if (error != 0)
			break;
	}

 failure:
	return error;
}

static void
aq_txrx_rings_free(struct aq_softc *sc)
{
	int n;

	for (n = 0; n < sc->sc_nqueues; n++) {
		aq_txring_free(sc, &sc->sc_queue[n].txring);
		mutex_destroy(&sc->sc_queue[n].txring.txr_mutex);

		aq_tx_pcq_free(sc, &sc->sc_queue[n].txring);

		aq_rxring_free(sc, &sc->sc_queue[n].rxring);
		mutex_destroy(&sc->sc_queue[n].rxring.rxr_mutex);
	}
}

static int
aq_tx_pcq_alloc(struct aq_softc *sc, struct aq_txring *txring)
{
	int error = 0;
	txring->txr_softint = NULL;

	txring->txr_pcq = pcq_create(AQ_TXD_NUM, KM_NOSLEEP);
	if (txring->txr_pcq == NULL) {
		aprint_error_dev(sc->sc_dev,
		    "unable to allocate pcq for TXring[%d]\n",
		    txring->txr_index);
		error = ENOMEM;
		goto done;
	}

	txring->txr_softint = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
	    aq_deferred_transmit, txring);
	if (txring->txr_softint == NULL) {
		aprint_error_dev(sc->sc_dev,
		    "unable to establish softint for TXring[%d]\n",
		    txring->txr_index);
		error = ENOENT;
	}

 done:
	return error;
}

static void
aq_tx_pcq_free(struct aq_softc *sc, struct aq_txring *txring)
{
	struct mbuf *m;

	if (txring->txr_softint != NULL) {
		softint_disestablish(txring->txr_softint);
		txring->txr_softint = NULL;
	}

	if (txring->txr_pcq != NULL) {
		while ((m = pcq_get(txring->txr_pcq)) != NULL)
			m_freem(m);
		pcq_destroy(txring->txr_pcq);
		txring->txr_pcq = NULL;
	}
}

#if NSYSMON_ENVSYS > 0
static void
aq_temp_refresh(struct sysmon_envsys *sme, envsys_data_t *edata)
{
	struct aq_softc *sc;
	uint32_t temp;
	int error;

	sc = sme->sme_cookie;

	error = sc->sc_fw_ops->get_temperature(sc, &temp);
	if (error == 0) {
		edata->value_cur = temp;
		edata->state = ENVSYS_SVALID;
	} else {
		edata->state = ENVSYS_SINVALID;
	}
}
#endif

static void
aq_tick(void *arg)
{
	struct aq_softc *sc = arg;

	if (sc->sc_poll_linkstat || sc->sc_detect_linkstat) {
		sc->sc_detect_linkstat = false;
		aq_update_link_status(sc);
	}

#ifdef AQ_EVENT_COUNTERS
	if (sc->sc_poll_statistics)
		aq_update_statistics(sc);
#endif

	if (sc->sc_poll_linkstat
#ifdef AQ_EVENT_COUNTERS
	    || sc->sc_poll_statistics
#endif
	    ) {
		callout_schedule(&sc->sc_tick_ch, hz);
	}
}

/* interrupt enable/disable */
static void
aq_enable_intr(struct aq_softc *sc, bool link, bool txrx)
{
	uint32_t imask = 0;
	int i;

	if (txrx) {
		for (i = 0; i < sc->sc_nqueues; i++) {
			imask |= __BIT(sc->sc_tx_irq[i]);
			imask |= __BIT(sc->sc_rx_irq[i]);
		}
	}

	if (link)
		imask |= __BIT(sc->sc_linkstat_irq);

	AQ_WRITE_REG(sc, AQ_INTR_MASK_REG, imask);
	AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, 0xffffffff);
}

static int
aq_legacy_intr(void *arg)
{
	struct aq_softc *sc = arg;
	uint32_t status;
	int nintr = 0;

	status = AQ_READ_REG(sc, AQ_INTR_STATUS_REG);
	AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, 0xffffffff);

	if (status & __BIT(sc->sc_linkstat_irq)) {
		sc->sc_detect_linkstat = true;
		callout_schedule(&sc->sc_tick_ch, 0);
		nintr++;
	}

	if (status & __BIT(sc->sc_rx_irq[0])) {
		nintr += aq_rx_intr(&sc->sc_queue[0].rxring);
	}

	if (status & __BIT(sc->sc_tx_irq[0])) {
		nintr += aq_tx_intr(&sc->sc_queue[0].txring);
	}

	return nintr;
}

static int
aq_txrx_intr(void *arg)
{
	struct aq_queue *queue = arg;
	struct aq_softc *sc = queue->sc;
	struct aq_txring *txring = &queue->txring;
	struct aq_rxring *rxring = &queue->rxring;
	uint32_t status;
	int nintr = 0;
	int txringidx, rxringidx, txirq, rxirq;

	txringidx = txring->txr_index;
	rxringidx = rxring->rxr_index;
	txirq = sc->sc_tx_irq[txringidx];
	rxirq = sc->sc_rx_irq[rxringidx];

	status = AQ_READ_REG(sc, AQ_INTR_STATUS_REG);
	if ((status & (__BIT(txirq) | __BIT(rxirq))) == 0) {
		/* stray interrupt? */
		return 0;
	}

	nintr += aq_rx_intr(rxring);
	nintr += aq_tx_intr(txring);

	return nintr;
}

static int
aq_link_intr(void *arg)
{
	struct aq_softc *sc = arg;
	uint32_t status;
	int nintr = 0;

	status = AQ_READ_REG(sc, AQ_INTR_STATUS_REG);
	if (status & __BIT(sc->sc_linkstat_irq)) {
		sc->sc_detect_linkstat = true;
		callout_schedule(&sc->sc_tick_ch, 0);
		AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG,
		    __BIT(sc->sc_linkstat_irq));
		nintr++;
	}

	return nintr;
}

static void
aq_txring_reset(struct aq_softc *sc, struct aq_txring *txring, bool start)
{
	const int ringidx = txring->txr_index;
	int i;

	mutex_enter(&txring->txr_mutex);

	txring->txr_prodidx = 0;
	txring->txr_considx = 0;
	txring->txr_nfree = AQ_TXD_NUM;
	txring->txr_active = false;

	/* free mbufs untransmitted */
	for (i = 0; i < AQ_TXD_NUM; i++) {
		if (txring->txr_mbufs[i].m != NULL) {
			m_freem(txring->txr_mbufs[i].m);
			txring->txr_mbufs[i].m = NULL;
		}
	}

	/* disable DMA */
	AQ_WRITE_REG_BIT(sc, TX_DMA_DESC_REG(ringidx), TX_DMA_DESC_EN, 0);

	if (start) {
		/* TX descriptor physical address */
		paddr_t paddr = txring->txr_txdesc_dmamap->dm_segs[0].ds_addr;
		AQ_WRITE_REG(sc, TX_DMA_DESC_BASE_ADDRLSW_REG(ringidx), paddr);
		AQ_WRITE_REG(sc, TX_DMA_DESC_BASE_ADDRMSW_REG(ringidx),
		    (uint32_t)((uint64_t)paddr >> 32));

		/* TX descriptor size */
		AQ_WRITE_REG_BIT(sc, TX_DMA_DESC_REG(ringidx), TX_DMA_DESC_LEN,
		    AQ_TXD_NUM / 8);

		/* reload TAIL pointer */
		txring->txr_prodidx = txring->txr_considx =
		    AQ_READ_REG(sc, TX_DMA_DESC_TAIL_PTR_REG(ringidx));
		AQ_WRITE_REG(sc, TX_DMA_DESC_WRWB_THRESH_REG(ringidx), 0);

		/* Mapping interrupt vector */
		AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_TX_REG(ringidx),
		    AQ_INTR_IRQ_MAP_TX_IRQMAP(ringidx), sc->sc_tx_irq[ringidx]);
		AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_TX_REG(ringidx),
		    AQ_INTR_IRQ_MAP_TX_EN(ringidx), true);

		/* enable DMA */
		AQ_WRITE_REG_BIT(sc, TX_DMA_DESC_REG(ringidx),
		    TX_DMA_DESC_EN, 1);

		const int cpuid = 0;	/* XXX? */
		AQ_WRITE_REG_BIT(sc, TDM_DCAD_REG(ringidx),
		    TDM_DCAD_CPUID, cpuid);
		AQ_WRITE_REG_BIT(sc, TDM_DCAD_REG(ringidx),
		    TDM_DCAD_CPUID_EN, 0);

		txring->txr_active = true;
	}

	mutex_exit(&txring->txr_mutex);
}

static int
aq_rxring_reset(struct aq_softc *sc, struct aq_rxring *rxring, bool start)
{
	const int ringidx = rxring->rxr_index;
	int i;
	int error = 0;

	mutex_enter(&rxring->rxr_mutex);
	rxring->rxr_active = false;

	/* disable DMA */
	AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx), RX_DMA_DESC_EN, 0);

	/* free all RX mbufs */
	aq_rxdrain(sc, rxring);

	if (start) {
		for (i = 0; i < AQ_RXD_NUM; i++) {
			error = aq_rxring_add(sc, rxring, i);
			if (error != 0) {
				aq_rxdrain(sc, rxring);
				return error;
			}
			aq_rxring_reset_desc(sc, rxring, i);
		}

		/* RX descriptor physical address */
		paddr_t paddr = rxring->rxr_rxdesc_dmamap->dm_segs[0].ds_addr;
		AQ_WRITE_REG(sc, RX_DMA_DESC_BASE_ADDRLSW_REG(ringidx), paddr);
		AQ_WRITE_REG(sc, RX_DMA_DESC_BASE_ADDRMSW_REG(ringidx),
		    (uint32_t)((uint64_t)paddr >> 32));

		/* RX descriptor size */
		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx), RX_DMA_DESC_LEN,
		    AQ_RXD_NUM / 8);

		/* maximum receive frame size */
		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_BUFSIZE_REG(ringidx),
		    RX_DMA_DESC_BUFSIZE_DATA, MCLBYTES / 1024);
		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_BUFSIZE_REG(ringidx),
		    RX_DMA_DESC_BUFSIZE_HDR, 0 / 1024);

		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx),
		    RX_DMA_DESC_HEADER_SPLIT, 0);
		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx),
		    RX_DMA_DESC_VLAN_STRIP,
		    (sc->sc_ethercom.ec_capenable & ETHERCAP_VLAN_HWTAGGING) ?
		    1 : 0);

		/*
		 * reload TAIL pointer, and update readidx
		 * (HEAD pointer cannot write)
		 */
		rxring->rxr_readidx = AQ_READ_REG_BIT(sc,
		    RX_DMA_DESC_HEAD_PTR_REG(ringidx), RX_DMA_DESC_HEAD_PTR);
		AQ_WRITE_REG(sc, RX_DMA_DESC_TAIL_PTR_REG(ringidx),
		    (rxring->rxr_readidx + AQ_RXD_NUM - 1) % AQ_RXD_NUM);

		/* Rx ring set mode */

		/* Mapping interrupt vector */
		AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_RX_REG(ringidx),
		    AQ_INTR_IRQ_MAP_RX_IRQMAP(ringidx), sc->sc_rx_irq[ringidx]);
		AQ_WRITE_REG_BIT(sc, AQ_INTR_IRQ_MAP_RX_REG(ringidx),
		    AQ_INTR_IRQ_MAP_RX_EN(ringidx), 1);

		const int cpuid = 0;	/* XXX? */
		AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
		    RX_DMA_DCAD_CPUID, cpuid);
		AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
		    RX_DMA_DCAD_DESC_EN, 0);
		AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
		    RX_DMA_DCAD_HEADER_EN, 0);
		AQ_WRITE_REG_BIT(sc, RX_DMA_DCAD_REG(ringidx),
		    RX_DMA_DCAD_PAYLOAD_EN, 0);

		/* enable DMA. start receiving */
		AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(ringidx),
		    RX_DMA_DESC_EN, 1);

		rxring->rxr_active = true;
	}

	mutex_exit(&rxring->rxr_mutex);
	return error;
}

#define TXRING_NEXTIDX(idx)	\
	(((idx) >= (AQ_TXD_NUM - 1)) ? 0 : ((idx) + 1))
#define RXRING_NEXTIDX(idx)	\
	(((idx) >= (AQ_RXD_NUM - 1)) ? 0 : ((idx) + 1))

static int
aq_encap_txring(struct aq_softc *sc, struct aq_txring *txring, struct mbuf **mp)
{
	bus_dmamap_t map;
	struct mbuf *m = *mp;
	uint32_t ctl1, ctl1_ctx, ctl2;
	int idx, i, error;

	idx = txring->txr_prodidx;
	map = txring->txr_mbufs[idx].dmamap;

	error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
	    BUS_DMA_WRITE | BUS_DMA_NOWAIT);
	if (error == EFBIG) {
		struct mbuf *n;
		n = m_defrag(m, M_DONTWAIT);
		if (n == NULL)
			return EFBIG;
		/* m_defrag() preserve m */
		KASSERT(n == m);
		error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
		    BUS_DMA_WRITE | BUS_DMA_NOWAIT);
	}
	if (error != 0)
		return error;

	/*
	 * check spaces of free descriptors.
	 * +1 is additional descriptor for context (vlan, etc,.)
	 */
	if ((map->dm_nsegs + 1) > txring->txr_nfree) {
		device_printf(sc->sc_dev,
		    "TX: not enough descriptors left %d for %d segs\n",
		    txring->txr_nfree, map->dm_nsegs + 1);
		bus_dmamap_unload(sc->sc_dmat, map);
		return ENOBUFS;
	}

	/* sync dma for mbuf */
	bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
	    BUS_DMASYNC_PREWRITE);

	ctl1_ctx = 0;
	ctl2 = __SHIFTIN(m->m_pkthdr.len, AQ_TXDESC_CTL2_LEN);

	if (vlan_has_tag(m)) {
		ctl1 = AQ_TXDESC_CTL1_TYPE_TXC;
		ctl1 |= __SHIFTIN(vlan_get_tag(m), AQ_TXDESC_CTL1_VID);

		ctl1_ctx |= AQ_TXDESC_CTL1_CMD_VLAN;
		ctl2 |= AQ_TXDESC_CTL2_CTX_EN;

		/* fill context descriptor and forward index */
		txring->txr_txdesc[idx].buf_addr = 0;
		txring->txr_txdesc[idx].ctl1 = htole32(ctl1);
		txring->txr_txdesc[idx].ctl2 = 0;

		idx = TXRING_NEXTIDX(idx);
		txring->txr_nfree--;
	}

	if (m->m_pkthdr.csum_flags & M_CSUM_IPv4)
		ctl1_ctx |= AQ_TXDESC_CTL1_CMD_IP4CSUM;
	if (m->m_pkthdr.csum_flags &
	    (M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) {
		ctl1_ctx |= AQ_TXDESC_CTL1_CMD_L4CSUM;
	}

	/* fill descriptor(s) */
	for (i = 0; i < map->dm_nsegs; i++) {
		ctl1 = ctl1_ctx | AQ_TXDESC_CTL1_TYPE_TXD |
		    __SHIFTIN(map->dm_segs[i].ds_len, AQ_TXDESC_CTL1_BLEN);
		ctl1 |= AQ_TXDESC_CTL1_CMD_FCS;

		if (i == 0) {
			/* remember mbuf of these descriptors */
			txring->txr_mbufs[idx].m = m;
		} else {
			txring->txr_mbufs[idx].m = NULL;
		}

		if (i == map->dm_nsegs - 1) {
			/* last segment, mark an EndOfPacket, and cause intr */
			ctl1 |= AQ_TXDESC_CTL1_EOP | AQ_TXDESC_CTL1_CMD_WB;
		}

		txring->txr_txdesc[idx].buf_addr =
		    htole64(map->dm_segs[i].ds_addr);
		txring->txr_txdesc[idx].ctl1 = htole32(ctl1);
		txring->txr_txdesc[idx].ctl2 = htole32(ctl2);

		bus_dmamap_sync(sc->sc_dmat, txring->txr_txdesc_dmamap,
		    sizeof(aq_tx_desc_t) * idx, sizeof(aq_tx_desc_t),
		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);

		idx = TXRING_NEXTIDX(idx);
		txring->txr_nfree--;
	}

	txring->txr_prodidx = idx;

	return 0;
}

static int
aq_tx_intr(void *arg)
{
	struct aq_txring *txring = arg;
	struct aq_softc *sc = txring->txr_sc;
	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
	struct mbuf *m;
	const int ringidx = txring->txr_index;
	unsigned int idx, hw_head, n = 0;

	mutex_enter(&txring->txr_mutex);

	if (!txring->txr_active)
		goto tx_intr_done;

	hw_head = AQ_READ_REG_BIT(sc, TX_DMA_DESC_HEAD_PTR_REG(ringidx),
	    TX_DMA_DESC_HEAD_PTR);
	if (hw_head == txring->txr_considx) {
		goto tx_intr_done;
	}

	for (idx = txring->txr_considx; idx != hw_head;
	    idx = TXRING_NEXTIDX(idx), n++) {

		if ((m = txring->txr_mbufs[idx].m) != NULL) {
			bus_dmamap_unload(sc->sc_dmat,
			    txring->txr_mbufs[idx].dmamap);

			txring->txr_opackets++;
			txring->txr_obytes += m->m_pkthdr.len;
			if (m->m_flags & M_MCAST)
				txring->txr_omcasts++;

			m_freem(m);
			txring->txr_mbufs[idx].m = NULL;
		}

		txring->txr_nfree++;
	}
	txring->txr_considx = idx;

	if (ringidx == 0 && txring->txr_nfree >= AQ_TXD_MIN)
		ifp->if_flags &= ~IFF_OACTIVE;

	/* no more pending TX packet, cancel watchdog */
	if (txring->txr_nfree >= AQ_TXD_NUM)
		ifp->if_timer = 0;

 tx_intr_done:
	mutex_exit(&txring->txr_mutex);

	AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, __BIT(sc->sc_tx_irq[ringidx]));
	return n;
}

static int
aq_rx_intr(void *arg)
{
	struct aq_rxring *rxring = arg;
	struct aq_softc *sc = rxring->rxr_sc;
	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
	const int ringidx = rxring->rxr_index;
	aq_rx_desc_t *rxd;
	struct mbuf *m, *m0, *mprev, *new_m;
	uint32_t rxd_type, rxd_hash __unused;
	uint16_t rxd_status, rxd_pktlen;
	uint16_t rxd_nextdescptr __unused, rxd_vlan __unused;
	unsigned int idx, n = 0;

	mutex_enter(&rxring->rxr_mutex);

	if (!rxring->rxr_active)
		goto rx_intr_done;

	if (rxring->rxr_readidx == AQ_READ_REG_BIT(sc,
	    RX_DMA_DESC_HEAD_PTR_REG(ringidx), RX_DMA_DESC_HEAD_PTR)) {
		goto rx_intr_done;
	}

	m0 = mprev = NULL;
	for (idx = rxring->rxr_readidx;
	    idx != AQ_READ_REG_BIT(sc, RX_DMA_DESC_HEAD_PTR_REG(ringidx),
	    RX_DMA_DESC_HEAD_PTR); idx = RXRING_NEXTIDX(idx), n++) {

		bus_dmamap_sync(sc->sc_dmat, rxring->rxr_rxdesc_dmamap,
		    sizeof(aq_rx_desc_t) * idx, sizeof(aq_rx_desc_t),
		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);

		rxd = &rxring->rxr_rxdesc[idx];
		rxd_status = le16toh(rxd->wb.status);

		if ((rxd_status & RXDESC_STATUS_DD) == 0)
			break;	/* not yet done */

		rxd_type = le32toh(rxd->wb.type);
		rxd_pktlen = le16toh(rxd->wb.pkt_len);
		rxd_nextdescptr = le16toh(rxd->wb.next_desc_ptr);
		rxd_hash = le32toh(rxd->wb.rss_hash);
		rxd_vlan = le16toh(rxd->wb.vlan);

		if ((rxd_status & RXDESC_STATUS_MACERR) ||
		    (rxd_type & RXDESC_TYPE_MAC_DMA_ERR)) {
			rxring->rxr_ierrors++;
			goto rx_next;
		}

		bus_dmamap_sync(sc->sc_dmat, rxring->rxr_mbufs[idx].dmamap, 0,
		    rxring->rxr_mbufs[idx].dmamap->dm_mapsize,
		    BUS_DMASYNC_POSTREAD);
		m = rxring->rxr_mbufs[idx].m;

		new_m = aq_alloc_mbuf();
		if (new_m == NULL) {
			/*
			 * cannot allocate new mbuf.
			 * discard this packet, and reuse mbuf for next.
			 */
			rxring->rxr_iqdrops++;
			goto rx_next;
		}
		rxring->rxr_mbufs[idx].m = NULL;
		aq_rxring_setmbuf(sc, rxring, idx, new_m);

		if (m0 == NULL) {
			m0 = m;
		} else {
			if (m->m_flags & M_PKTHDR)
				m_remove_pkthdr(m);
			mprev->m_next = m;
		}
		mprev = m;

		if ((rxd_status & RXDESC_STATUS_EOP) == 0) {
			m->m_len = MCLBYTES;
		} else {
			/* last buffer */
			m->m_len = rxd_pktlen % MCLBYTES;
			m0->m_pkthdr.len = rxd_pktlen;
			/* VLAN offloading */
			if ((sc->sc_ethercom.ec_capenable &
			    ETHERCAP_VLAN_HWTAGGING) &&
			    (__SHIFTOUT(rxd_type, RXDESC_TYPE_PKTTYPE_VLAN) ||
			    __SHIFTOUT(rxd_type,
			    RXDESC_TYPE_PKTTYPE_VLAN_DOUBLE))) {
				vlan_set_tag(m0, rxd_vlan);
			}

			/* Checksum offloading */
			unsigned int pkttype_eth =
			    __SHIFTOUT(rxd_type, RXDESC_TYPE_PKTTYPE_ETHER);
			if ((ifp->if_capabilities & IFCAP_CSUM_IPv4_Rx) &&
			    (pkttype_eth == RXDESC_TYPE_PKTTYPE_ETHER_IPV4) &&
			    __SHIFTOUT(rxd_type,
			    RXDESC_TYPE_IPV4_CSUM_CHECKED)) {
				m0->m_pkthdr.csum_flags |= M_CSUM_IPv4;
				if (__SHIFTOUT(rxd_status,
				    RXDESC_STATUS_IPV4_CSUM_NG))
					m0->m_pkthdr.csum_flags |=
					    M_CSUM_IPv4_BAD;
			}
#if notyet
			/*
			 * XXX: aq always marks BAD for fragmented packet.
			 * we should peek L3 header, and ignore cksum flags
			 * if the packet is fragmented.
			 */
			if (__SHIFTOUT(rxd_type,
			    RXDESC_TYPE_TCPUDP_CSUM_CHECKED)) {
				bool checked = false;
				unsigned int pkttype_proto =
				    __SHIFTOUT(rxd_type,
				    RXDESC_TYPE_PKTTYPE_PROTO);

				if (pkttype_proto ==
				    RXDESC_TYPE_PKTTYPE_PROTO_TCP) {
					if ((pkttype_eth ==
					    RXDESC_TYPE_PKTTYPE_ETHER_IPV4) &&
					    (ifp->if_capabilities &
					    IFCAP_CSUM_TCPv4_Rx)) {
						m0->m_pkthdr.csum_flags |=
						    M_CSUM_TCPv4;
						checked = true;
					} else if ((pkttype_eth ==
					    RXDESC_TYPE_PKTTYPE_ETHER_IPV6) &&
					    (ifp->if_capabilities &
					    IFCAP_CSUM_TCPv6_Rx)) {
						m0->m_pkthdr.csum_flags |=
						    M_CSUM_TCPv6;
						checked = true;
					}
				} else if (pkttype_proto ==
				    RXDESC_TYPE_PKTTYPE_PROTO_UDP) {
					if ((pkttype_eth ==
					    RXDESC_TYPE_PKTTYPE_ETHER_IPV4) &&
					    (ifp->if_capabilities &
					    IFCAP_CSUM_UDPv4_Rx)) {
						m0->m_pkthdr.csum_flags |=
						    M_CSUM_UDPv4;
						checked = true;
					} else if ((pkttype_eth ==
					    RXDESC_TYPE_PKTTYPE_ETHER_IPV6) &&
					    (ifp->if_capabilities &
					    IFCAP_CSUM_UDPv6_Rx)) {
						m0->m_pkthdr.csum_flags |=
						    M_CSUM_UDPv6;
						checked = true;
					}
				}
				if (checked &&
				    (__SHIFTOUT(rxd_status,
				    RXDESC_STATUS_TCPUDP_CSUM_ERROR) ||
				    !__SHIFTOUT(rxd_status,
				    RXDESC_STATUS_TCPUDP_CSUM_OK))) {
					m0->m_pkthdr.csum_flags |=
					    M_CSUM_TCP_UDP_BAD;
				}
			}
#endif
			m_set_rcvif(m0, ifp);
			rxring->rxr_ipackets++;
			rxring->rxr_ibytes += m0->m_pkthdr.len;
			if_percpuq_enqueue(ifp->if_percpuq, m0);
			m0 = mprev = NULL;
		}

 rx_next:
		aq_rxring_reset_desc(sc, rxring, idx);
		AQ_WRITE_REG(sc, RX_DMA_DESC_TAIL_PTR_REG(ringidx), idx);
	}
	rxring->rxr_readidx = idx;

 rx_intr_done:
	mutex_exit(&rxring->rxr_mutex);

	AQ_WRITE_REG(sc, AQ_INTR_STATUS_CLR_REG, __BIT(sc->sc_rx_irq[ringidx]));
	return n;
}

static int
aq_vlan_cb(struct ethercom *ec, uint16_t vid, bool set)
{
	struct ifnet *ifp = &ec->ec_if;
	struct aq_softc *sc = ifp->if_softc;

	aq_set_vlan_filters(sc);
	return 0;
}

static int
aq_ifflags_cb(struct ethercom *ec)
{
	struct ifnet *ifp = &ec->ec_if;
	struct aq_softc *sc = ifp->if_softc;
	int i, ecchange, error = 0;
	unsigned short iffchange;

	AQ_LOCK(sc);

	iffchange = ifp->if_flags ^ sc->sc_if_flags;
	if ((iffchange & IFF_PROMISC) != 0)
		error = aq_set_filter(sc);

	ecchange = ec->ec_capenable ^ sc->sc_ec_capenable;
	if (ecchange & ETHERCAP_VLAN_HWTAGGING) {
		for (i = 0; i < AQ_RINGS_NUM; i++) {
			AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_REG(i),
			    RX_DMA_DESC_VLAN_STRIP,
			    (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) ?
			    1 : 0);
		}
	}

	/* vlan configuration depends on also interface promiscuous mode */
	if ((ecchange & ETHERCAP_VLAN_HWFILTER) || (iffchange & IFF_PROMISC))
		aq_set_vlan_filters(sc);

	sc->sc_ec_capenable = ec->ec_capenable;
	sc->sc_if_flags = ifp->if_flags;

	AQ_UNLOCK(sc);

	return error;
}

static int
aq_init(struct ifnet *ifp)
{
	struct aq_softc *sc = ifp->if_softc;
	int i, error = 0;

	AQ_LOCK(sc);

	aq_set_vlan_filters(sc);
	aq_set_capability(sc);

	for (i = 0; i < sc->sc_nqueues; i++) {
		aq_txring_reset(sc, &sc->sc_queue[i].txring, true);
	}

	/* invalidate RX descriptor cache */
	AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT,
	    AQ_READ_REG_BIT(sc,
	    RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT) ^ 1);

	/* start RX */
	for (i = 0; i < sc->sc_nqueues; i++) {
		error = aq_rxring_reset(sc, &sc->sc_queue[i].rxring, true);
		if (error != 0) {
			device_printf(sc->sc_dev, "%s: cannot allocate rxbuf\n",
			    __func__);
			goto aq_init_failure;
		}
	}
	aq_init_rss(sc);
	aq_hw_l3_filter_set(sc);

	/* need to start callout? */
	if (sc->sc_poll_linkstat
#ifdef AQ_EVENT_COUNTERS
	    || sc->sc_poll_statistics
#endif
	    ) {
		callout_schedule(&sc->sc_tick_ch, hz);
	}

	/* ready */
	ifp->if_flags |= IFF_RUNNING;
	ifp->if_flags &= ~IFF_OACTIVE;

	/* start TX and RX */
	aq_enable_intr(sc, true, true);
	AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_EN, 1);
	AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_BUF_EN, 1);

 aq_init_failure:
	sc->sc_if_flags = ifp->if_flags;

	AQ_UNLOCK(sc);

	return error;
}

static void
aq_send_common_locked(struct ifnet *ifp, struct aq_softc *sc,
    struct aq_txring *txring, bool is_transmit)
{
	struct mbuf *m;
	int npkt, error;

	if ((ifp->if_flags & IFF_RUNNING) == 0)
		return;

	for (npkt = 0; ; npkt++) {
		if (is_transmit)
			m = pcq_peek(txring->txr_pcq);
		else
			IFQ_POLL(&ifp->if_snd, m);

		if (m == NULL)
			break;

		if (txring->txr_nfree < AQ_TXD_MIN)
			break;

		if (is_transmit)
			pcq_get(txring->txr_pcq);
		else
			IFQ_DEQUEUE(&ifp->if_snd, m);

		error = aq_encap_txring(sc, txring, &m);
		if (error != 0) {
			/* too many mbuf chains? or not enough descriptors? */
			m_freem(m);
			txring->txr_oerrors++;
			if (txring->txr_index == 0 && error == ENOBUFS)
				ifp->if_flags |= IFF_OACTIVE;
			break;
		}

		/* update tail ptr */
		AQ_WRITE_REG(sc, TX_DMA_DESC_TAIL_PTR_REG(txring->txr_index),
		    txring->txr_prodidx);

		/* Pass the packet to any BPF listeners */
		bpf_mtap(ifp, m, BPF_D_OUT);
	}

	if (txring->txr_index == 0 && txring->txr_nfree < AQ_TXD_MIN)
		ifp->if_flags |= IFF_OACTIVE;

	if (npkt)
		ifp->if_timer = 5;
}

static void
aq_start(struct ifnet *ifp)
{
	struct aq_softc *sc;
	struct aq_txring *txring;

	sc = ifp->if_softc;
	txring = &sc->sc_queue[0].txring; /* aq_start() always use TX ring[0] */

	mutex_enter(&txring->txr_mutex);
	if (txring->txr_active && !ISSET(ifp->if_flags, IFF_OACTIVE))
		aq_send_common_locked(ifp, sc, txring, false);
	mutex_exit(&txring->txr_mutex);
}

static inline unsigned int
aq_select_txqueue(struct aq_softc *sc, struct mbuf *m)
{
	return (cpu_index(curcpu()) % sc->sc_nqueues);
}

static int
aq_transmit(struct ifnet *ifp, struct mbuf *m)
{
	struct aq_softc *sc = ifp->if_softc;
	struct aq_txring *txring;
	int ringidx;

	ringidx = aq_select_txqueue(sc, m);
	txring = &sc->sc_queue[ringidx].txring;

	if (__predict_false(!pcq_put(txring->txr_pcq, m))) {
		m_freem(m);
		return ENOBUFS;
	}

	if (mutex_tryenter(&txring->txr_mutex)) {
		aq_send_common_locked(ifp, sc, txring, true);
		mutex_exit(&txring->txr_mutex);
	} else {
		softint_schedule(txring->txr_softint);
	}
	return 0;
}

static void
aq_deferred_transmit(void *arg)
{
	struct aq_txring *txring = arg;
	struct aq_softc *sc = txring->txr_sc;
	struct ifnet *ifp = &sc->sc_ethercom.ec_if;

	mutex_enter(&txring->txr_mutex);
	if (pcq_peek(txring->txr_pcq) != NULL)
		aq_send_common_locked(ifp, sc, txring, true);
	mutex_exit(&txring->txr_mutex);
}

static void
aq_stop(struct ifnet *ifp, int disable)
{
	struct aq_softc *sc = ifp->if_softc;
	int i;

	AQ_LOCK(sc);

	ifp->if_timer = 0;

	/* disable tx/rx interrupts */
	aq_enable_intr(sc, true, false);

	AQ_WRITE_REG_BIT(sc, TPB_TX_BUF_REG, TPB_TX_BUF_EN, 0);
	for (i = 0; i < sc->sc_nqueues; i++) {
		aq_txring_reset(sc, &sc->sc_queue[i].txring, false);
	}

	AQ_WRITE_REG_BIT(sc, RPB_RPF_RX_REG, RPB_RPF_RX_BUF_EN, 0);
	for (i = 0; i < sc->sc_nqueues; i++) {
		aq_rxring_reset(sc, &sc->sc_queue[i].rxring, false);
	}

	/* invalidate RX descriptor cache */
	AQ_WRITE_REG_BIT(sc, RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT,
	    AQ_READ_REG_BIT(sc,
	    RX_DMA_DESC_CACHE_INIT_REG, RX_DMA_DESC_CACHE_INIT) ^ 1);

	ifp->if_timer = 0;

	if (!disable) {
		/* when pmf stop, disable link status intr and callout */
		aq_enable_intr(sc, false, false);
		callout_stop(&sc->sc_tick_ch);
	}

	ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);

	AQ_UNLOCK(sc);
}

static void
aq_watchdog(struct ifnet *ifp)
{
	struct aq_softc *sc = ifp->if_softc;
	struct aq_txring *txring;
	int n, head, tail;

	AQ_LOCK(sc);

	device_printf(sc->sc_dev, "%s: INTR_MASK/STATUS = %08x/%08x\n",
	    __func__, AQ_READ_REG(sc, AQ_INTR_MASK_REG),
	    AQ_READ_REG(sc, AQ_INTR_STATUS_REG));

	for (n = 0; n < sc->sc_nqueues; n++) {
		txring = &sc->sc_queue[n].txring;
		head = AQ_READ_REG_BIT(sc,
		    TX_DMA_DESC_HEAD_PTR_REG(txring->txr_index),
		    TX_DMA_DESC_HEAD_PTR),
		tail = AQ_READ_REG(sc,
		    TX_DMA_DESC_TAIL_PTR_REG(txring->txr_index));

		device_printf(sc->sc_dev, "%s: TXring[%d] HEAD/TAIL=%d/%d\n",
		    __func__, txring->txr_index, head, tail);

		aq_tx_intr(txring);
	}

	AQ_UNLOCK(sc);

	aq_init(ifp);
}

static int
aq_ioctl(struct ifnet *ifp, unsigned long cmd, void *data)
{
	struct aq_softc *sc __unused;
	struct ifreq *ifr __unused;
	uint64_t opackets, oerrors, obytes, omcasts;
	uint64_t ipackets, ierrors, ibytes, iqdrops;
	int error, i, s;

	sc = (struct aq_softc *)ifp->if_softc;
	ifr = (struct ifreq *)data;
	error = 0;

	switch (cmd) {
	case SIOCGIFDATA:
	case SIOCZIFDATA:
		opackets = oerrors = obytes = omcasts = 0;
		ipackets = ierrors = ibytes = iqdrops = 0;
		for (i = 0; i < sc->sc_nqueues; i++) {
			struct aq_txring *txring = &sc->sc_queue[i].txring;
			mutex_enter(&txring->txr_mutex);
			if (cmd == SIOCZIFDATA) {
				txring->txr_opackets = 0;
				txring->txr_obytes = 0;
				txring->txr_omcasts = 0;
				txring->txr_oerrors = 0;
			} else {
				opackets += txring->txr_opackets;
				oerrors += txring->txr_oerrors;
				obytes += txring->txr_obytes;
				omcasts += txring->txr_omcasts;
			}
			mutex_exit(&txring->txr_mutex);

			struct aq_rxring *rxring = &sc->sc_queue[i].rxring;
			mutex_enter(&rxring->rxr_mutex);
			if (cmd == SIOCZIFDATA) {
				rxring->rxr_ipackets = 0;
				rxring->rxr_ibytes = 0;
				rxring->rxr_ierrors = 0;
				rxring->rxr_iqdrops = 0;
			} else {
				ipackets += rxring->rxr_ipackets;
				ierrors += rxring->rxr_ierrors;
				ibytes += rxring->rxr_ibytes;
				iqdrops += rxring->rxr_iqdrops;
			}
			mutex_exit(&rxring->rxr_mutex);
		}
		ifp->if_opackets = opackets;
		ifp->if_oerrors = oerrors;
		ifp->if_obytes = obytes;
		ifp->if_omcasts = omcasts;
		ifp->if_ipackets = ipackets;
		ifp->if_ierrors = ierrors;
		ifp->if_ibytes = ibytes;
		ifp->if_iqdrops = iqdrops;
		break;
	}

	s = splnet();
	error = ether_ioctl(ifp, cmd, data);
	splx(s);

	if (error != ENETRESET)
		return error;

	switch (cmd) {
	case SIOCSIFCAP:
		error = aq_set_capability(sc);
		break;
	case SIOCADDMULTI:
	case SIOCDELMULTI:
		if ((ifp->if_flags & IFF_RUNNING) == 0)
			break;

		/*
		 * Multicast list has changed; set the hardware filter
		 * accordingly.
		 */
		error = aq_set_filter(sc);
		break;
	}

	return error;
}


MODULE(MODULE_CLASS_DRIVER, if_aq, "pci");

#ifdef _MODULE
#include "ioconf.c"
#endif

static int
if_aq_modcmd(modcmd_t cmd, void *opaque)
{
	int error = 0;

	switch (cmd) {
	case MODULE_CMD_INIT:
#ifdef _MODULE
		error = config_init_component(cfdriver_ioconf_if_aq,
		    cfattach_ioconf_if_aq, cfdata_ioconf_if_aq);
#endif
		return error;
	case MODULE_CMD_FINI:
#ifdef _MODULE
		error = config_fini_component(cfdriver_ioconf_if_aq,
		    cfattach_ioconf_if_aq, cfdata_ioconf_if_aq);
#endif
		return error;
	default:
		return ENOTTY;
	}
}