 @@ -1,14 +1,14 @@
-/*	$NetBSD: if_wm.c,v 1.662 2020/01/24 02:50:41 knakahara Exp $	*/
+/*	$NetBSD: if_wm.c,v 1.663 2020/01/29 06:44:27 thorpej Exp $	*/
 /*
  * Copyright (c) 2001, 2002, 2003, 2004 Wasabi Systems, Inc.
  * All rights reserved.
+ *
  * Written by Jason R. Thorpe for Wasabi Systems, Inc.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
 @@ -72,27 +72,27 @@
+ *
  * TODO (in order of importance):
+ *
  *	- Check XXX'ed comments
  *	- TX Multi queue improvement (refine queue selection logic)
  *	- Split header buffer for newer descriptors
  *	- EEE (Energy Efficiency Ethernet) for I354
  *	- Virtual Function
  *	- Set LED correctly (based on contents in EEPROM)
  *	- Rework how parameters are loaded from the EEPROM.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_wm.c,v 1.662 2020/01/24 02:50:41 knakahara Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_wm.c,v 1.663 2020/01/29 06:44:27 thorpej Exp $");
 #ifdef _KERNEL_OPT
 #include "opt_net_mpsafe.h"
 #include "opt_if_wm.h"
 #endif
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/callout.h>
 #include <sys/mbuf.h>
 #include <sys/malloc.h>
 #include <sys/kmem.h>
 #include <sys/kernel.h>
 @@ -3256,27 +3256,27 @@ wm_watchdog_txq_locked(struct ifnet *ifp
 	if (txq->txq_free == WM_NTXDESC(txq)) {
 		log(LOG_ERR, "%s: device timeout (lost interrupt)\n",
 		    device_xname(sc->sc_dev));
 	} else {
 #ifdef WM_DEBUG
 		int i, j;
 		struct wm_txsoft *txs;
 #endif
 		log(LOG_ERR,
 		    "%s: device timeout (txfree %d txsfree %d txnext %d)\n",
 		    device_xname(sc->sc_dev), txq->txq_free, txq->txq_sfree,
 		    txq->txq_next);
-		ifp->if_oerrors++;
+		if_statinc(ifp, if_oerrors);
 #ifdef WM_DEBUG
 		for (i = txq->txq_sdirty; i != txq->txq_snext;
 		    i = WM_NEXTTXS(txq, i)) {
 			txs = &txq->txq_soft[i];
 			printf("txs %d tx %d -> %d\n",
 			    i, txs->txs_firstdesc, txs->txs_lastdesc);
 			for (j = txs->txs_firstdesc; ; j = WM_NEXTTX(txq, j)) {
 				if ((sc->sc_flags & WM_F_NEWQUEUE) != 0) {
 					printf("\tdesc %d: 0x%" PRIx64 "\n", j,
 					    txq->txq_nq_descs[j].nqtx_data.nqtxd_addr);
 					printf("\t %#08x%08x\n",
 					    txq->txq_nq_descs[j].nqtx_data.nqtxd_fields,
 					    txq->txq_nq_descs[j].nqtx_data.nqtxd_cmdlen);
 @@ -3321,45 +3321,47 @@ wm_tick(void *arg)
 		splx(s);
 #endif
 		return;
+	}
 	if (sc->sc_type >= WM_T_82542_2_1) {
 		WM_EVCNT_ADD(&sc->sc_ev_rx_xon, CSR_READ(sc, WMREG_XONRXC));
 		WM_EVCNT_ADD(&sc->sc_ev_tx_xon, CSR_READ(sc, WMREG_XONTXC));
 		WM_EVCNT_ADD(&sc->sc_ev_rx_xoff, CSR_READ(sc, WMREG_XOFFRXC));
 		WM_EVCNT_ADD(&sc->sc_ev_tx_xoff, CSR_READ(sc, WMREG_XOFFTXC));
 		WM_EVCNT_ADD(&sc->sc_ev_rx_macctl, CSR_READ(sc, WMREG_FCRUC));
+	}
-	ifp->if_collisions += CSR_READ(sc, WMREG_COLC);
+	net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
-	ifp->if_ierrors += 0ULL /* ensure quad_t */
+	if_statadd_ref(nsr, if_collisions, CSR_READ(sc, WMREG_COLC));
 	if_statadd_ref(nsr, if_ierrors, 0ULL /* ensure quad_t */
 	    + CSR_READ(sc, WMREG_CRCERRS)
 	    + CSR_READ(sc, WMREG_ALGNERRC)
 	    + CSR_READ(sc, WMREG_SYMERRC)
 	    + CSR_READ(sc, WMREG_RXERRC)
 	    + CSR_READ(sc, WMREG_SEC)
 	    + CSR_READ(sc, WMREG_CEXTERR)
-	    + CSR_READ(sc, WMREG_RLEC);
+	    + CSR_READ(sc, WMREG_RLEC));
 	/*
 	 * WMREG_RNBC is incremented when there is no available buffers in host
 	 * memory. It does not mean the number of dropped packet. Because
 	 * ethernet controller can receive packets in such case if there is
 	 * space in phy's FIFO.
+	 *
 	 * If you want to know the nubmer of WMREG_RMBC, you should use such as
 	 * own EVCNT instead of if_iqdrops.
 	 */
-	ifp->if_iqdrops += CSR_READ(sc, WMREG_MPC);
+	if_statadd_ref(nsr, if_iqdrops, CSR_READ(sc, WMREG_MPC));
 	IF_STAT_PUTREF(ifp);
 	if (sc->sc_flags & WM_F_HAS_MII)
 		mii_tick(&sc->sc_mii);
 	else if ((sc->sc_type >= WM_T_82575) && (sc->sc_type <= WM_T_I211)
 	    && (sc->sc_mediatype == WM_MEDIATYPE_SERDES))
 		wm_serdes_tick(sc);
 	else
 		wm_tbi_tick(sc);
 	WM_CORE_UNLOCK(sc);
 	wm_watchdog(ifp);
 @@ -5837,28 +5839,28 @@ wm_init_locked(struct ifnet *ifp)
 #ifdef __NO_STRICT_ALIGNMENT
 	sc->sc_align_tweak = 0;
 #else
 	if ((ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN) > (MCLBYTES - 2))
 		sc->sc_align_tweak = 0;
 	else
 		sc->sc_align_tweak = 2;
 #endif /* __NO_STRICT_ALIGNMENT */
 	/* Cancel any pending I/O. */
 	wm_stop_locked(ifp, 0);
 	/* Update statistics before reset */
-	ifp->if_collisions += CSR_READ(sc, WMREG_COLC);
+	if_statadd2(ifp, if_collisions, CSR_READ(sc, WMREG_COLC),
-	ifp->if_ierrors += CSR_READ(sc, WMREG_RXERRC);
+	    if_ierrors, CSR_READ(sc, WMREG_RXERRC));
 	/* PCH_SPT hardware workaround */
 	if (sc->sc_type == WM_T_PCH_SPT)
 		wm_flush_desc_rings(sc);
 	/* Reset the chip to a known state. */
 	wm_reset(sc);
 	/*
 	 * AMT based hardware can now take control from firmware
 	 * Do this after reset.
 	 */
 	if ((sc->sc_flags & WM_F_HAS_AMT) != 0)
 @@ -7552,27 +7554,27 @@ wm_select_txqueue(struct ifnet *ifp, str
+ *
  *	Start packet transmission on the interface.
  */
 static void
 wm_start(struct ifnet *ifp)
+{
 	struct wm_softc *sc = ifp->if_softc;
 	struct wm_txqueue *txq = &sc->sc_queue[0].wmq_txq;
 #ifdef WM_MPSAFE
 	KASSERT(if_is_mpsafe(ifp));
 #endif
 	/*
-	 * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c.
+	 * if_obytes and if_omcasts are added in if_transmit()@if.c.
 	 */
 	mutex_enter(txq->txq_lock);
 	if (!txq->txq_stopping)
 		wm_start_locked(ifp);
 	mutex_exit(txq->txq_lock);
+}
 static void
 wm_start_locked(struct ifnet *ifp)
+{
 	struct wm_softc *sc = ifp->if_softc;
 	struct wm_txqueue *txq = &sc->sc_queue[0].wmq_txq;
 @@ -7586,30 +7588,31 @@ wm_transmit(struct ifnet *ifp, struct mb
 	int qid;
 	struct wm_softc *sc = ifp->if_softc;
 	struct wm_txqueue *txq;
 	qid = wm_select_txqueue(ifp, m);
 	txq = &sc->sc_queue[qid].wmq_txq;
 	if (__predict_false(!pcq_put(txq->txq_interq, m))) {
 		m_freem(m);
 		WM_Q_EVCNT_INCR(txq, pcqdrop);
 		return ENOBUFS;
+	}
-	/* XXX NOMPSAFE: ifp->if_data should be percpu. */
+	net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
-	ifp->if_obytes += m->m_pkthdr.len;
+	if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len);
 	if (m->m_flags & M_MCAST)
-		ifp->if_omcasts++;
+		if_statinc_ref(nsr, if_omcasts);
 	IF_STAT_PUTREF(ifp);
 	if (mutex_tryenter(txq->txq_lock)) {
 		if (!txq->txq_stopping)
 			wm_transmit_locked(ifp, txq);
 		mutex_exit(txq->txq_lock);
+	}
 	return 0;
+}
 static void
 wm_transmit_locked(struct ifnet *ifp, struct wm_txqueue *txq)
+{
 @@ -8157,27 +8160,27 @@ wm_nq_tx_offload(struct wm_softc *sc, st
+ *
  *	Start packet transmission on the interface for NEWQUEUE devices
  */
 static void
 wm_nq_start(struct ifnet *ifp)
+{
 	struct wm_softc *sc = ifp->if_softc;
 	struct wm_txqueue *txq = &sc->sc_queue[0].wmq_txq;
 #ifdef WM_MPSAFE
 	KASSERT(if_is_mpsafe(ifp));
 #endif
 	/*
-	 * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c.
+	 * if_obytes and if_omcasts are added in if_transmit()@if.c.
 	 */
 	mutex_enter(txq->txq_lock);
 	if (!txq->txq_stopping)
 		wm_nq_start_locked(ifp);
 	mutex_exit(txq->txq_lock);
+}
 static void
 wm_nq_start_locked(struct ifnet *ifp)
+{
 	struct wm_softc *sc = ifp->if_softc;
 	struct wm_txqueue *txq = &sc->sc_queue[0].wmq_txq;
 @@ -8191,30 +8194,31 @@ wm_nq_transmit(struct ifnet *ifp, struct
 	int qid;
 	struct wm_softc *sc = ifp->if_softc;
 	struct wm_txqueue *txq;
 	qid = wm_select_txqueue(ifp, m);
 	txq = &sc->sc_queue[qid].wmq_txq;
 	if (__predict_false(!pcq_put(txq->txq_interq, m))) {
 		m_freem(m);
 		WM_Q_EVCNT_INCR(txq, pcqdrop);
 		return ENOBUFS;
+	}
-	/* XXX NOMPSAFE: ifp->if_data should be percpu. */
+	net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
-	ifp->if_obytes += m->m_pkthdr.len;
+	if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len);
 	if (m->m_flags & M_MCAST)
-		ifp->if_omcasts++;
+		if_statinc_ref(nsr, if_omcasts);
 	IF_STAT_PUTREF(ifp);
 	/*
 	 * The situations which this mutex_tryenter() fails at running time
 	 * are below two patterns.
 	 *     (1) contention with interrupt handler(wm_txrxintr_msix())
 	 *     (2) contention with deferred if_start softint(wm_handle_queue())
 	 * In the case of (1), the last packet enqueued to txq->txq_interq is
 	 * dequeued by wm_deferred_start_locked(). So, it does not get stuck.
 	 * In the case of (2), the last packet enqueued to txq->txq_interq is
 	 * also dequeued by wm_deferred_start_locked(). So, it does not get
 	 * stuck, either.
 	 */
 	if (mutex_tryenter(txq->txq_lock)) {
 @@ -8639,38 +8643,38 @@ wm_txeof(struct wm_txqueue *txq, u_int l
 		 * 82574 and newer's document says the status field has neither
 		 * EC (Excessive Collision) bit nor LC (Late Collision) bit
 		 * (reserved). Refer "PCIe GbE Controller Open Source Software
 		 * Developer's Manual", 82574 datasheet and newer.
+		 *
 		 * XXX I saw the LC bit was set on I218 even though the media
 		 * was full duplex, so the bit might be used for other
 		 * meaning ...(I have no document).
 		 */
 		if (((status & (WTX_ST_EC | WTX_ST_LC)) != 0)
 		    && ((sc->sc_type < WM_T_82574)
 			|| (sc->sc_type == WM_T_80003))) {
-			ifp->if_oerrors++;
+			if_statinc(ifp, if_oerrors);
 			if (status & WTX_ST_LC)
 				log(LOG_WARNING, "%s: late collision\n",
 				    device_xname(sc->sc_dev));
 			else if (status & WTX_ST_EC) {
-				ifp->if_collisions +=
+				if_statadd(ifp, if_collisions,
-				    TX_COLLISION_THRESHOLD + 1;
+				    TX_COLLISION_THRESHOLD + 1);
 				log(LOG_WARNING, "%s: excessive collisions\n",
 				    device_xname(sc->sc_dev));
+			}
 		} else
-			ifp->if_opackets++;
+			if_statinc(ifp, if_opackets);
 		txq->txq_packets++;
 		txq->txq_bytes += txs->txs_mbuf->m_pkthdr.len;
 		txq->txq_free += txs->txs_ndesc;
 		bus_dmamap_sync(sc->sc_dmat, txs->txs_dmamap,
 , txs->txs_dmamap->dm_mapsize, BUS_DMASYNC_POSTWRITE);
 		bus_dmamap_unload(sc->sc_dmat, txs->txs_dmamap);
 		m_freem(txs->txs_mbuf);
 		txs->txs_mbuf = NULL;
+	}
 	/* Update the dirty transmit buffer pointer. */
 @@ -8972,27 +8976,27 @@ wm_rxeof(struct wm_rxqueue *rxq, u_int l
 		m = rxs->rxs_mbuf;
 		/*
 		 * Add a new receive buffer to the ring, unless of
 		 * course the length is zero. Treat the latter as a
 		 * failed mapping.
 		 */
 		if ((len == 0) || (wm_add_rxbuf(rxq, i) != 0)) {
 			/*
 			 * Failed, throw away what we've done so
 			 * far, and discard the rest of the packet.
 			 */
-			ifp->if_ierrors++;
+			if_statinc(ifp, if_ierrors);
 			bus_dmamap_sync(sc->sc_dmat, rxs->rxs_dmamap, 0,
 			    rxs->rxs_dmamap->dm_mapsize, BUS_DMASYNC_PREREAD);
 			wm_init_rxdesc(rxq, i);
 			if (!wm_rxdesc_is_eop(rxq, status))
 				rxq->rxq_discard = 1;
 			if (rxq->rxq_head != NULL)
 				m_freem(rxq->rxq_head);
 			WM_RXCHAIN_RESET(rxq);
 			DPRINTF(WM_DEBUG_RX,
 			    ("%s: RX: Rx buffer allocation failed, "
 			    "dropping packet%s\n", device_xname(sc->sc_dev),
 				rxq->rxq_discard ? " (discard)" : ""));
 			continue;