 @@ -1,35 +1,35 @@
-/*	$NetBSD: if_vmx.c,v 1.48 2019/08/19 05:25:38 knakahara Exp $	*/
+/*	$NetBSD: if_vmx.c,v 1.49 2019/08/30 05:03:32 knakahara Exp $	*/
 /*	$OpenBSD: if_vmx.c,v 1.16 2014/01/22 06:04:17 brad Exp $	*/
 /*
  * Copyright (c) 2013 Tsubai Masanari
  * Copyright (c) 2013 Bryan Venteicher <bryanv@FreeBSD.org>
+ *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
  * copyright notice and this permission notice appear in all copies.
+ *
  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1.48 2019/08/19 05:25:38 knakahara Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_vmx.c,v 1.49 2019/08/30 05:03:32 knakahara Exp $");
 #include <sys/param.h>
 #include <sys/cpu.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/bus.h>
 #include <sys/device.h>
 #include <sys/mbuf.h>
 #include <sys/sockio.h>
 #include <sys/pcq.h>
 #include <sys/workqueue.h>
 #include <sys/interrupt.h>
 @@ -197,70 +197,67 @@ struct vmxnet3_txqueue {
 	struct vmxnet3_comp_ring vxtxq_comp_ring;
 	struct vmxnet3_txq_stats vxtxq_stats;
 	struct vmxnet3_txq_shared *vxtxq_ts;
 	char vxtxq_name[16];
 	void *vxtxq_si;
 	struct evcnt vxtxq_intr;
 	struct evcnt vxtxq_defer;
 	struct evcnt vxtxq_deferreq;
 	struct evcnt vxtxq_pcqdrop;
 	struct evcnt vxtxq_transmitdef;
 	struct evcnt vxtxq_watchdogto;
 	struct evcnt vxtxq_defragged;
 	struct evcnt vxtxq_defrag_failed;
 };
 struct vmxnet3_rxq_stats {
 	uint64_t vmrxs_ipackets;	/* if_ipackets */
 	uint64_t vmrxs_ibytes;		/* if_ibytes */
 	uint64_t vmrxs_iqdrops;		/* if_iqdrops */
 	uint64_t vmrxs_ierrors;		/* if_ierrors */
 };
 struct vmxnet3_rxqueue {
 	kmutex_t *vxrxq_mtx;
 	struct vmxnet3_softc *vxrxq_sc;
 	struct mbuf *vxrxq_mhead;
 	struct mbuf *vxrxq_mtail;
 	struct vmxnet3_rxring vxrxq_cmd_ring[VMXNET3_RXRINGS_PERQ];
 	struct vmxnet3_comp_ring vxrxq_comp_ring;
 	struct vmxnet3_rxq_stats vxrxq_stats;
 	struct vmxnet3_rxq_shared *vxrxq_rs;
 	char vxrxq_name[16];
 	struct evcnt vxrxq_intr;
 	struct evcnt vxrxq_defer;
 	struct evcnt vxrxq_deferreq;
 	struct evcnt vxrxq_mgetcl_failed;
 	struct evcnt vxrxq_mbuf_load_failed;
 };
 struct vmxnet3_queue {
 	int vxq_id;
 	int vxq_intr_idx;
 	struct vmxnet3_txqueue vxq_txqueue;
 	struct vmxnet3_rxqueue vxq_rxqueue;
 	void *vxq_si;
 	bool vxq_workqueue;
 	struct work vxq_wq_cookie;
 };
 struct vmxnet3_statistics {
 	uint32_t vmst_defragged;
 	uint32_t vmst_defrag_failed;
 	uint32_t vmst_mgetcl_failed;
 	uint32_t vmst_mbuf_load_failed;
 };
 struct vmxnet3_softc {
 	device_t vmx_dev;
 	struct ethercom vmx_ethercom;
 	struct ifmedia vmx_media;
 	struct vmxnet3_driver_shared *vmx_ds;
 	int vmx_flags;
 #define VMXNET3_FLAG_NO_MSIX	(1 << 0)
 #define VMXNET3_FLAG_RSS	(1 << 1)
 #define VMXNET3_FLAG_ATTACHED	(1 << 2)
 	struct vmxnet3_queue *vmx_queue;
 	struct pci_attach_args *vmx_pa;
 @@ -271,27 +268,26 @@ struct vmxnet3_softc {
 	bus_space_handle_t vmx_ioh0;
 	bus_space_handle_t vmx_ioh1;
 	bus_size_t vmx_ios0;
 	bus_size_t vmx_ios1;
 	bus_dma_tag_t vmx_dmat;
 	int vmx_link_active;
 	int vmx_ntxqueues;
 	int vmx_nrxqueues;
 	int vmx_ntxdescs;
 	int vmx_nrxdescs;
 	int vmx_max_rxsegs;
 	struct vmxnet3_statistics vmx_stats;
 	struct evcnt vmx_event_intr;
 	struct evcnt vmx_event_link;
 	struct evcnt vmx_event_txqerror;
 	struct evcnt vmx_event_rxqerror;
 	struct evcnt vmx_event_dic;
 	struct evcnt vmx_event_debug;
 	int vmx_intr_type;
 	int vmx_intr_mask_mode;
 	int vmx_event_intr_idx;
 	int vmx_nintrs;
 	pci_intr_handle_t *vmx_intrs;	/* legacy use vmx_intrs[0] */
 	void *vmx_ihs[VMXNET3_MAX_INTRS];
 @@ -385,27 +381,28 @@ void vmxnet3_free_rxq_data(struct vmxnet
 int vmxnet3_alloc_queue_data(struct vmxnet3_softc *);
 void vmxnet3_free_queue_data(struct vmxnet3_softc *);
 int vmxnet3_alloc_mcast_table(struct vmxnet3_softc *);
 void vmxnet3_free_mcast_table(struct vmxnet3_softc *);
 void vmxnet3_init_shared_data(struct vmxnet3_softc *);
 void vmxnet3_reinit_rss_shared_data(struct vmxnet3_softc *);
 void vmxnet3_reinit_shared_data(struct vmxnet3_softc *);
 int vmxnet3_alloc_data(struct vmxnet3_softc *);
 void vmxnet3_free_data(struct vmxnet3_softc *);
 int vmxnet3_setup_interface(struct vmxnet3_softc *);
 void vmxnet3_evintr(struct vmxnet3_softc *);
 bool vmxnet3_txq_eof(struct vmxnet3_txqueue *, u_int);
-int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxring *);
+int vmxnet3_newbuf(struct vmxnet3_softc *, struct vmxnet3_rxqueue *,
     struct vmxnet3_rxring *);
 void vmxnet3_rxq_eof_discard(struct vmxnet3_rxqueue *,
     struct vmxnet3_rxring *, int);
 void vmxnet3_rxq_discard_chain(struct vmxnet3_rxqueue *);
 void vmxnet3_rx_csum(struct vmxnet3_rxcompdesc *, struct mbuf *);
 void vmxnet3_rxq_input(struct vmxnet3_rxqueue *,
     struct vmxnet3_rxcompdesc *, struct mbuf *);
 bool vmxnet3_rxq_eof(struct vmxnet3_rxqueue *, u_int);
 int vmxnet3_legacy_intr(void *);
 int vmxnet3_txrxq_intr(void *);
 void vmxnet3_handle_queue(void *);
 void vmxnet3_handle_queue_work(struct work *, void *);
 int vmxnet3_event_intr(void *);
 @@ -1961,37 +1958,45 @@ vmxnet3_setup_stats(struct vmxnet3_softc
 		txq = &vmxq->vxq_txqueue;
 		evcnt_attach_dynamic(&txq->vxtxq_intr, EVCNT_TYPE_INTR,
 		    NULL, txq->vxtxq_name, "Interrupt on queue");
 		evcnt_attach_dynamic(&txq->vxtxq_defer, EVCNT_TYPE_MISC,
 		    NULL, txq->vxtxq_name, "Handled queue in softint/workqueue");
 		evcnt_attach_dynamic(&txq->vxtxq_deferreq, EVCNT_TYPE_MISC,
 		    NULL, txq->vxtxq_name, "Requested in softint/workqueue");
 		evcnt_attach_dynamic(&txq->vxtxq_pcqdrop, EVCNT_TYPE_MISC,
 		    NULL, txq->vxtxq_name, "Dropped in pcq");
 		evcnt_attach_dynamic(&txq->vxtxq_transmitdef, EVCNT_TYPE_MISC,
 		    NULL, txq->vxtxq_name, "Deferred transmit");
 		evcnt_attach_dynamic(&txq->vxtxq_watchdogto, EVCNT_TYPE_MISC,
 		    NULL, txq->vxtxq_name, "Watchdog timeount");
 		evcnt_attach_dynamic(&txq->vxtxq_defragged, EVCNT_TYPE_MISC,
 		    NULL, txq->vxtxq_name, "m_defrag sucessed");
 		evcnt_attach_dynamic(&txq->vxtxq_defrag_failed, EVCNT_TYPE_MISC,
 		    NULL, txq->vxtxq_name, "m_defrag failed");
+	}
 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
 		vmxq = &sc->vmx_queue[i];
 		rxq = &vmxq->vxq_rxqueue;
 		evcnt_attach_dynamic(&rxq->vxrxq_intr, EVCNT_TYPE_INTR,
 		    NULL, rxq->vxrxq_name, "Interrupt on queue");
 		evcnt_attach_dynamic(&rxq->vxrxq_defer, EVCNT_TYPE_MISC,
 		    NULL, rxq->vxrxq_name, "Handled queue in softint/workqueue");
 		evcnt_attach_dynamic(&rxq->vxrxq_deferreq, EVCNT_TYPE_MISC,
 		    NULL, rxq->vxrxq_name, "Requested in softint/workqueue");
 		evcnt_attach_dynamic(&rxq->vxrxq_mgetcl_failed, EVCNT_TYPE_MISC,
 		    NULL, rxq->vxrxq_name, "MCLGET failed");
 		evcnt_attach_dynamic(&rxq->vxrxq_mbuf_load_failed, EVCNT_TYPE_MISC,
 		    NULL, rxq->vxrxq_name, "bus_dmamap_load_mbuf failed");
+	}
 	evcnt_attach_dynamic(&sc->vmx_event_intr, EVCNT_TYPE_INTR,
 	    NULL, device_xname(sc->vmx_dev), "Interrupt for other events");
 	evcnt_attach_dynamic(&sc->vmx_event_link, EVCNT_TYPE_MISC,
 	    NULL, device_xname(sc->vmx_dev), "Link status event");
 	evcnt_attach_dynamic(&sc->vmx_event_txqerror, EVCNT_TYPE_MISC,
 	    NULL, device_xname(sc->vmx_dev), "Tx queue error event");
 	evcnt_attach_dynamic(&sc->vmx_event_rxqerror, EVCNT_TYPE_MISC,
 	    NULL, device_xname(sc->vmx_dev), "Rx queue error event");
 	evcnt_attach_dynamic(&sc->vmx_event_dic, EVCNT_TYPE_MISC,
 	    NULL, device_xname(sc->vmx_dev), "Device impl change event");
 	evcnt_attach_dynamic(&sc->vmx_event_debug, EVCNT_TYPE_MISC,
 @@ -2007,34 +2012,38 @@ vmxnet3_teardown_stats(struct vmxnet3_so
 	struct vmxnet3_txqueue *txq;
 	struct vmxnet3_rxqueue *rxq;
 	int i;
 	for (i = 0; i < sc->vmx_ntxqueues; i++) {
 		vmxq = &sc->vmx_queue[i];
 		txq = &vmxq->vxq_txqueue;
 		evcnt_detach(&txq->vxtxq_intr);
 		evcnt_detach(&txq->vxtxq_defer);
 		evcnt_detach(&txq->vxtxq_deferreq);
 		evcnt_detach(&txq->vxtxq_pcqdrop);
 		evcnt_detach(&txq->vxtxq_transmitdef);
 		evcnt_detach(&txq->vxtxq_watchdogto);
 		evcnt_detach(&txq->vxtxq_defragged);
 		evcnt_detach(&txq->vxtxq_defrag_failed);
+	}
 	for (i = 0; i < sc->vmx_nrxqueues; i++) {
 		vmxq = &sc->vmx_queue[i];
 		rxq = &vmxq->vxq_rxqueue;
 		evcnt_detach(&rxq->vxrxq_intr);
 		evcnt_detach(&rxq->vxrxq_defer);
 		evcnt_detach(&rxq->vxrxq_deferreq);
 		evcnt_detach(&rxq->vxrxq_mgetcl_failed);
 		evcnt_detach(&rxq->vxrxq_mbuf_load_failed);
+	}
 	evcnt_detach(&sc->vmx_event_intr);
 	evcnt_detach(&sc->vmx_event_link);
 	evcnt_detach(&sc->vmx_event_txqerror);
 	evcnt_detach(&sc->vmx_event_rxqerror);
 	evcnt_detach(&sc->vmx_event_dic);
 	evcnt_detach(&sc->vmx_event_debug);
+}
 void
 vmxnet3_evintr(struct vmxnet3_softc *sc)
+{
 @@ -2144,64 +2153,65 @@ vmxnet3_txq_eof(struct vmxnet3_txqueue *
 			txb->vtxb_m = NULL;
+		}
 		txr->vxtxr_next = (txcd->eop_idx + 1) % txr->vxtxr_ndesc;
+	}
 	if (txr->vxtxr_head == txr->vxtxr_next)
 		txq->vxtxq_watchdog = 0;
 	return more;
+}
 int
-vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxring *rxr)
+vmxnet3_newbuf(struct vmxnet3_softc *sc, struct vmxnet3_rxqueue *rxq,
     struct vmxnet3_rxring *rxr)
+{
 	struct mbuf *m;
 	struct vmxnet3_rxdesc *rxd;
 	struct vmxnet3_rxbuf *rxb;
 	bus_dma_tag_t tag;
 	bus_dmamap_t dmap;
 	int idx, btype, error;
 	tag = sc->vmx_dmat;
 	dmap = rxr->vxrxr_spare_dmap;
 	idx = rxr->vxrxr_fill;
 	rxd = &rxr->vxrxr_rxd[idx];
 	rxb = &rxr->vxrxr_rxbuf[idx];
 	/* Don't allocate buffers for ring 2 for now. */
 	if (rxr->vxrxr_rid != 0)
 		return -1;
 	btype = VMXNET3_BTYPE_HEAD;
 	MGETHDR(m, M_DONTWAIT, MT_DATA);
 	if (m == NULL)
 		return (ENOBUFS);
 	MCLGET(m, M_DONTWAIT);
 	if ((m->m_flags & M_EXT) == 0) {
-		sc->vmx_stats.vmst_mgetcl_failed++;
+		rxq->vxrxq_mgetcl_failed.ev_count++;
 		m_freem(m);
 		return (ENOBUFS);
+	}
 	m->m_pkthdr.len = m->m_len = JUMBO_LEN;
 	m_adj(m, ETHER_ALIGN);
 	error = bus_dmamap_load_mbuf(sc->vmx_dmat, dmap, m, BUS_DMA_NOWAIT);
 	if (error) {
 		m_freem(m);
-		sc->vmx_stats.vmst_mbuf_load_failed++;
+		rxq->vxrxq_mbuf_load_failed.ev_count++;
 		return (error);
+	}
 	if (rxb->vrxb_m != NULL) {
 		bus_dmamap_sync(tag, rxb->vrxb_dmamap,
 , rxb->vrxb_dmamap->dm_mapsize,
 		    BUS_DMASYNC_POSTREAD);
 		bus_dmamap_unload(tag, rxb->vrxb_dmamap);
+	}
 	rxr->vxrxr_spare_dmap = rxb->vrxb_dmamap;
 	rxb->vrxb_dmamap = dmap;
 	rxb->vrxb_m = m;
 @@ -2393,46 +2403,46 @@ vmxnet3_rxq_eof(struct vmxnet3_rxqueue *
 			/* start of frame w/o head buffer */
 			KASSERT(rxd->btype == VMXNET3_BTYPE_HEAD);
 			/* start of frame not in ring 0 */
 			KASSERT(rxr == &rxq->vxrxq_cmd_ring[0]);
 			/* duplicate start of frame? */
 			KASSERT(m_head == NULL);
 			if (length == 0) {
 				/* Just ignore this descriptor. */
 				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
 				goto nextp;
+			}
-			if (vmxnet3_newbuf(sc, rxr) != 0) {
+			if (vmxnet3_newbuf(sc, rxq, rxr) != 0) {
 				rxq->vxrxq_stats.vmrxs_iqdrops++;
 				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
 				if (!rxcd->eop)
 					vmxnet3_rxq_discard_chain(rxq);
 				goto nextp;
+			}
 			m_set_rcvif(m, ifp);
 			m->m_pkthdr.len = m->m_len = length;
 			m->m_pkthdr.csum_flags = 0;
 			m_head = m_tail = m;
 		} else {
 			/* non start of frame w/o body buffer */
 			KASSERT(rxd->btype == VMXNET3_BTYPE_BODY);
 			/* frame not started? */
 			KASSERT(m_head != NULL);
-			if (vmxnet3_newbuf(sc, rxr) != 0) {
+			if (vmxnet3_newbuf(sc, rxq, rxr) != 0) {
 				rxq->vxrxq_stats.vmrxs_iqdrops++;
 				vmxnet3_rxq_eof_discard(rxq, rxr, idx);
 				if (!rxcd->eop)
 					vmxnet3_rxq_discard_chain(rxq);
 				m_freem(m_head);
 				m_head = m_tail = NULL;
 				goto nextp;
+			}
 			m->m_len = length;
 			m_head->m_pkthdr.len += length;
 			m_tail->m_next = m;
 			m_tail = m;
 @@ -2774,27 +2784,27 @@ vmxnet3_rxinit(struct vmxnet3_softc *sc,
 	int i, populate, idx, error;
 	/* LRO and jumbo frame is not supported yet */
 	populate = 1;
 	for (i = 0; i < populate; i++) {
 		rxr = &rxq->vxrxq_cmd_ring[i];
 		rxr->vxrxr_fill = 0;
 		rxr->vxrxr_gen = VMXNET3_INIT_GEN;
 		memset(rxr->vxrxr_rxd, 0,
 		    rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
 		for (idx = 0; idx < rxr->vxrxr_ndesc; idx++) {
-			error = vmxnet3_newbuf(sc, rxr);
+			error = vmxnet3_newbuf(sc, rxq, rxr);
 			if (error)
 				return (error);
+		}
+	}
 	for (/**/; i < VMXNET3_RXRINGS_PERQ; i++) {
 		rxr = &rxq->vxrxq_cmd_ring[i];
 		rxr->vxrxr_fill = 0;
 		rxr->vxrxr_gen = 0;
 		memset(rxr->vxrxr_rxd, 0,
 		    rxr->vxrxr_ndesc * sizeof(struct vmxnet3_rxdesc));
+	}
 @@ -3008,29 +3018,29 @@ vmxnet3_txq_load_mbuf(struct vmxnet3_txq
 	if (error == 0 || error != EFBIG)
 		return (error);
 	m = m_defrag(m, M_NOWAIT);
 	if (m != NULL) {
 		*m0 = m;
 		error = bus_dmamap_load_mbuf(tag, dmap, m, BUS_DMA_NOWAIT);
 	} else
 		error = ENOBUFS;
 	if (error) {
 		m_freem(*m0);
 		*m0 = NULL;
-		txq->vxtxq_sc->vmx_stats.vmst_defrag_failed++;
+		txq->vxtxq_defrag_failed.ev_count++;
 	} else
-		txq->vxtxq_sc->vmx_stats.vmst_defragged++;
+		txq->vxtxq_defragged.ev_count++;
 	return (error);
+}
 void
 vmxnet3_txq_unload_mbuf(struct vmxnet3_txqueue *txq, bus_dmamap_t dmap)
+{
 	bus_dmamap_unload(txq->vxtxq_sc->vmx_dmat, dmap);
+}
 int
 vmxnet3_txq_encap(struct vmxnet3_txqueue *txq, struct mbuf **m0)