 @@ -1,14 +1,14 @@
-/*	$NetBSD: if_vioif.c,v 1.101 2023/03/23 02:57:54 yamaguchi Exp $	*/
+/*	$NetBSD: if_vioif.c,v 1.102 2023/03/23 03:02:17 yamaguchi Exp $	*/
 /*
  * Copyright (c) 2020 The NetBSD Foundation, Inc.
  * Copyright (c) 2010 Minoura Makoto.
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
 @@ -17,27 +17,27 @@
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.101 2023/03/23 02:57:54 yamaguchi Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.102 2023/03/23 03:02:17 yamaguchi Exp $");
 #ifdef _KERNEL_OPT
 #include "opt_net_mpsafe.h"
 #endif
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/atomic.h>
 #include <sys/bus.h>
 #include <sys/condvar.h>
 #include <sys/device.h>
 #include <sys/evcnt.h>
 @@ -256,26 +256,27 @@ struct vioif_netqueue {
 	void			*netq_softint;
 	struct vioif_work	 netq_work;
 	bool			 netq_workqueue;
 	char			 netq_evgroup[32];
 	struct evcnt		 netq_mbuf_load_failed;
 	struct evcnt		 netq_enqueue_failed;
 	void			*netq_ctx;
 };
 struct vioif_tx_context {
 	bool			 txc_link_active;
 	bool			 txc_no_free_slots;
 	pcq_t			*txc_intrq;
 	void			*txc_deferred_transmit;
 	struct evcnt		 txc_defrag_failed;
 };
 struct vioif_rx_context {
 	struct evcnt		 rxc_mbuf_enobufs;
 };
 struct vioif_ctrlqueue {
 	struct virtqueue		*ctrlq_vq;
 	enum {
 		FREE, INUSE, DONE
 @@ -720,27 +721,26 @@ vioif_init(struct ifnet *ifp)
 	virtio_reinit_end(vsc);
 	if (sc->sc_has_ctrl)
 		virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
 	r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs);
 	if (r == 0)
 		sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs;
 	else
 		sc->sc_act_nvq_pairs = 1;
 	SET(ifp->if_flags, IFF_RUNNING);
 	CLR(ifp->if_flags, IFF_OACTIVE);
 	vioif_net_intr_enable(sc, vsc);
 	vioif_update_link_status(sc);
 	r = vioif_rx_filter(sc);
 	return r;
+}
 static void
 vioif_stop(struct ifnet *ifp, int disable)
+{
 	struct vioif_softc *sc = ifp->if_softc;
 @@ -850,27 +850,32 @@ vioif_transmit(struct ifnet *ifp, struct
 		mutex_exit(&netq->netq_lock);
+	}
 	return 0;
+}
 void
 vioif_watchdog(struct ifnet *ifp)
+{
 	struct vioif_softc *sc = ifp->if_softc;
 	struct vioif_netqueue *netq;
 	int i;
-	if (ifp->if_flags & IFF_RUNNING) {
+	if (ISSET(ifp->if_flags, IFF_RUNNING)) {
 		if (ISSET(ifp->if_flags, IFF_DEBUG)) {
 			log(LOG_DEBUG, "%s: watchdog timed out\n",
 			    ifp->if_xname);
+		}
 		for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
 			netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
 			mutex_enter(&netq->netq_lock);
 			if (!netq->netq_running_handle) {
 				netq->netq_running_handle = true;
 				vioif_net_sched_handle(sc, netq);
+			}
 			mutex_exit(&netq->netq_lock);
+		}
+	}
+}
 @@ -1486,26 +1491,27 @@ vioif_netqueue_init(struct vioif_softc *
 		break;
 	case VIOIF_NETQ_TX:
 		txc = kmem_zalloc(sizeof(*txc), KM_SLEEP);
 		netq->netq_ctx = (void *)txc;
 		txc->txc_deferred_transmit = softint_establish(softint_flags,
 		    vioif_deferred_transmit, netq);
 		if (txc->txc_deferred_transmit == NULL) {
 			aprint_error_dev(sc->sc_dev,
 			    "couldn't establish softint for "
 			    "tx deferred transmit\n");
 			goto err;
+		}
 		txc->txc_link_active = VIOIF_IS_LINK_ACTIVE(sc);
 		txc->txc_no_free_slots = false;
 		txc->txc_intrq = pcq_create(vq->vq_num, KM_SLEEP);
 		break;
+	}
 	return 0;
 err:
 	netq->netq_ctx = NULL;
 	if (rxc != NULL) {
 		kmem_free(rxc, sizeof(*rxc));
+	}
 @@ -1961,38 +1967,37 @@ vioif_send_common_locked(struct ifnet *i
 	struct vioif_tx_context *txc;
 	struct vioif_net_map *map;
 	struct mbuf *m;
 	int queued = 0;
 	KASSERT(mutex_owned(&netq->netq_lock));
 	if (netq->netq_stopping ||
 	    !ISSET(ifp->if_flags, IFF_RUNNING))
 		return;
 	txc = netq->netq_ctx;
-	if (!txc->txc_link_active)
+	if (!txc->txc_link_active ||
-		return;
+	    txc->txc_no_free_slots)
 	if (!is_transmit &&
 	    ISSET(ifp->if_flags, IFF_OACTIVE))
 		return;
 	for (;;) {
 		int slot, r;
 		r = virtio_enqueue_prep(vsc, vq, &slot);
-		if (r == EAGAIN)
+		if (r == EAGAIN) {
 			txc->txc_no_free_slots = true;
 			break;
+		}
 		if (__predict_false(r != 0))
 			panic("enqueue_prep for tx buffers");
 		if (is_transmit)
 			m = pcq_get(txc->txc_intrq);
 		else
 			IFQ_DEQUEUE(&ifp->if_snd, m);
 		if (m == NULL) {
 			virtio_enqueue_abort(vsc, vq, slot);
 			break;
+		}
 @@ -2039,90 +2044,103 @@ vioif_send_common_locked(struct ifnet *i
 		queued++;
 		bpf_mtap(ifp, m, BPF_D_OUT);
+	}
 	if (queued > 0) {
 		vioif_notify(vsc, vq);
 		ifp->if_timer = 5;
+	}
+}
 /* dequeue sent mbufs */
 static bool
 vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
-    struct vioif_netqueue *netq, u_int limit)
+    struct vioif_netqueue *netq, u_int limit, size_t *ndeqp)
+{
 	struct virtqueue *vq = netq->netq_vq;
 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
 	struct vioif_net_map *map;
 	struct mbuf *m;
 	int slot, len;
-	bool more = false;
+	bool more;
 	size_t ndeq;
 	KASSERT(mutex_owned(&netq->netq_lock));
 	more = false;
 	ndeq = 0;
 	if (virtio_vq_is_enqueued(vsc, vq) == false)
-		return false;
+		goto done;
-	for (;;) {
+	for (;;ndeq++) {
 		if (limit-- == 0) {
 			more = true;
 			break;
+		}
 		if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
 			break;
 		map = &netq->netq_maps[slot];
 		KASSERT(map->vnm_mbuf != NULL);
 		m = vioif_net_dequeue_commit(vsc, vq, slot,
 		    map, BUS_DMASYNC_POSTWRITE);
 		KASSERT(m != NULL);
 		if_statinc(ifp, if_opackets);
 		m_freem(m);
+	}
 done:
 	if (ndeqp != NULL)
 		*ndeqp = ndeq;
 	return more;
+}
 static void
 vioif_tx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc,
     struct vioif_netqueue *netq)
+{
 	struct vioif_tx_context *txc;
 	struct vioif_net_map *map;
 	struct mbuf *m;
 	unsigned int i, vq_num;
 	bool more;
 	mutex_enter(&netq->netq_lock);
 	txc = netq->netq_ctx;
 	vq_num = netq->netq_vq->vq_num;
 	for (;;) {
-		more = vioif_tx_deq_locked(sc, vsc, netq, vq_num);
+		more = vioif_tx_deq_locked(sc, vsc, netq, vq_num, NULL);
 		if (more == false)
 			break;
+	}
 	for (i = 0; i < vq_num; i++) {
 		map = &netq->netq_maps[i];
 		m = map->vnm_mbuf;
 		if (m == NULL)
 			continue;
 		vioif_net_unload_mbuf(vsc, map);
 		m_freem(m);
+	}
 	txc->txc_no_free_slots = false;
 	mutex_exit(&netq->netq_lock);
+}
 static void
 vioif_start_locked(struct ifnet *ifp, struct vioif_netqueue *netq)
+{
 	/*
 	 * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c.
 	 */
 	vioif_send_common_locked(ifp, netq, false);
+}
 @@ -2147,52 +2165,57 @@ vioif_deferred_transmit(void *arg)
 	mutex_exit(&netq->netq_lock);
+}
 static void
 vioif_tx_handle_locked(struct vioif_netqueue *netq, u_int limit)
+{
 	struct virtqueue *vq = netq->netq_vq;
 	struct vioif_tx_context *txc = netq->netq_ctx;
 	struct virtio_softc *vsc = vq->vq_owner;
 	struct vioif_softc *sc = device_private(virtio_child(vsc));
 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
 	bool more;
 	int enqueued;
 	size_t ndeq;
 	KASSERT(mutex_owned(&netq->netq_lock));
 	KASSERT(!netq->netq_stopping);
-	more = vioif_tx_deq_locked(sc, vsc, netq, limit);
+	more = vioif_tx_deq_locked(sc, vsc, netq, limit, &ndeq);
 	if (txc->txc_no_free_slots && ndeq > 0) {
 		txc->txc_no_free_slots = false;
 		softint_schedule(txc->txc_deferred_transmit);
+	}
 	if (more) {
 		vioif_net_sched_handle(sc, netq);
 		return;
+	}
 	enqueued = (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX) ?
 	    virtio_postpone_intr_smart(vsc, vq):
 	    virtio_start_vq_intr(vsc, vq);
 	if (enqueued != 0) {
 		virtio_stop_vq_intr(vsc, vq);
 		vioif_net_sched_handle(sc, netq);
 		return;
+	}
 	netq->netq_running_handle = false;
 	/* for ALTQ */
-	if (netq == &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]) {
+	if (netq == &sc->sc_netqs[VIOIF_NETQ_TXQID(0)])
 		if_schedule_deferred_start(ifp);
 		ifp->if_flags &= ~IFF_OACTIVE;
 	softint_schedule(txc->txc_deferred_transmit);
+}
 static int
 vioif_tx_intr(void *arg)
+{
 	struct vioif_netqueue *netq = arg;
 	struct virtqueue *vq = netq->netq_vq;
 	struct virtio_softc *vsc = vq->vq_owner;
 	struct vioif_softc *sc = device_private(virtio_child(vsc));
 	u_int limit;
 	mutex_enter(&netq->netq_lock);