 @@ -1,14 +1,14 @@
-/*	$NetBSD: vnet.c,v 1.3 2021/03/11 19:34:11 palle Exp $	*/
+/*	$NetBSD: vnet.c,v 1.4 2021/03/13 20:21:37 palle Exp $	*/
 /*	$OpenBSD: vnet.c,v 1.62 2020/07/10 13:26:36 patrick Exp $	*/
 /*
  * Copyright (c) 2009, 2015 Mark Kettenis
+ *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
  * copyright notice and this permission notice appear in all copies.
+ *
  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 @@ -211,26 +211,27 @@ void vnet_attach (device_t, device_t, vo
 CFATTACH_DECL_NEW(vnet, sizeof(struct vnet_softc),
     vnet_match, vnet_attach, NULL, NULL);
 int	vnet_tx_intr(void *);
 int	vnet_rx_intr(void *);
 void	vnet_handshake(void *);
 void	vio_rx_data(struct ldc_conn *, struct ldc_pkt *);
 void	vnet_rx_vio_ctrl(struct vnet_softc *, struct vio_msg *);
 void	vnet_rx_vio_ver_info(struct vnet_softc *, struct vio_msg_tag *);
 void	vnet_rx_vio_attr_info(struct vnet_softc *, struct vio_msg_tag *);
 void	vnet_rx_vio_dring_reg(struct vnet_softc *, struct vio_msg_tag *);
 void	vnet_rx_vio_rdx(struct vnet_softc *sc, struct vio_msg_tag *);
 void	vnet_rx_vio_mcast_info(struct vnet_softc *sc, struct vio_msg_tag *);
 void	vnet_rx_vio_data(struct vnet_softc *sc, struct vio_msg *);
 void	vnet_rx_vio_desc_data(struct vnet_softc *sc, struct vio_msg_tag *);
 void	vnet_rx_vio_dring_data(struct vnet_softc *sc, struct vio_msg_tag *);
 void	vnet_ldc_reset(struct ldc_conn *);
 void	vnet_ldc_start(struct ldc_conn *);
 void	vnet_sendmsg(struct vnet_softc *, void *, size_t);
 void	vnet_send_ver_info(struct vnet_softc *, uint16_t, uint16_t);
 void	vnet_send_attr_info(struct vnet_softc *);
 void	vnet_send_dring_reg(struct vnet_softc *);
 void	vio_send_rdx(struct vnet_softc *);
 void	vnet_send_dring_data(struct vnet_softc *, uint32_t);
 @@ -645,29 +646,35 @@ vnet_rx_vio_ctrl(struct vnet_softc *sc,
 	switch (tag->stype_env) {
 	case VIO_VER_INFO:
 		vnet_rx_vio_ver_info(sc, tag);
 		break;
 	case VIO_ATTR_INFO:
 		vnet_rx_vio_attr_info(sc, tag);
 		break;
 	case VIO_DRING_REG:
 		vnet_rx_vio_dring_reg(sc, tag);
 		break;
 	case VIO_RDX:
 		vnet_rx_vio_rdx(sc, tag);
 		break;
 	case VNET_MCAST_INFO:
 		vnet_rx_vio_mcast_info(sc, tag);
 		break;
 	default:
 		printf("%s: CTRL/0x%02x/0x%04x FIXME\n",
 				 __func__, tag->stype, tag->stype_env);
 #if 0
 		Debugger();
 #endif
 		break;
+	}
+}
 void
 vnet_rx_vio_ver_info(struct vnet_softc *sc, struct vio_msg_tag *tag)
+{
 #if 0
 	DPRINTF(("%s: entry\n", __func__));
 #endif
 	struct vio_ver_info *vi = (struct vio_ver_info *)tag;
 	switch (vi->tag.stype) {
 @@ -729,27 +736,29 @@ void
 vnet_rx_vio_attr_info(struct vnet_softc *sc, struct vio_msg_tag *tag)
+{
 #if 0
 	DPRINTF(("%s: entry\n", __func__));
 #endif
 	struct vnet_attr_info *ai = (struct vnet_attr_info *)tag;
 	switch (ai->tag.stype) {
 	case VIO_SUBTYPE_INFO:
 #if 0
 		DPRINTF(("CTRL/INFO/ATTR_INFO\n"));
 #endif
 		sc->sc_xfer_mode = ai->xfer_mode;
 #if 0
 		DPRINTF(("sc_xfer_mode %d\n", sc->sc_xfer_mode));
 #endif
 		ai->tag.stype = VIO_SUBTYPE_ACK;
 		ai->tag.sid = sc->sc_local_sid;
 		vnet_sendmsg(sc, ai, sizeof(*ai));
 		sc->sc_vio_state |= VIO_RCV_ATTR_INFO;
 		break;
 	case VIO_SUBTYPE_ACK:
 #if 0
 		DPRINTF(("CTRL/ACK/ATTR_INFO\n"));
 #endif
 		if (!ISSET(sc->sc_vio_state, VIO_SND_ATTR_INFO)) {
 			ldc_reset(&sc->sc_lc);
 @@ -883,26 +892,60 @@ FIXME openbsd
 	ifp->if_flags &= ~IFF_OACTIVE;
 #endif
 		vnet_start(ifp);
 #if 0
 FIXME openbsd
 		KERNEL_UNLOCK();
 #else
 	KERNEL_UNLOCK_ONE(curlwp);
 #endif
+	}
+}
 void
 vnet_rx_vio_mcast_info(struct vnet_softc *sc, struct vio_msg_tag *tag)
+{
 #if 0
 	DPRINTF(("%s: entry\n", __func__));
 #endif
 	switch(tag->stype) {
 		case VIO_SUBTYPE_INFO:
 #if 0
 			DPRINTF(("CTRL/INFO/MCAST_INFO\n"));
 #endif
 			break;
 		case VIO_SUBTYPE_ACK:
 #if 0
 			DPRINTF(("CTRL/ACK/MCAST_INFO\n"));
 #endif
 			break;
 		case VIO_SUBTYPE_NACK:
 #if 0
 			DPRINTF(("CTRL/NACK/MCAST_INFO\n"));
 #endif
 			break;
 		default:
 			printf("%s: CTRL/0x%02x/0x%04x\n",
 				   __func__, tag->stype, tag->stype_env);
 			break;
+	}
+}
 void
 vnet_rx_vio_data(struct vnet_softc *sc, struct vio_msg *vm)
+{
 #if 0
 	DPRINTF(("%s: entry\n", __func__));
 #endif
 	struct vio_msg_tag *tag = (struct vio_msg_tag *)&vm->type;
 	if (!ISSET(sc->sc_vio_state, VIO_RCV_RDX) ||
 	    !ISSET(sc->sc_vio_state, VIO_ACK_RDX)) {
 		DPRINTF(("Spurious DATA/0x%02x/0x%04x\n", tag->stype,
 		    tag->stype_env));
 		return;
+	}
 @@ -1067,27 +1110,27 @@ FIXME openbsd
 #else
 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
 #endif
 #if 1
 	struct mbuf *m = NULL;
 #endif
 	paddr_t pa;
 	psize_t nbytes;
 	int err;
 	switch(tag->stype) {
 	case VIO_SUBTYPE_INFO:
+	{
 #if 0
 		DPRINTF(("%s: VIO_SUBTYPE_INFO\n", __func__));
 #endif
 		struct vnet_desc desc;
 		uint64_t cookie;
 		paddr_t desc_pa;
 		int idx, ack_end_idx = -1;
 #if 0
 FIXME openbsd
 		struct mbuf_list ml = MBUF_LIST_INITIALIZER();
 #endif
 		idx = dm->start_idx;
 		for (;;) {
 @@ -1138,54 +1181,56 @@ FIXME openbsd
 #endif
 			MCLGET(m, M_DONTWAIT);
 			if ((m->m_flags & M_EXT) == 0)
 				break;
 #if 0
 			DPRINTF(("%s: after MCLGET\n", __func__));
 #endif
 #endif
 #endif
 #if 1
 			m->m_len = m->m_pkthdr.len = desc.nbytes;
 #endif
 			nbytes = roundup(desc.nbytes + VNET_ETHER_ALIGN, 8);
-#if 1
+#if 0
-			DPRINTF(("%s: nbytes %" PRId64 " desc.nbytes %" PRId32 "\n",
+			DPRINTF(("%s: RX frame size %" PRId16 "\n",
-					 __func__, nbytes, desc.nbytes));
+					 __func__, (int)nbytes));
 			uint8_t buf[ETHER_MAX_LEN];
 			pmap_extract(pmap_kernel(), (vaddr_t)buf, &pa);
 			err = hv_ldc_copy(lc->lc_id, LDC_COPY_IN,
 			    desc.cookie[0].addr, pa, nbytes, &nbytes);
 			if (err != H_EOK) {
 					//m_freem(m);
 				goto skip;
+			}
 			for (int i = 0; i < desc.nbytes; i++) {
 					if (i % 16 == 0) {
-						printf("\n");
+						DPRINTF(("\n"));
+					}
-					printf("%02x ", buf[i]);
+					DPRINTF(("%02x ", buf[i]));
+			}
-			printf("\n");
+			DPRINTF(("\n"));
 			DPRINTF(("\n"));
 #endif
 #if 1
 			pmap_extract(pmap_kernel(), (vaddr_t)m->m_data, &pa);
 			err = hv_ldc_copy(lc->lc_id, LDC_COPY_IN,
 			    desc.cookie[0].addr, pa, nbytes, &nbytes);
 			if (err != H_EOK) {
 				m_freem(m);
 				goto skip;
+			}
 			m->m_data += VNET_ETHER_ALIGN;
 			m_set_rcvif(m, ifp);
 #if 0
 FIXME openbsd
 			ml_enqueue(&ml, m);
 #else
 #if 0
 			DPRINTF(("%s: before if_percpuq_enqueue\n", __func__));
 #endif
 			if_percpuq_enqueue(ifp->if_percpuq, m);
 #if 0
 			DPRINTF(("%s: after if_percpuq_enqueue\n", __func__));
 #endif
 #endif
 @@ -1214,27 +1259,27 @@ FIXME openbd
 			dm->tag.stype = VIO_SUBTYPE_NACK;
 		} else {
 			dm->tag.stype = VIO_SUBTYPE_ACK;
 			dm->end_idx = ack_end_idx;
+		}
 		dm->tag.sid = sc->sc_local_sid;
 		dm->proc_state = VIO_DP_STOPPED;
 		vnet_sendmsg(sc, dm, sizeof(*dm));
 		break;
+	}
 	case VIO_SUBTYPE_ACK:
+	{
 #if 0
 		DPRINTF(("%s: VIO_SUBTYPE_ACK\n", __func__));
 #endif
 		struct ldc_map *map = sc->sc_lm;
 		u_int cons, count;
 		sc->sc_peer_state = dm->proc_state;
 		cons = sc->sc_tx_cons & (sc->sc_vd->vd_nentries - 1);
 		while (sc->sc_vd->vd_desc[cons].hdr.dstate == VIO_DESC_DONE) {
 			map->lm_slot[sc->sc_vsd[cons].vsd_map_idx].entry = 0;
 #if 0
 FIXME openbsd
 			atomic_dec_int(&map->lm_count);
 @@ -1389,97 +1434,112 @@ vnet_send_attr_info(struct vnet_softc *s
 	ai.tag.type = VIO_TYPE_CTRL;
 	ai.tag.stype = VIO_SUBTYPE_INFO;
 	ai.tag.stype_env = VIO_ATTR_INFO;
 	ai.tag.sid = sc->sc_local_sid;
 	ai.xfer_mode = VIO_DRING_MODE;
 	ai.addr_type = VNET_ADDR_ETHERMAC;
 	ai.ack_freq = 0;
 	ai.addr = 0;
 	for (i = 0; i < ETHER_ADDR_LEN; i++) {
 		ai.addr <<= 8;
 		ai.addr |= sc->sc_macaddr[i];
+	}
 	ai.mtu = ETHER_MAX_LEN - ETHER_CRC_LEN;
 #if 0
 	DPRINTF(("%s: ai.addr %" PRIx64 "\n",
 			 __func__, ai.addr));
 	DPRINTF(("%s: ai.mtu %" PRId64 "\n",
 			 __func__, ai.mtu));
 #endif
 	vnet_sendmsg(sc, &ai, sizeof(ai));
 	sc->sc_vio_state |= VIO_SND_ATTR_INFO;
+}
 void
 vnet_send_dring_reg(struct vnet_softc *sc)
+{
-#if 1
+#if 0
 	DPRINTF(("%s: entry\n", __func__));
 #endif
 	struct vio_dring_reg dr;
 	bzero(&dr, sizeof(dr));
 	dr.tag.type = VIO_TYPE_CTRL;
 	dr.tag.stype = VIO_SUBTYPE_INFO;
 	dr.tag.stype_env = VIO_DRING_REG;
 	dr.tag.sid = sc->sc_local_sid;
 	dr.dring_ident = 0;
 	dr.num_descriptors = sc->sc_vd->vd_nentries;
 	dr.descriptor_size = sizeof(struct vnet_desc);
 	dr.options = VIO_TX_RING;
 	dr.ncookies = 1;
 	dr.cookie[0].addr = 0;
 	dr.cookie[0].size = PAGE_SIZE;
 	vnet_sendmsg(sc, &dr, sizeof(dr));
 	sc->sc_vio_state |= VIO_SND_DRING_REG;
 };
 void
 vio_send_rdx(struct vnet_softc *sc)
+{
-#if 1
+#if 0
 	DPRINTF(("%s: entry\n", __func__));
 #endif
 	struct vio_msg_tag tag;
 	tag.type = VIO_TYPE_CTRL;
 	tag.stype = VIO_SUBTYPE_INFO;
 	tag.stype_env = VIO_RDX;
 	tag.sid = sc->sc_local_sid;
 	vnet_sendmsg(sc, &tag, sizeof(tag));
 	sc->sc_vio_state |= VIO_SND_RDX;
+}
 void
 vnet_send_dring_data(struct vnet_softc *sc, uint32_t start_idx)
+{
 #if 0
 	DPRINTF(("%s: entry\n", __func__));
 #endif
 	struct vio_dring_msg dm;
 	u_int peer_state;
 	peer_state = atomic_swap_uint(&sc->sc_peer_state, VIO_DP_ACTIVE);
-	if (peer_state == VIO_DP_ACTIVE)
+	if (peer_state == VIO_DP_ACTIVE) {
 		DPRINTF(("%s: peer_state == VIO_DP_ACTIVE\n", __func__));
 		return;
+	}
 	bzero(&dm, sizeof(dm));
 	dm.tag.type = VIO_TYPE_DATA;
 	dm.tag.stype = VIO_SUBTYPE_INFO;
 	dm.tag.stype_env = VIO_DRING_DATA;
 	dm.tag.sid = sc->sc_local_sid;
 	dm.seq_no = sc->sc_seq_no++;
 #if 0
 	DPRINTF(("%s: dm.seq_no %" PRId64 "\n",
 			 __func__, dm.seq_no));
 #endif
 	dm.dring_ident = sc->sc_dring_ident;
 	dm.start_idx = start_idx;
 	dm.end_idx = -1;
 	vnet_sendmsg(sc, &dm, sizeof(dm));
 #if 0
 	DPRINTF(("%s: exit\n", __func__));
 #endif
+}
 void
 vnet_start(struct ifnet *ifp)
+{
 #if 0
 	DPRINTF(("%s: entry\n", __func__));
 #endif
 	struct vnet_softc *sc = ifp->if_softc;
 	struct ldc_conn *lc = &sc->sc_lc;
 	struct ldc_map *map = sc->sc_lm;
 	struct mbuf *m;
 	paddr_t pa;
 @@ -1505,36 +1565,40 @@ FIXME openbsd
 #endif
 #if 0
 FIXME openbsd
 	if (ifq_empty(&ifp->if_snd))
 #else
 	if (IFQ_IS_EMPTY(&ifp->if_snd))
 #endif
+	{
 #if 0
 		DPRINTF(("%s: queue is empty\n", __func__));
 #endif
 		return;
 	} else {
 #if 0
 		DPRINTF(("%s: queue size %d\n", __func__, ifp->if_snd.ifq_len));
 #endif
+	}
 	/*
 	 * We cannot transmit packets until a VIO connection has been
 	 * established.
 	 */
 	if (!ISSET(sc->sc_vio_state, VIO_RCV_RDX) ||
 	    !ISSET(sc->sc_vio_state, VIO_ACK_RDX))
+	{
 #if 0
 		DPRINTF(("%s: vio connection not established yet\n", __func__));
 #endif
 		return;
+	}
 	/*
 	 * Make sure there is room in the LDC transmit queue to send a
 	 * DRING_DATA message.
 	 */
 	err = hv_ldc_tx_get_state(lc->lc_id, &tx_head, &tx_tail, &tx_state);
 	if (err != H_EOK) {
 		DPRINTF(("%s: no room in ldc transmit queue\n", __func__));
 		return;
 @@ -1553,69 +1617,106 @@ FIXME openbsd
 			return;
+		}
+	}
 	if (sc->sc_xfer_mode == VIO_DESC_MODE) {
 		DPRINTF(("%s: vio_desc_mode\n", __func__));
 		vnet_start_desc(ifp);
 		return;
+	}
 	start = prod = sc->sc_tx_prod & (sc->sc_vd->vd_nentries - 1);
 	while (sc->sc_vd->vd_desc[prod].hdr.dstate == VIO_DESC_FREE) {
 		count = sc->sc_tx_prod - sc->sc_tx_cons;
 #if 0
 		DPRINTF(("%s: count %d\n", __func__, count));
 #endif
 		if (count >= (sc->sc_vd->vd_nentries - 1) ||
 		    map->lm_count >= map->lm_nentries) {
 			DPRINTF(("%s: count issue\n", __func__));
 #if 0
 FIXME openbsd
 			ifq_set_oactive(&ifp->if_snd);
 #else
 			ifp->if_flags |= IFF_OACTIVE;
 #endif
 			break;
+		}
 		buf = pool_get(&sc->sc_pool, PR_NOWAIT|PR_ZERO);
 		if (buf == NULL) {
 			DPRINTF(("%s: buff is NULL\n", __func__));
 #if 0
 FIXME openbsd
 			ifq_set_oactive(&ifp->if_snd);
 #else
 			ifp->if_flags |= IFF_OACTIVE;
 #endif
 			break;
+		}
 #if 0
 FIXME openbsd
 		m = ifq_dequeue(&ifp->if_snd);
 #else
 		IFQ_DEQUEUE(&ifp->if_snd, m);
 #endif
 		if (m == NULL) {
 #if 0
 			DPRINTF(("%s: m is NULL\n", __func__));
 #endif
 			pool_put(&sc->sc_pool, buf);
 			break;
 		} else {
 #if 0
 			DPRINTF(("%s: m is not NULL\n", __func__));
 #endif
+		}
 #if 1
 		m_copydata(m, 0, m->m_pkthdr.len, buf + VNET_ETHER_ALIGN);
 #if 0
 		DPRINTF(("%s: TX frame size %" PRId16 "\n",
 				 __func__, m->m_pkthdr.len));
 		for (int i = 0; i < m->m_pkthdr.len; i++) {
 			if (i % 16 == 0) {
 				DPRINTF(("\n"));
+			}
 			DPRINTF(("%02x ", buf[VNET_ETHER_ALIGN+i]));
+		}
 		DPRINTF(("\n"));
 #endif
 #else
 		m_copydata(m, 0, m->m_pkthdr.len, buf);
 		DPRINTF(("%s: TX frame size %" PRId16 "\n",
 				 __func__, m->m_pkthdr.len));
 		for (int i = 0; i < m->m_pkthdr.len; i++) {
 			if (i % 16 == 0) {
 				DPRINTF(("\n"));
+			}
 			DPRINTF(("%02x ", buf[i]));
+		}
 		DPRINTF(("\n"));
 #endif
 #if NBPFILTER > 0
 		/*
 		 * If BPF is listening on this interface, let it see the
 		 * packet before we commit it to the wire.
 		 */
 		if (ifp->if_bpf)
+		{
 			DPRINTF(("%s: before bpf\n", __func__));
 			bpf_mtap(ifp->if_bpf, m, BPF_DIRECTION_OUT);
 			DPRINTF(("%s: after bpf\n", __func__));
+		}
 #endif
 		pmap_extract(pmap_kernel(), (vaddr_t)buf, &pa);
 		KASSERT((pa & ~PAGE_MASK) == (pa & LDC_MTE_RA_MASK));
 		while (map->lm_slot[map->lm_next].entry != 0) {
 			map->lm_next++;
 			map->lm_next &= (map->lm_nentries - 1);
+		}
 		map->lm_slot[map->lm_next].entry = (pa & LDC_MTE_RA_MASK);
 		map->lm_slot[map->lm_next].entry |= LDC_MTE_CPR;
 #if 0
 FIXME openbsd
 		atomic_inc_int(&map->lm_count);
 @@ -1637,30 +1738,36 @@ FIXME openbsd
 		sc->sc_vsd[prod].vsd_map_idx = map->lm_next;
 		sc->sc_vsd[prod].vsd_buf = buf;
 		sc->sc_tx_prod++;
 		prod = sc->sc_tx_prod & (sc->sc_vd->vd_nentries - 1);
 		m_freem(m);
+	}
 	membar_producer();
 	if (start != prod && sc->sc_peer_state != VIO_DP_ACTIVE) {
 #if 0
 		DPRINTF(("%s: before vnet_send_dring_data()\n", __func__));
 #endif
 		vnet_send_dring_data(sc, start);
 		ifp->if_timer = 5;
 	} else {
 		DPRINTF(("%s: no vnet_send_dring_data()\n", __func__));
+	}
 #if 0
 #if 0
 	DPRINTF(("%s: exit\n", __func__));
 #endif
+}
 void
 vnet_start_desc(struct ifnet *ifp)
+{
 	DPRINTF(("%s: entry\n", __func__));
 	struct vnet_softc *sc = ifp->if_softc;
 	struct ldc_map *map = sc->sc_lm;
 	struct vnet_desc_msg dm;
 @@ -1819,37 +1926,39 @@ void
 vnet_watchdog(struct ifnet *ifp)
+{
 	DPRINTF(("%s: entry\n", __func__));
 	struct vnet_softc *sc = ifp->if_softc;
 	printf("%s: watchdog timeout\n", sc->sc_dv.dv_xname);
+}
 int
 vnet_media_change(struct ifnet *ifp)
+{
 #if 0
 	DPRINTF(("%s: entry\n", __func__));
 #endif
 	return (0);
+}
 void
 vnet_media_status(struct ifnet *ifp, struct ifmediareq *imr)
+{
 #if 0
 	DPRINTF(("%s: entry\n", __func__));
 #endif
 	imr->ifm_active = IFM_ETHER | IFM_AUTO;
 	imr->ifm_status = IFM_AVALID;
 #if 0
 FIXME openbsd
 	if (LINK_STATE_IS_UP(ifp->if_link_state) &&
 #else
 	if (ifp->if_link_state == LINK_STATE_UP &&
 #endif
 	    ifp->if_flags & IFF_UP)
 		imr->ifm_status |= IFM_ACTIVE;
+}