 @@ -1,14 +1,14 @@
-/*	$NetBSD: if_tap.c,v 1.101 2017/10/30 16:01:19 ozaki-r Exp $	*/
+/*	$NetBSD: if_tap.c,v 1.102 2017/11/29 19:21:44 jmcneill Exp $	*/
 /*
  *  Copyright (c) 2003, 2004, 2008, 2009 The NetBSD Foundation.
  *  All rights reserved.
+ *
  *  Redistribution and use in source and binary forms, with or without
  *  modification, are permitted provided that the following conditions
  *  are met:
  *  1. Redistributions of source code must retain the above copyright
  *     notice, this list of conditions and the following disclaimer.
  *  2. Redistributions in binary form must reproduce the above copyright
  *     notice, this list of conditions and the following disclaimer in the
  *     documentation and/or other materials provided with the distribution.
 @@ -23,47 +23,48 @@
  *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  *  POSSIBILITY OF SUCH DAMAGE.
  */
 /*
  * tap(4) is a virtual Ethernet interface.  It appears as a real Ethernet
  * device to the system, but can also be accessed by userland through a
  * character device interface, which allows reading and injecting frames.
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_tap.c,v 1.101 2017/10/30 16:01:19 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_tap.c,v 1.102 2017/11/29 19:21:44 jmcneill Exp $");
 #if defined(_KERNEL_OPT)
 #include "opt_modular.h"
 #include "opt_compat_netbsd.h"
 #endif
 #include <sys/param.h>
 #include <sys/atomic.h>
 #include <sys/conf.h>
 #include <sys/cprng.h>
 #include <sys/device.h>
 #include <sys/file.h>
 #include <sys/filedesc.h>
 #include <sys/intr.h>
 #include <sys/kauth.h>
 #include <sys/kernel.h>
 #include <sys/kmem.h>
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <sys/condvar.h>
 #include <sys/poll.h>
 #include <sys/proc.h>
 #include <sys/select.h>
 #include <sys/sockio.h>
 #include <sys/stat.h>
 #include <sys/sysctl.h>
 #include <sys/systm.h>
 #include <net/if.h>
 #include <net/if_dl.h>
 #include <net/if_ether.h>
 #include <net/if_media.h>
 #include <net/if_tap.h>
 @@ -99,28 +100,28 @@ static void	sysctl_tap_setup(struct sysc
  */
 struct tap_softc {
 	device_t	sc_dev;
 	struct ifmedia	sc_im;
 	struct ethercom	sc_ec;
 	int		sc_flags;
 #define	TAP_INUSE	0x00000001	/* tap device can only be opened once */
 #define TAP_ASYNCIO	0x00000002	/* user is using async I/O (SIGIO) on the device */
 #define TAP_NBIO	0x00000004	/* user wants calls to avoid blocking */
 #define TAP_GOING	0x00000008	/* interface is being destroyed */
 	struct selinfo	sc_rsel;
 	pid_t		sc_pgid; /* For async. IO */
-	kmutex_t	sc_rdlock;
+	kmutex_t	sc_lock;
-	kmutex_t	sc_kqlock;
+	kcondvar_t	sc_cv;
 	void		*sc_sih;
 	struct timespec sc_atime;
 	struct timespec sc_mtime;
 	struct timespec sc_btime;
 };
 /* autoconf(9) glue */
 static int	tap_match(device_t, cfdata_t, void *);
 static void	tap_attach(device_t, device_t, void *);
 static int	tap_detach(device_t, int);
 CFATTACH_DECL_NEW(tap, sizeof(struct tap_softc),
 @@ -172,27 +173,27 @@ static int	tap_cdev_kqfilter(dev_t, stru
 const struct cdevsw tap_cdevsw = {
 	.d_open = tap_cdev_open,
 	.d_close = tap_cdev_close,
 	.d_read = tap_cdev_read,
 	.d_write = tap_cdev_write,
 	.d_ioctl = tap_cdev_ioctl,
 	.d_stop = nostop,
 	.d_tty = notty,
 	.d_poll = tap_cdev_poll,
 	.d_mmap = nommap,
 	.d_kqfilter = tap_cdev_kqfilter,
 	.d_discard = nodiscard,
-	.d_flag = D_OTHER
+	.d_flag = D_OTHER | D_MPSAFE
 };
 #define TAP_CLONER	0xfffff		/* Maximal minor value */
 /* kqueue-related routines */
 static void	tap_kqdetach(struct knote *);
 static int	tap_kqread(struct knote *, long);
 /*
  * Those are needed by the if_media interface.
  */
 static int	tap_mediachange(struct ifnet *);
 @@ -305,42 +306,28 @@ tap_attach(device_t parent, device_t sel
 	const struct sysctlnode *node;
 	int error;
 	uint8_t enaddr[ETHER_ADDR_LEN] =
 	    { 0xf2, 0x0b, 0xa4, 0xff, 0xff, 0xff };
 	char enaddrstr[3 * ETHER_ADDR_LEN];
 	sc->sc_dev = self;
 	sc->sc_sih = NULL;
 	getnanotime(&sc->sc_btime);
 	sc->sc_atime = sc->sc_mtime = sc->sc_btime;
 	sc->sc_flags = 0;
 	selinit(&sc->sc_rsel);
-	/*
+	cv_init(&sc->sc_cv, "tapread");
-	 * Initialize the two locks for the device.
+	mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NET);
 	 * We need a lock here because even though the tap device can be
 	 * opened only once, the file descriptor might be passed to another
 	 * process, say a fork(2)ed child.
 	 * The Giant saves us from most of the hassle, but since the read
 	 * operation can sleep, we don't want two processes to wake up at
 	 * the same moment and both try and dequeue a single packet.
 	 * The queue for event listeners (used by kqueue(9), see below) has
 	 * to be protected too, so use a spin lock.
 	 */
 	mutex_init(&sc->sc_rdlock, MUTEX_DEFAULT, IPL_NONE);
 	mutex_init(&sc->sc_kqlock, MUTEX_DEFAULT, IPL_VM);
 	if (!pmf_device_register(self, NULL, NULL))
 		aprint_error_dev(self, "couldn't establish power handler\n");
 	/*
 	 * In order to obtain unique initial Ethernet address on a host,
 	 * do some randomisation.  It's not meant for anything but avoiding
 	 * hard-coding an address.
 	 */
 	cprng_fast(&enaddr[3], 3);
 	aprint_verbose_dev(self, "Ethernet address %s\n",
 	    ether_snprintf(enaddrstr, sizeof(enaddrstr), enaddr));
 @@ -375,32 +362,32 @@ tap_attach(device_t parent, device_t sel
 	ifp->if_start	= tap_start;
 	ifp->if_stop	= tap_stop;
 	ifp->if_init	= tap_init;
 	IFQ_SET_READY(&ifp->if_snd);
 	sc->sc_ec.ec_capabilities = ETHERCAP_VLAN_MTU | ETHERCAP_JUMBO_MTU;
 	/* Those steps are mandatory for an Ethernet driver. */
 	error = if_initialize(ifp);
 	if (error != 0) {
 		aprint_error_dev(self, "if_initialize failed(%d)\n", error);
 		ifmedia_removeall(&sc->sc_im);
 		pmf_device_deregister(self);
-		mutex_destroy(&sc->sc_rdlock);
+		mutex_destroy(&sc->sc_lock);
 		mutex_destroy(&sc->sc_kqlock);
 		seldestroy(&sc->sc_rsel);
 		return; /* Error */
+	}
 	ifp->if_percpuq = if_percpuq_create(ifp);
 	ether_ifattach(ifp, enaddr);
 	if_register(ifp);
 	/*
 	 * Add a sysctl node for that interface.
+	 *
 	 * The pointer transmitted is not a string, but instead a pointer to
 	 * the softc structure, which we can use to build the string value on
 	 * the fly in the helper function of the node.  See the comments for
 	 * tap_sysctl_handler for details.
+	 *
 	 * Usually sysctl_createv is called with CTL_CREATE as the before-last
 	 * component.  However, we can allocate a number ourselves, as we are
 @@ -418,54 +405,51 @@ tap_attach(device_t parent, device_t sel
 		    "sysctl_createv returned %d, ignoring\n", error);
+}
 /*
  * When detaching, we do the inverse of what is done in the attach
  * routine, in reversed order.
  */
 static int
 tap_detach(device_t self, int flags)
+{
 	struct tap_softc *sc = device_private(self);
 	struct ifnet *ifp = &sc->sc_ec.ec_if;
 	int error;
 	int s;
 	sc->sc_flags |= TAP_GOING;
 	s = splnet();
 	tap_stop(ifp, 1);
 	if_down(ifp);
 	splx(s);
 	if (sc->sc_sih != NULL) {
 		softint_disestablish(sc->sc_sih);
 		sc->sc_sih = NULL;
+	}
 	/*
 	 * Destroying a single leaf is a very straightforward operation using
 	 * sysctl_destroyv.  One should be sure to always end the path with
 	 * CTL_EOL.
 	 */
 	if ((error = sysctl_destroyv(NULL, CTL_NET, AF_LINK, tap_node,
 	    device_unit(sc->sc_dev), CTL_EOL)) != 0)
 		aprint_error_dev(self,
 		    "sysctl_destroyv returned %d, ignoring\n", error);
 	ether_ifdetach(ifp);
 	if_detach(ifp);
 	ifmedia_removeall(&sc->sc_im);
 	seldestroy(&sc->sc_rsel);
-	mutex_destroy(&sc->sc_rdlock);
+	mutex_destroy(&sc->sc_lock);
-	mutex_destroy(&sc->sc_kqlock);
+	cv_destroy(&sc->sc_cv);
 	pmf_device_deregister(self);
 	return 0;
+}
 /*
  * This function is called by the ifmedia layer to notify the driver
  * that the user requested a media change.  A real driver would
  * reconfigure the hardware.
  */
 static int
 tap_mediachange(struct ifnet *ifp)
 @@ -506,45 +490,48 @@ tap_mediastatus(struct ifnet *ifp, struc
  * userland.  For that we stay in OACTIVE mode while the userland gets
  * the packets, and we send a signal to the processes waiting to read.
+ *
  * wakeup(sc) is the counterpart to the tsleep call in
  * tap_dev_read, while selnotify() is used for kevent(2) and
  * poll(2) (which includes select(2)) listeners.
  */
 static void
 tap_start(struct ifnet *ifp)
+{
 	struct tap_softc *sc = (struct tap_softc *)ifp->if_softc;
 	struct mbuf *m0;
 	mutex_enter(&sc->sc_lock);
 	if ((sc->sc_flags & TAP_INUSE) == 0) {
 		/* Simply drop packets */
 		for(;;) {
 			IFQ_DEQUEUE(&ifp->if_snd, m0);
 			if (m0 == NULL)
-				return;
+				goto done;
 			ifp->if_opackets++;
 			bpf_mtap(ifp, m0);
 			m_freem(m0);
+		}
 	} else if (!IFQ_IS_EMPTY(&ifp->if_snd)) {
 		ifp->if_flags |= IFF_OACTIVE;
-		wakeup(sc);
+		cv_broadcast(&sc->sc_cv);
 		selnotify(&sc->sc_rsel, 0, 1);
 		if (sc->sc_flags & TAP_ASYNCIO)
 			softint_schedule(sc->sc_sih);
+	}
 done:
 	mutex_exit(&sc->sc_lock);
+}
 static void
 tap_softintr(void *cookie)
+{
 	struct tap_softc *sc;
 	struct ifnet *ifp;
 	int a, b;
 	sc = cookie;
 	if (sc->sc_flags & TAP_ASYNCIO) {
 		ifp = &sc->sc_ec.ec_if;
 @@ -635,31 +622,33 @@ tap_init(struct ifnet *ifp)
 /*
  * _stop() is called when an interface goes down.  It is our
  * responsability to validate that state by clearing the
  * IFF_RUNNING flag.
+ *
  * We have to wake up all the sleeping processes to have the pending
  * read requests cancelled.
  */
 static void
 tap_stop(struct ifnet *ifp, int disable)
+{
 	struct tap_softc *sc = (struct tap_softc *)ifp->if_softc;
 	mutex_enter(&sc->sc_lock);
 	ifp->if_flags &= ~IFF_RUNNING;
-	wakeup(sc);
+	cv_broadcast(&sc->sc_cv);
 	selnotify(&sc->sc_rsel, 0, 1);
 	if (sc->sc_flags & TAP_ASYNCIO)
 		softint_schedule(sc->sc_sih);
 	mutex_exit(&sc->sc_lock);
+}
 /*
  * The 'create' command of ifconfig can be used to create
  * any numbered instance of a given device.  Thus we have to
  * make sure we have enough room in cd_devs to create the
  * user-specified instance.  config_attach_pseudo will do this
  * for us.
  */
 static int
 tap_clone_create(struct if_clone *ifc, int unit)
+{
 	if (tap_clone_creator(unit) == NULL) {
 @@ -921,100 +910,90 @@ tap_fops_read(file_t *fp, off_t *offp, s
 	KERNEL_LOCK(1, NULL);
 	error = tap_dev_read(fp->f_devunit, uio, flags);
 	KERNEL_UNLOCK_ONE(NULL);
 	return error;
+}
 static int
 tap_dev_read(int unit, struct uio *uio, int flags)
+{
 	struct tap_softc *sc = device_lookup_private(&tap_cd, unit);
 	struct ifnet *ifp;
 	struct mbuf *m, *n;
-	int error = 0, s;
+	int error = 0;
 	if (sc == NULL)
 		return ENXIO;
 	getnanotime(&sc->sc_atime);
 	ifp = &sc->sc_ec.ec_if;
 	if ((ifp->if_flags & IFF_UP) == 0)
 		return EHOSTDOWN;
 	/*
 	 * In the TAP_NBIO case, we have to make sure we won't be sleeping
 	 */
 	if ((sc->sc_flags & TAP_NBIO) != 0) {
-		if (!mutex_tryenter(&sc->sc_rdlock))
+		if (!mutex_tryenter(&sc->sc_lock))
 			return EWOULDBLOCK;
 	} else {
-		mutex_enter(&sc->sc_rdlock);
+		mutex_enter(&sc->sc_lock);
+	}
 	s = splnet();
 	if (IFQ_IS_EMPTY(&ifp->if_snd)) {
 		ifp->if_flags &= ~IFF_OACTIVE;
 		/*
 		 * We must release the lock before sleeping, and re-acquire it
 		 * after.
 		 */
 		mutex_exit(&sc->sc_rdlock);
 		if (sc->sc_flags & TAP_NBIO)
 			error = EWOULDBLOCK;
 		else
-			error = tsleep(sc, PSOCK|PCATCH, "tap", 0);
+			error = cv_wait_sig(&sc->sc_cv, &sc->sc_lock);
 		splx(s);
-		if (error != 0)
+		if (error != 0) {
 			mutex_exit(&sc->sc_lock);
 			return error;
+		}
 		/* The device might have been downed */
-		if ((ifp->if_flags & IFF_UP) == 0)
+		if ((ifp->if_flags & IFF_UP) == 0) {
 			mutex_exit(&sc->sc_lock);
 			return EHOSTDOWN;
 		if ((sc->sc_flags & TAP_NBIO)) {
 			if (!mutex_tryenter(&sc->sc_rdlock))
 				return EWOULDBLOCK;
 		} else {
 			mutex_enter(&sc->sc_rdlock);
+		}
 		s = splnet();
+	}
 	IFQ_DEQUEUE(&ifp->if_snd, m);
 	mutex_exit(&sc->sc_lock);
 	ifp->if_flags &= ~IFF_OACTIVE;
 	splx(s);
 	if (m == NULL) {
 		error = 0;
 		goto out;
+	}
 	ifp->if_opackets++;
 	bpf_mtap(ifp, m);
 	/*
 	 * One read is one packet.
 	 */
 	do {
 		error = uiomove(mtod(m, void *),
 		    min(m->m_len, uio->uio_resid), uio);
 		m = n = m_free(m);
 	} while (m != NULL && uio->uio_resid > 0 && error == 0);
 	if (m != NULL)
 		m_freem(m);
 out:
 	mutex_exit(&sc->sc_rdlock);
 	return error;
+}
 static int
 tap_fops_stat(file_t *fp, struct stat *st)
+{
 	int error = 0;
 	struct tap_softc *sc;
 	int unit = fp->f_devunit;
 	(void)memset(st, 0, sizeof(*st));
 	KERNEL_LOCK(1, NULL);
 @@ -1051,27 +1030,26 @@ tap_fops_write(file_t *fp, off_t *offp,
 	error = tap_dev_write(fp->f_devunit, uio, flags);
 	KERNEL_UNLOCK_ONE(NULL);
 	return error;
+}
 static int
 tap_dev_write(int unit, struct uio *uio, int flags)
+{
 	struct tap_softc *sc =
 	    device_lookup_private(&tap_cd, unit);
 	struct ifnet *ifp;
 	struct mbuf *m, **mp;
 	int error = 0;
 	int s;
 	if (sc == NULL)
 		return ENXIO;
 	getnanotime(&sc->sc_mtime);
 	ifp = &sc->sc_ec.ec_if;
 	/* One write, one packet, that's the rule */
 	MGETHDR(m, M_DONTWAIT, MT_DATA);
 	if (m == NULL) {
 		ifp->if_ierrors++;
 		return ENOBUFS;
+	}
 @@ -1088,29 +1066,27 @@ tap_dev_write(int unit, struct uio *uio,
+		}
 		(*mp)->m_len = min(MHLEN, uio->uio_resid);
 		error = uiomove(mtod(*mp, void *), (*mp)->m_len, uio);
 		mp = &(*mp)->m_next;
+	}
 	if (error) {
 		ifp->if_ierrors++;
 		m_freem(m);
 		return error;
+	}
 	m_set_rcvif(m, ifp);
-	s = splnet();
+	if_percpuq_enqueue(ifp->if_percpuq, m);
 	if_input(ifp, m);
 	splx(s);
 	return 0;
+}
 static int
 tap_cdev_ioctl(dev_t dev, u_long cmd, void *data, int flags,
     struct lwp *l)
+{
 	return tap_dev_ioctl(minor(dev), cmd, data, l);
+}
 static int
 tap_fops_ioctl(file_t *fp, u_long cmd, void *data)
 @@ -1211,29 +1187,29 @@ tap_dev_poll(int unit, int events, struc
 		return POLLERR;
 	if (events & (POLLIN|POLLRDNORM)) {
 		struct ifnet *ifp = &sc->sc_ec.ec_if;
 		struct mbuf *m;
 		int s;
 		s = splnet();
 		IFQ_POLL(&ifp->if_snd, m);
 		if (m != NULL)
 			revents |= events & (POLLIN|POLLRDNORM);
 		else {
-			mutex_spin_enter(&sc->sc_kqlock);
+			mutex_spin_enter(&sc->sc_lock);
 			selrecord(l, &sc->sc_rsel);
-			mutex_spin_exit(&sc->sc_kqlock);
+			mutex_spin_exit(&sc->sc_lock);
+		}
 		splx(s);
+	}
 	revents |= events & (POLLOUT|POLLWRNORM);
 	return revents;
+}
 static struct filterops tap_read_filterops = { 1, NULL, tap_kqdetach,
 	tap_kqread };
 static struct filterops tap_seltrue_filterops = { 1, NULL, tap_kqdetach,
 	filt_seltrue };
 @@ -1262,42 +1238,42 @@ tap_dev_kqfilter(int unit, struct knote
 	switch(kn->kn_filter) {
 	case EVFILT_READ:
 		kn->kn_fop = &tap_read_filterops;
 		break;
 	case EVFILT_WRITE:
 		kn->kn_fop = &tap_seltrue_filterops;
 		break;
 	default:
 		KERNEL_UNLOCK_ONE(NULL);
 		return EINVAL;
+	}
 	kn->kn_hook = sc;
-	mutex_spin_enter(&sc->sc_kqlock);
+	mutex_spin_enter(&sc->sc_lock);
 	SLIST_INSERT_HEAD(&sc->sc_rsel.sel_klist, kn, kn_selnext);
-	mutex_spin_exit(&sc->sc_kqlock);
+	mutex_spin_exit(&sc->sc_lock);
 	KERNEL_UNLOCK_ONE(NULL);
 	return 0;
+}
 static void
 tap_kqdetach(struct knote *kn)
+{
 	struct tap_softc *sc = (struct tap_softc *)kn->kn_hook;
 	KERNEL_LOCK(1, NULL);
-	mutex_spin_enter(&sc->sc_kqlock);
+	mutex_spin_enter(&sc->sc_lock);
 	SLIST_REMOVE(&sc->sc_rsel.sel_klist, kn, knote, kn_selnext);
-	mutex_spin_exit(&sc->sc_kqlock);
+	mutex_spin_exit(&sc->sc_lock);
 	KERNEL_UNLOCK_ONE(NULL);
+}
 static int
 tap_kqread(struct knote *kn, long hint)
+{
 	struct tap_softc *sc = (struct tap_softc *)kn->kn_hook;
 	struct ifnet *ifp = &sc->sc_ec.ec_if;
 	struct mbuf *m;
 	int s, rv;
 	KERNEL_LOCK(1, NULL);
 	s = splnet();