 @@ -1,14 +1,14 @@
-/*	$NetBSD: if_wg.c,v 1.48 2020/08/31 20:31:43 riastradh Exp $	*/
+/*	$NetBSD: if_wg.c,v 1.49 2020/08/31 20:33:58 riastradh Exp $	*/
 /*
  * Copyright (C) Ryota Ozaki <ozaki.ryota@gmail.com>
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -31,27 +31,27 @@
 /*
  * This network interface aims to implement the WireGuard protocol.
  * The implementation is based on the paper of WireGuard as of
  * 2018-06-30 [1].  The paper is referred in the source code with label
  * [W].  Also the specification of the Noise protocol framework as of
  * 2018-07-11 [2] is referred with label [N].
+ *
  * [1] https://www.wireguard.com/papers/wireguard.pdf
  * [2] http://noiseprotocol.org/noise.pdf
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.48 2020/08/31 20:31:43 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.49 2020/08/31 20:33:58 riastradh Exp $");
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
 #endif
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/atomic.h>
 #include <sys/callout.h>
 #include <sys/cprng.h>
 #include <sys/cpu.h>
 #include <sys/device.h>
 @@ -127,53 +127,50 @@ __KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.
  *   - It has a softint that is used to send packets over an wg interface
  *     to a peer
  *   - It has a pair of session instances (struct wg_session)
  *   - It has a pair of endpoint instances (struct wg_sockaddr)
  *     - Normally one endpoint is used and the second one is used only on
  *       a peer migration (a change of peer's IP address)
  *   - It has a list of IP addresses and sub networks called allowedips
  *     (struct wg_allowedip)
  *     - A packets sent over a session is allowed if its destination matches
  *       any IP addresses or sub networks of the list
  * - struct wg_session represents a session of a secure tunnel with a peer
  *   - Two instances of sessions belong to a peer; a stable session and a
  *     unstable session
- *   - A handshake process of a session always starts with a unstable instace
+ *   - A handshake process of a session always starts with a unstable instance
  *   - Once a session is established, its instance becomes stable and the
  *     other becomes unstable instead
  *   - Data messages are always sent via a stable session
+ *
  * Locking notes:
  * - wg interfaces (struct wg_softc, wg) is listed in wg_softcs.list and
  *   protected by wg_softcs.lock
- * - Each wg has a mutex(9) and a rwlock(9)
+ * - Each wg has a mutex(9) wg_lock, and a rwlock(9) wg_rwlock
- *   - The mutex (wg_lock) protects its peer list (wg_peers)
+ *   - Changes to the peer list are serialized by wg_lock
- *   - A peer on the list is also protected by pserialize(9) or psref(9)
+ *   - The peer list may be read with pserialize(9) and psref(9)
  *   - The rwlock (wg_rwlock) protects the routing tables (wg_rtable_ipv[46])
- * - Each peer (struct wg_peer, wgp) has a mutex
+ *     => XXX replace by pserialize when routing table is psz-safe
- *   - The mutex (wgp_lock) protects wgp_session_unstable and wgp_state
+ * - Each peer (struct wg_peer, wgp) has a mutex wgp_lock, which can be taken
- * - Each session (struct wg_session, wgs) has a mutex
+ *   only in thread context and serializes:
- *   - The mutex (wgs_lock) protects its state (wgs_state) and its handshake
+ *   - the stable and unstable session pointers
- *     states
+ *   - all unstable session state
- *   - wgs_state of a unstable session can be changed while it never be
+ * - Packet processing may be done in softint context:
- *     changed on a stable session, so once get a session instace via
+ *   - The stable session can be read under pserialize(9) or psref(9)
- *     wgp_session_stable we can safely access wgs_state without
+ *     - The stable session is always ESTABLISHED
  *     holding wgs_lock
  *   - A session is protected by pserialize or psref like wgp
  *     - On a session swap, we must wait for all readers to release a
  *       reference to a stable session before changing wgs_state and
  *       session states
  * - Lock order: wg_lock -> wgp_lock
  * Lock order: wg_lock -> wgp_lock -> wgs_lock
  */
 #define WGLOG(level, fmt, args...)					      \
 	log(level, "%s: " fmt, __func__, ##args)
 /* Debug options */
 #ifdef WG_DEBUG
 /* Output debug logs */
 #ifndef WG_DEBUG_LOG
 #define WG_DEBUG_LOG
 #endif
 /* Output trace logs */
 @@ -434,41 +431,40 @@ struct wg_worker {
 	bool		wgw_todie;
 	struct socket	*wgw_so4;
 	struct socket	*wgw_so6;
 	int		wgw_wakeup_reasons;
 #define WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4	__BIT(0)
 #define WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6	__BIT(1)
 #define WG_WAKEUP_REASON_PEER			__BIT(2)
 };
 struct wg_session {
 	struct wg_peer	*wgs_peer;
 	struct psref_target
 			wgs_psref;
 	kmutex_t	*wgs_lock;
 	int		wgs_state;
 #define WGS_STATE_UNKNOWN	0
 #define WGS_STATE_INIT_ACTIVE	1
 #define WGS_STATE_INIT_PASSIVE	2
 #define WGS_STATE_ESTABLISHED	3
 #define WGS_STATE_DESTROYING	4
 	time_t		wgs_time_established;
 	time_t		wgs_time_last_data_sent;
 	bool		wgs_is_initiator;
-	uint32_t	wgs_sender_index;
+	uint32_t	wgs_local_index;
-	uint32_t	wgs_receiver_index;
+	uint32_t	wgs_remote_index;
 #ifdef __HAVE_ATOMIC64_LOADSTORE
 	volatile uint64_t
 			wgs_send_counter;
 #else
 	kmutex_t	wgs_send_counter_lock;
 	uint64_t	wgs_send_counter;
 #endif
 	struct {
 		kmutex_t	lock;
 		struct sliwin	window;
 	}		*wgs_recvwin;
 @@ -527,38 +523,32 @@ struct wg_ppsratecheck {
 struct wg_softc;
 struct wg_peer {
 	struct wg_softc		*wgp_sc;
 	char			wgp_name[WG_PEER_NAME_MAXLEN + 1];
 	struct pslist_entry	wgp_peerlist_entry;
 	pserialize_t		wgp_psz;
 	struct psref_target	wgp_psref;
 	kmutex_t		*wgp_lock;
 	uint8_t	wgp_pubkey[WG_STATIC_KEY_LEN];
 	struct wg_sockaddr	*wgp_endpoint;
 	struct wg_sockaddr	*wgp_endpoint0;
-	bool			wgp_endpoint_changing;
+	volatile unsigned	wgp_endpoint_changing;
 	bool			wgp_endpoint_available;
 			/* The preshared key (optional) */
 	uint8_t		wgp_psk[WG_PRESHARED_KEY_LEN];
 	int wgp_state;
 #define WGP_STATE_INIT		0
 #define WGP_STATE_ESTABLISHED	1
 #define WGP_STATE_GIVEUP	2
 #define WGP_STATE_DESTROYING	3
 	void		*wgp_si;
 	pcq_t		*wgp_q;
 	struct wg_session	*wgp_session_stable;
 	struct wg_session	*wgp_session_unstable;
 	/* timestamp in big-endian */
 	wg_timestamp_t	wgp_timestamp_latest_init;
 	struct timespec		wgp_last_handshake_time;
 	callout_t		wgp_rekey_timer;
 	callout_t		wgp_handshake_timeout_timer;
 @@ -575,29 +565,31 @@ struct wg_peer {
 	bool			wgp_last_sent_mac1_valid;
 	uint8_t			wgp_last_sent_cookie[WG_COOKIE_LEN];
 	bool			wgp_last_sent_cookie_valid;
 	time_t			wgp_last_msg_received_time[WG_MSG_TYPE_MAX];
 	time_t			wgp_last_genrandval_time;
 	uint32_t		wgp_randval;
 	struct wg_ppsratecheck	wgp_ppsratecheck;
 	volatile unsigned int	wgp_tasks;
 #define WGP_TASK_SEND_INIT_MESSAGE		__BIT(0)
-#define WGP_TASK_ENDPOINT_CHANGED		__BIT(1)
+#define WGP_TASK_RETRY_HANDSHAKE		__BIT(1)
-#define WGP_TASK_SEND_KEEPALIVE_MESSAGE		__BIT(2)
+#define WGP_TASK_ESTABLISH_SESSION		__BIT(2)
-#define WGP_TASK_DESTROY_PREV_SESSION		__BIT(3)
+#define WGP_TASK_ENDPOINT_CHANGED		__BIT(3)
 #define WGP_TASK_SEND_KEEPALIVE_MESSAGE		__BIT(4)
 #define WGP_TASK_DESTROY_PREV_SESSION		__BIT(5)
 };
 struct wg_ops;
 struct wg_softc {
 	struct ifnet	wg_if;
 	LIST_ENTRY(wg_softc) wg_list;
 	kmutex_t	*wg_lock;
 	krwlock_t	*wg_rwlock;
 	uint8_t		wg_privkey[WG_STATIC_KEY_LEN];
 	uint8_t		wg_pubkey[WG_STATIC_KEY_LEN];
 @@ -642,28 +634,28 @@ static unsigned wg_reject_after_time = W
 static unsigned wg_rekey_attempt_time = WG_REKEY_ATTEMPT_TIME;
 static unsigned wg_rekey_timeout = WG_REKEY_TIMEOUT;
 static unsigned wg_keepalive_timeout = WG_KEEPALIVE_TIMEOUT;
 static struct mbuf *
 		wg_get_mbuf(size_t, size_t);
 static void	wg_wakeup_worker(struct wg_worker *, int);
 static int	wg_send_data_msg(struct wg_peer *, struct wg_session *,
 		    struct mbuf *);
 static int	wg_send_cookie_msg(struct wg_softc *, struct wg_peer *,
 		    const uint32_t, const uint8_t [], const struct sockaddr *);
-static int	wg_send_handshake_msg_resp(struct wg_softc *,
+static int	wg_send_handshake_msg_resp(struct wg_softc *, struct wg_peer *,
-		    struct wg_peer *, const struct wg_msg_init *);
+		    struct wg_session *, const struct wg_msg_init *);
 static void	wg_send_keepalive_msg(struct wg_peer *, struct wg_session *);
 static struct wg_peer *
 		wg_pick_peer_by_sa(struct wg_softc *, const struct sockaddr *,
 		    struct psref *);
 static struct wg_peer *
 		wg_lookup_peer_by_pubkey(struct wg_softc *,
 		    const uint8_t [], struct psref *);
 static struct wg_session *
 		wg_lookup_session_by_index(struct wg_softc *,
 		    const uint32_t, struct psref *);
 @@ -681,26 +673,28 @@ static void	wg_clear_states(struct wg_se
 static void	wg_get_peer(struct wg_peer *, struct psref *);
 static void	wg_put_peer(struct wg_peer *, struct psref *);
 static int	wg_send_so(struct wg_peer *, struct mbuf *);
 static int	wg_send_udp(struct wg_peer *, struct mbuf *);
 static int	wg_output(struct ifnet *, struct mbuf *,
 			   const struct sockaddr *, const struct rtentry *);
 static void	wg_input(struct ifnet *, struct mbuf *, const int);
 static int	wg_ioctl(struct ifnet *, u_long, void *);
 static int	wg_bind_port(struct wg_softc *, const uint16_t);
 static int	wg_init(struct ifnet *);
 static void	wg_stop(struct ifnet *, int);
 static void	wg_purge_pending_packets(struct wg_peer *);
 static int	wg_clone_create(struct if_clone *, int);
 static int	wg_clone_destroy(struct ifnet *);
 struct wg_ops {
 	int (*send_hs_msg)(struct wg_peer *, struct mbuf *);
 	int (*send_data_msg)(struct wg_peer *, struct mbuf *);
 	void (*input)(struct ifnet *, struct mbuf *, const int);
 	int (*bind_port)(struct wg_softc *, const uint16_t);
 };
 struct wg_ops wg_ops_rumpkernel = {
 	.send_hs_msg	= wg_send_so,
 	.send_data_msg	= wg_send_udp,
 @@ -1090,131 +1084,142 @@ wg_algo_tai64n(wg_timestamp_t timestamp)
+{
 	struct timespec ts;
 	/* FIXME strict TAI64N (https://cr.yp.to/libtai/tai64.html) */
 	getnanotime(&ts);
 	/* TAI64 label in external TAI64 format */
 	be32enc(timestamp, 0x40000000UL + (ts.tv_sec >> 32));
 	/* second beginning from 1970 TAI */
 	be32enc(timestamp + 4, ts.tv_sec & 0xffffffffU);
 	/* nanosecond in big-endian format */
 	be32enc(timestamp + 8, ts.tv_nsec);
+}
-static struct wg_session *
+/*
-wg_get_unstable_session(struct wg_peer *wgp, struct psref *psref)
+ * wg_get_stable_session(wgp, psref)
+ *
-	int s;
+ *	Get a passive reference to the current stable session, or
-	struct wg_session *wgs;
+ *	return NULL if there is no current stable session.
+ *
-	s = pserialize_read_enter();
+ *	The pointer is always there but the session is not necessarily
-	wgs = wgp->wgp_session_unstable;
+ *	ESTABLISHED; if it is not ESTABLISHED, return NULL.  However,
-	psref_acquire(psref, &wgs->wgs_psref, wg_psref_class);
+ *	the session may transition from ESTABLISHED to DESTROYING while
-	pserialize_read_exit(s);
+ *	holding the passive reference.
-	return wgs;
+ */
 static struct wg_session *
 wg_get_stable_session(struct wg_peer *wgp, struct psref *psref)
+{
 	int s;
 	struct wg_session *wgs;
 	s = pserialize_read_enter();
-	wgs = wgp->wgp_session_stable;
+	wgs = atomic_load_consume(&wgp->wgp_session_stable);
-	psref_acquire(psref, &wgs->wgs_psref, wg_psref_class);
+	if (__predict_false(wgs->wgs_state != WGS_STATE_ESTABLISHED))
 		wgs = NULL;
 	else
 		psref_acquire(psref, &wgs->wgs_psref, wg_psref_class);
 	pserialize_read_exit(s);
 	return wgs;
 static void
 wg_get_session(struct wg_session *wgs, struct psref *psref)
-	psref_acquire(psref, &wgs->wgs_psref, wg_psref_class);
+	return wgs;
+}
 static void
 wg_put_session(struct wg_session *wgs, struct psref *psref)
+{
 	psref_release(psref, &wgs->wgs_psref, wg_psref_class);
+}
-static struct wg_session *
+static void
-wg_lock_unstable_session(struct wg_peer *wgp)
+wg_destroy_session(struct wg_softc *wg, struct wg_session *wgs)
+{
-	struct wg_session *wgs;
+	struct wg_peer *wgp = wgs->wgs_peer;
 	struct wg_session *wgs0 __diagused;
 	void *garbage;
-	mutex_enter(wgp->wgp_lock);
+	KASSERT(mutex_owned(wgp->wgp_lock));
-	wgs = wgp->wgp_session_unstable;
+	KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN);
 	mutex_enter(wgs->wgs_lock);
 	mutex_exit(wgp->wgp_lock);
 	return wgs;
-#if 0
+	/* Remove the session from the table.  */
-static void
+	wgs0 = thmap_del(wg->wg_sessions_byindex,
-wg_unlock_session(struct wg_peer *wgp, struct wg_session *wgs)
+	    &wgs->wgs_local_index, sizeof(wgs->wgs_local_index));
 	KASSERT(wgs0 == wgs);
 	garbage = thmap_stage_gc(wg->wg_sessions_byindex);
 	/* Wait for passive references to drain.  */
 	pserialize_perform(wgp->wgp_psz);
 	psref_target_destroy(&wgs->wgs_psref, wg_psref_class);
-	mutex_exit(wgs->wgs_lock);
+	/* Free memory, zero state, and transition to UNKNOWN.  */
 	thmap_gc(wg->wg_sessions_byindex, garbage);
 	wg_clear_states(wgs);
 	wgs->wgs_state = WGS_STATE_UNKNOWN;
+}
 #endif
-static uint32_t
+/*
-wg_assign_sender_index(struct wg_softc *wg, struct wg_session *wgs)
+ * wg_get_session_index(wg, wgs)
+ *
  *	Choose a session index for wgs->wgs_local_index, and store it
  *	in wg's table of sessions by index.
+ *
  *	wgs must be the unstable session of its peer, and must be
  *	transitioning out of the UNKNOWN state.
  */
 static void
 wg_get_session_index(struct wg_softc *wg, struct wg_session *wgs)
+{
-	struct wg_peer *wgp = wgs->wgs_peer;
+	struct wg_peer *wgp __diagused = wgs->wgs_peer;
 	struct wg_session *wgs0;
 	uint32_t index;
 	void *garbage;
-	mutex_enter(wgs->wgs_lock);
+	KASSERT(mutex_owned(wgp->wgp_lock));
 	KASSERT(wgs == wgp->wgp_session_unstable);
 	KASSERT(wgs->wgs_state == WGS_STATE_UNKNOWN);
-	/* Release the current index, if there is one.  */
+	do {
-	while ((index = wgs->wgs_sender_index) != 0) {
+		/* Pick a uniform random index.  */
-		/* Remove the session by index.  */
+		index = cprng_strong32();
 		thmap_del(wg->wg_sessions_byindex, &index, sizeof index);
-		wgs->wgs_sender_index = 0;
+		/* Try to take it.  */
-		mutex_exit(wgs->wgs_lock);
+		wgs->wgs_local_index = index;
 		wgs0 = thmap_put(wg->wg_sessions_byindex,
-		/* Wait for all thmap_gets to complete, and GC.  */
+		    &wgs->wgs_local_index, sizeof wgs->wgs_local_index, wgs);
 		garbage = thmap_stage_gc(wg->wg_sessions_byindex);
 		mutex_enter(wgs->wgs_peer->wgp_lock);
 		pserialize_perform(wgp->wgp_psz);
 		mutex_exit(wgs->wgs_peer->wgp_lock);
 		thmap_gc(wg->wg_sessions_byindex, garbage);
-		mutex_enter(wgs->wgs_lock);
+		/* If someone else beat us, start over.  */
 	} while (__predict_false(wgs0 != wgs));
+}
-restart:
+/*
-	/* Pick a uniform random nonzero index.  */
+ * wg_put_session_index(wg, wgs)
 	while (__predict_false((index = cprng_strong32()) == 0))
-		continue;
+ *	Remove wgs from the table of sessions by index, wait for any
  *	passive references to drain, and transition the session to the
-	/* Try to take it.  */
+ *	UNKNOWN state.
 	wgs->wgs_sender_index = index;
-	wgs0 = thmap_put(wg->wg_sessions_byindex,
+ *	wgs must be the unstable session of its peer, and must not be
-	    &wgs->wgs_sender_index, sizeof wgs->wgs_sender_index, wgs);
+ *	UNKNOWN or ESTABLISHED.
  */
-	/* If someone else beat us, start over.  */
+static void
-	if (__predict_false(wgs0 != wgs))
+wg_put_session_index(struct wg_softc *wg, struct wg_session *wgs)
 		goto restart;
 	struct wg_peer *wgp = wgs->wgs_peer;
-	mutex_exit(wgs->wgs_lock);
+	KASSERT(mutex_owned(wgp->wgp_lock));
 	KASSERT(wgs == wgp->wgp_session_unstable);
 	KASSERT(wgs->wgs_state != WGS_STATE_UNKNOWN);
 	KASSERT(wgs->wgs_state != WGS_STATE_ESTABLISHED);
-	return index;
+	wg_destroy_session(wg, wgs);
 	psref_target_init(&wgs->wgs_psref, wg_psref_class);
+}
 /*
  * Handshake patterns
+ *
  * [W] 5: "These messages use the "IK" pattern from Noise"
  * [N] 7.5. Interactive handshake patterns (fundamental)
  *     "The first character refers to the initiator’s static key:"
  *     "I = Static key for initiator Immediately transmitted to responder,
  *          despite reduced or absent identity hiding"
  *     "The second character refers to the responder’s static key:"
  *     "K = Static key for responder Known to initiator"
  *     "IK:
 @@ -1229,28 +1234,32 @@ restart:
  *        -> e, es, s, ss
  *        <- e, ee, se, psk"
  */
 static void
 wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp,
     struct wg_session *wgs, struct wg_msg_init *wgmi)
+{
 	uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */
 	uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */
 	uint8_t cipher_key[WG_CIPHER_KEY_LEN];
 	uint8_t pubkey[WG_EPHEMERAL_KEY_LEN];
 	uint8_t privkey[WG_EPHEMERAL_KEY_LEN];
 	KASSERT(mutex_owned(wgp->wgp_lock));
 	KASSERT(wgs == wgp->wgp_session_unstable);
 	KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE);
 	wgmi->wgmi_type = htole32(WG_MSG_TYPE_INIT);
-	wgmi->wgmi_sender = wg_assign_sender_index(wg, wgs);
+	wgmi->wgmi_sender = wgs->wgs_local_index;
 	/* [W] 5.4.2: First Message: Initiator to Responder */
 	/* Ci := HASH(CONSTRUCTION) */
 	/* Hi := HASH(Ci || IDENTIFIER) */
 	wg_init_key_and_hash(ckey, hash);
 	/* Hi := HASH(Hi || Sr^pub) */
 	wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey));
 	WG_DUMP_HASH("hash", hash);
 	/* [N] 2.2: "e" */
 	/* Ei^priv, Ei^pub := DH-GENERATE() */
 @@ -1305,42 +1314,41 @@ wg_fill_msg_init(struct wg_softc *wg, st
 		memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2));
 	else {
 		wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2),
 		    wgp->wgp_latest_cookie, WG_COOKIE_LEN,
 		    (const uint8_t *)wgmi,
 		    offsetof(struct wg_msg_init, wgmi_mac2),
 		    NULL, 0);
+	}
 	memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey));
 	memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey));
 	memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash));
 	memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey));
-	WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_sender_index);
+	WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_local_index);
+}
 static void
 wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi,
     const struct sockaddr *src)
+{
 	uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */
 	uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */
 	uint8_t cipher_key[WG_CIPHER_KEY_LEN];
 	uint8_t peer_pubkey[WG_STATIC_KEY_LEN];
 	struct wg_peer *wgp;
 	struct wg_session *wgs;
 	int error, ret;
 	struct psref psref_peer;
 	struct psref psref_session;
 	uint8_t mac1[WG_MAC_LEN];
 	WG_TRACE("init msg received");
 	wg_algo_mac_mac1(mac1, sizeof(mac1),
 	    wg->wg_pubkey, sizeof(wg->wg_pubkey),
 	    (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1));
 	/*
 	 * [W] 5.3: Denial of Service Mitigation & Cookies
 	 * "the responder, ..., must always reject messages with an invalid
 	 *  msg.mac1"
 	 */
 @@ -1388,153 +1396,145 @@ wg_handle_msg_init(struct wg_softc *wg,
 		WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG,
 		    "wg_algo_aead_dec for secret key failed\n");
 		return;
+	}
 	/* Hi := HASH(Hi || msg.static) */
 	wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static));
 	wgp = wg_lookup_peer_by_pubkey(wg, peer_pubkey, &psref_peer);
 	if (wgp == NULL) {
 		WG_DLOG("peer not found\n");
 		return;
+	}
 	/*
 	 * Lock the peer to serialize access to cookie state.
+	 *
 	 * XXX Can we safely avoid holding the lock across DH?  Take it
 	 * just to verify mac2 and then unlock/DH/lock?
 	 */
 	mutex_enter(wgp->wgp_lock);
 	if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_INIT))) {
 		WG_TRACE("under load");
 		/*
 		 * [W] 5.3: Denial of Service Mitigation & Cookies
 		 * "the responder, ..., and when under load may reject messages
 		 *  with an invalid msg.mac2.  If the responder receives a
 		 *  message with a valid msg.mac1 yet with an invalid msg.mac2,
 		 *  and is under load, it may respond with a cookie reply
 		 *  message"
 		 */
 		uint8_t zero[WG_MAC_LEN] = {0};
 		if (consttime_memequal(wgmi->wgmi_mac2, zero, sizeof(zero))) {
 			WG_TRACE("sending a cookie message: no cookie included");
 			(void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender,
 			    wgmi->wgmi_mac1, src);
-			goto out_wgp;
+			goto out;
+		}
 		if (!wgp->wgp_last_sent_cookie_valid) {
 			WG_TRACE("sending a cookie message: no cookie sent ever");
 			(void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender,
 			    wgmi->wgmi_mac1, src);
-			goto out_wgp;
+			goto out;
+		}
 		uint8_t mac2[WG_MAC_LEN];
 		wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie,
 		    WG_COOKIE_LEN, (const uint8_t *)wgmi,
 		    offsetof(struct wg_msg_init, wgmi_mac2), NULL, 0);
 		if (!consttime_memequal(mac2, wgmi->wgmi_mac2, sizeof(mac2))) {
 			WG_DLOG("mac2 is invalid\n");
-			goto out_wgp;
+			goto out;
+		}
 		WG_TRACE("under load, but continue to sending");
+	}
 	/* [N] 2.2: "ss" */
 	/* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */
 	wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey);
 	/* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */
 	wg_timestamp_t timestamp;
 	error = wg_algo_aead_dec(timestamp, sizeof(timestamp), cipher_key, 0,
 	    wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp),
 	    hash, sizeof(hash));
 	if (error != 0) {
 		WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
 		    "wg_algo_aead_dec for timestamp failed\n");
-		goto out_wgp;
+		goto out;
+	}
 	/* Hi := HASH(Hi || msg.timestamp) */
 	wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp));
 	wgs = wg_lock_unstable_session(wgp);
 	if (wgs->wgs_state == WGS_STATE_DESTROYING) {
 		/*
 		 * We can assume that the peer doesn't have an
 		 * established session, so clear it now.  If the timer
 		 * fired, tough -- it won't have any effect unless we
 		 * manage to transition back to WGS_STATE_DESTROYING.
 		 */
 		WG_TRACE("Session destroying, but force to clear");
 		callout_stop(&wgp->wgp_session_dtor_timer);
 		wg_clear_states(wgs);
 		wgs->wgs_state = WGS_STATE_UNKNOWN;
 	if (wgs->wgs_state == WGS_STATE_INIT_ACTIVE) {
 		WG_TRACE("Sesssion already initializing, ignoring the message");
 		mutex_exit(wgs->wgs_lock);
 		goto out_wgp;
 	if (wgs->wgs_state == WGS_STATE_INIT_PASSIVE) {
 		WG_TRACE("Sesssion already initializing, destroying old states");
 		wg_clear_states(wgs);
 	wgs->wgs_state = WGS_STATE_INIT_PASSIVE;
 	wg_get_session(wgs, &psref_session);
 	mutex_exit(wgs->wgs_lock);
 	/*
 	 * [W] 5.1 "The responder keeps track of the greatest timestamp
 	 *      received per peer and discards packets containing
 	 *      timestamps less than or equal to it."
 	 */
 	ret = memcmp(timestamp, wgp->wgp_timestamp_latest_init,
 	    sizeof(timestamp));
 	if (ret <= 0) {
 		WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
 		    "invalid init msg: timestamp is old\n");
 		goto out;
+	}
 	memcpy(wgp->wgp_timestamp_latest_init, timestamp, sizeof(timestamp));
 	/*
 	 * Message is good -- we're committing to handle it now, unless
 	 * we were already initiating a session.
 	 */
 	wgs = wgp->wgp_session_unstable;
 	switch (wgs->wgs_state) {
 	case WGS_STATE_UNKNOWN:		/* new session initiated by peer */
 		wg_get_session_index(wg, wgs);
 		break;
 	case WGS_STATE_INIT_ACTIVE:	/* we're already initiating, drop */
 		WG_TRACE("Session already initializing, ignoring the message");
 		goto out;
 	case WGS_STATE_INIT_PASSIVE:	/* peer is retrying, start over */
 		WG_TRACE("Session already initializing, destroying old states");
 		wg_clear_states(wgs);
 		/* keep session index */
 		break;
 	case WGS_STATE_ESTABLISHED:	/* can't happen */
 		panic("unstable session can't be established");
 		break;
 	case WGS_STATE_DESTROYING:	/* rekey initiated by peer */
 		WG_TRACE("Session destroying, but force to clear");
 		callout_stop(&wgp->wgp_session_dtor_timer);
 		wg_clear_states(wgs);
 		/* keep session index */
 		break;
 	default:
 		panic("invalid session state: %d", wgs->wgs_state);
+	}
 	wgs->wgs_state = WGS_STATE_INIT_PASSIVE;
 	memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash));
 	memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey));
 	memcpy(wgs->wgs_ephemeral_key_peer, wgmi->wgmi_ephemeral,
 	    sizeof(wgmi->wgmi_ephemeral));
 	wg_update_endpoint_if_necessary(wgp, src);
-	(void)wg_send_handshake_msg_resp(wg, wgp, wgmi);
+	(void)wg_send_handshake_msg_resp(wg, wgp, wgs, wgmi);
 	wg_calculate_keys(wgs, false);
 	wg_clear_states(wgs);
 	wg_put_session(wgs, &psref_session);
 	wg_put_peer(wgp, &psref_peer);
 	return;
 out:
 	mutex_enter(wgs->wgs_lock);
 	KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE);
 	wgs->wgs_state = WGS_STATE_UNKNOWN;
 	mutex_exit(wgs->wgs_lock);
 	wg_put_session(wgs, &psref_session);
 out_wgp:
 	wg_put_peer(wgp, &psref_peer);
 static void
 wg_schedule_handshake_timeout_timer(struct wg_peer *wgp)
 	mutex_enter(wgp->wgp_lock);
 	if (__predict_true(wgp->wgp_state != WGP_STATE_DESTROYING)) {
 		callout_schedule(&wgp->wgp_handshake_timeout_timer,
 		    MIN(wg_rekey_timeout, INT_MAX/hz) * hz);
 	mutex_exit(wgp->wgp_lock);
 	wg_put_peer(wgp, &psref_peer);
+}
 static struct socket *
 wg_get_so_by_af(struct wg_worker *wgw, const int af)
+{
 	return (af == AF_INET) ? wgw->wgw_so4 : wgw->wgw_so6;
+}
 static struct socket *
 wg_get_so_by_peer(struct wg_peer *wgp, struct wg_sockaddr *wgsa)
+{
 @@ -1575,89 +1575,93 @@ wg_send_so(struct wg_peer *wgp, struct m
 	error = sosend(so, wgsatosa(wgsa), NULL, m, NULL, 0, curlwp);
 	wg_put_sa(wgp, wgsa, &psref);
 	return error;
+}
 static int
 wg_send_handshake_msg_init(struct wg_softc *wg, struct wg_peer *wgp)
+{
 	int error;
 	struct mbuf *m;
 	struct wg_msg_init *wgmi;
 	struct wg_session *wgs;
 	struct psref psref;
-	wgs = wg_lock_unstable_session(wgp);
+	KASSERT(mutex_owned(wgp->wgp_lock));
 	if (wgs->wgs_state == WGS_STATE_DESTROYING) {
 	wgs = wgp->wgp_session_unstable;
 	/* XXX pull dispatch out into wg_task_send_init_message */
 	switch (wgs->wgs_state) {
 	case WGS_STATE_UNKNOWN:		/* new session initiated by us */
 		wg_get_session_index(wg, wgs);
 		break;
 	case WGS_STATE_INIT_ACTIVE:	/* we're already initiating, stop */
 		WG_TRACE("Session already initializing, skip starting new one");
 		return EBUSY;
 	case WGS_STATE_INIT_PASSIVE:	/* peer was trying -- XXX what now? */
 		WG_TRACE("Session already initializing, destroying old states");
 		wg_clear_states(wgs);
 		/* keep session index */
 		break;
 	case WGS_STATE_ESTABLISHED:	/* can't happen */
 		panic("unstable session can't be established");
 		break;
 	case WGS_STATE_DESTROYING:	/* rekey initiated by us too early */
 		WG_TRACE("Session destroying");
 		mutex_exit(wgs->wgs_lock);
 		/* XXX should wait? */
 		return EBUSY;
+	}
 	if (wgs->wgs_state == WGS_STATE_INIT_ACTIVE) {
 		WG_TRACE("Sesssion already initializing, skip starting a new one");
 		mutex_exit(wgs->wgs_lock);
 		return EBUSY;
 	if (wgs->wgs_state == WGS_STATE_INIT_PASSIVE) {
 		WG_TRACE("Sesssion already initializing, destroying old states");
 		wg_clear_states(wgs);
 	wgs->wgs_state = WGS_STATE_INIT_ACTIVE;
 	wg_get_session(wgs, &psref);
 	mutex_exit(wgs->wgs_lock);
 	m = m_gethdr(M_WAIT, MT_DATA);
 	m->m_pkthdr.len = m->m_len = sizeof(*wgmi);
 	wgmi = mtod(m, struct wg_msg_init *);
 	wg_fill_msg_init(wg, wgp, wgs, wgmi);
 	error = wg->wg_ops->send_hs_msg(wgp, m);
 	if (error == 0) {
 		WG_TRACE("init msg sent");
 		if (wgp->wgp_handshake_start_time == 0)
 			wgp->wgp_handshake_start_time = time_uptime;
-		wg_schedule_handshake_timeout_timer(wgp);
+		callout_schedule(&wgp->wgp_handshake_timeout_timer,
 		    MIN(wg_rekey_timeout, INT_MAX/hz) * hz);
 	} else {
-		mutex_enter(wgs->wgs_lock);
+		wg_put_session_index(wg, wgs);
 		KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE);
 		wgs->wgs_state = WGS_STATE_UNKNOWN;
 		mutex_exit(wgs->wgs_lock);
+	}
 	wg_put_session(wgs, &psref);
 	return error;
+}
 static void
 wg_fill_msg_resp(struct wg_softc *wg, struct wg_peer *wgp,
-    struct wg_msg_resp *wgmr, const struct wg_msg_init *wgmi)
+    struct wg_session *wgs, struct wg_msg_resp *wgmr,
     const struct wg_msg_init *wgmi)
+{
 	uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */
 	uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */
 	uint8_t cipher_key[WG_KDF_OUTPUT_LEN];
 	uint8_t pubkey[WG_EPHEMERAL_KEY_LEN];
 	uint8_t privkey[WG_EPHEMERAL_KEY_LEN];
 	struct wg_session *wgs;
 	struct psref psref;
-	wgs = wg_get_unstable_session(wgp, &psref);
+	KASSERT(mutex_owned(wgp->wgp_lock));
 	KASSERT(wgs == wgp->wgp_session_unstable);
 	KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE);
 	memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash));
 	memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey));
 	wgmr->wgmr_type = htole32(WG_MSG_TYPE_RESP);
-	wgmr->wgmr_sender = wg_assign_sender_index(wg, wgs);
+	wgmr->wgmr_sender = wgs->wgs_local_index;
 	wgmr->wgmr_receiver = wgmi->wgmi_sender;
 	/* [W] 5.4.3 Second Message: Responder to Initiator */
 	/* [N] 2.2: "e" */
 	/* Er^priv, Er^pub := DH-GENERATE() */
 	wg_algo_generate_keypair(pubkey, privkey);
 	/* Cr := KDF1(Cr, Er^pub) */
 	wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey));
 	/* msg.ephemeral := Er^pub */
 	memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral));
 	/* Hr := HASH(Hr || msg.ephemeral) */
 	wg_algo_hash(hash, pubkey, sizeof(pubkey));
 @@ -1708,41 +1712,46 @@ wg_fill_msg_resp(struct wg_softc *wg, st
 	else {
 		/* msg.mac2 := MAC(Lm, msg_b) */
 		wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2),
 		    wgp->wgp_latest_cookie, WG_COOKIE_LEN,
 		    (const uint8_t *)wgmr,
 		    offsetof(struct wg_msg_resp, wgmr_mac2),
 		    NULL, 0);
+	}
 	memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash));
 	memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey));
 	memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey));
 	memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey));
-	wgs->wgs_receiver_index = wgmi->wgmi_sender;
+	wgs->wgs_remote_index = wgmi->wgmi_sender;
-	WG_DLOG("sender=%x\n", wgs->wgs_sender_index);
+	WG_DLOG("sender=%x\n", wgs->wgs_local_index);
-	WG_DLOG("receiver=%x\n", wgs->wgs_receiver_index);
+	WG_DLOG("receiver=%x\n", wgs->wgs_remote_index);
 	wg_put_session(wgs, &psref);
+}
 static void
 wg_swap_sessions(struct wg_peer *wgp)
+{
 	struct wg_session *wgs, *wgs_prev;
 	KASSERT(mutex_owned(wgp->wgp_lock));
-	wgp->wgp_session_unstable = atomic_swap_ptr(&wgp->wgp_session_stable,
+	wgs = wgp->wgp_session_unstable;
-	    wgp->wgp_session_unstable);
+	KASSERT(wgs->wgs_state == WGS_STATE_ESTABLISHED);
 	KASSERT(wgp->wgp_session_stable->wgs_state == WGS_STATE_ESTABLISHED);
 	wgs_prev = wgp->wgp_session_stable;
 	KASSERT(wgs_prev->wgs_state == WGS_STATE_ESTABLISHED ||
 	    wgs_prev->wgs_state == WGS_STATE_UNKNOWN);
 	atomic_store_release(&wgp->wgp_session_stable, wgs);
 	wgp->wgp_session_unstable = wgs_prev;
+}
 static void
 wg_handle_msg_resp(struct wg_softc *wg, const struct wg_msg_resp *wgmr,
     const struct sockaddr *src)
+{
 	uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */
 	uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Kr */
 	uint8_t cipher_key[WG_KDF_OUTPUT_LEN];
 	struct wg_peer *wgp;
 	struct wg_session *wgs;
 	struct psref psref;
 	int error;
 @@ -1762,26 +1771,34 @@ wg_handle_msg_resp(struct wg_softc *wg,
 		WG_DLOG("mac1 is invalid\n");
 		return;
+	}
 	WG_TRACE("resp msg received");
 	wgs = wg_lookup_session_by_index(wg, wgmr->wgmr_receiver, &psref);
 	if (wgs == NULL) {
 		WG_TRACE("No session found");
 		return;
+	}
 	wgp = wgs->wgs_peer;
 	mutex_enter(wgp->wgp_lock);
 	/* If we weren't waiting for a handshake response, drop it.  */
 	if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) {
 		WG_TRACE("peer sent spurious handshake response, ignoring");
 		goto out;
+	}
 	if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_RESP))) {
 		WG_TRACE("under load");
 		/*
 		 * [W] 5.3: Denial of Service Mitigation & Cookies
 		 * "the responder, ..., and when under load may reject messages
 		 *  with an invalid msg.mac2.  If the responder receives a
 		 *  message with a valid msg.mac1 yet with an invalid msg.mac2,
 		 *  and is under load, it may respond with a cookie reply
 		 *  message"
 		 */
 		uint8_t zero[WG_MAC_LEN] = {0};
 		if (consttime_memequal(wgmr->wgmr_mac2, zero, sizeof(zero))) {
 			WG_TRACE("sending a cookie message: no cookie included");
 @@ -1855,90 +1872,100 @@ wg_handle_msg_resp(struct wg_softc *wg,
 	    sizeof(wgmr->wgmr_empty), hash, sizeof(hash));
 	WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty);
 	if (error != 0) {
 		WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
 		    "wg_algo_aead_dec for empty message failed\n");
 		goto out;
+	}
 	/* Hr := HASH(Hr || msg.empty) */
 	wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty));
+    }
 	memcpy(wgs->wgs_handshake_hash, hash, sizeof(wgs->wgs_handshake_hash));
 	memcpy(wgs->wgs_chaining_key, ckey, sizeof(wgs->wgs_chaining_key));
-	wgs->wgs_receiver_index = wgmr->wgmr_sender;
+	wgs->wgs_remote_index = wgmr->wgmr_sender;
-	WG_DLOG("receiver=%x\n", wgs->wgs_receiver_index);
+	WG_DLOG("receiver=%x\n", wgs->wgs_remote_index);
 	KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE);
 	wgs->wgs_state = WGS_STATE_ESTABLISHED;
 	wgs->wgs_time_established = time_uptime;
 	wgs->wgs_time_last_data_sent = 0;
 	wgs->wgs_is_initiator = true;
 	wg_calculate_keys(wgs, true);
 	wg_clear_states(wgs);
 	WG_TRACE("WGS_STATE_ESTABLISHED");
-	callout_halt(&wgp->wgp_handshake_timeout_timer, NULL);
+	callout_stop(&wgp->wgp_handshake_timeout_timer);
 	mutex_enter(wgp->wgp_lock);
 	wg_swap_sessions(wgp);
 	KASSERT(wgs == wgp->wgp_session_stable);
 	wgs_prev = wgp->wgp_session_unstable;
 	mutex_enter(wgs_prev->wgs_lock);
 	getnanotime(&wgp->wgp_last_handshake_time);
 	wgp->wgp_handshake_start_time = 0;
 	wgp->wgp_last_sent_mac1_valid = false;
 	wgp->wgp_last_sent_cookie_valid = false;
 	mutex_exit(wgp->wgp_lock);
 	wg_schedule_rekey_timer(wgp);
 	wg_update_endpoint_if_necessary(wgp, src);
 	/*
 	 * Send something immediately (same as the official implementation)
 	 * XXX if there are pending data packets, we don't need to send
 	 *     a keepalive message.
 	 */
 	wg_send_keepalive_msg(wgp, wgs);
 	/* Anyway run a softint to flush pending packets */
 	kpreempt_disable();
 	softint_schedule(wgp->wgp_si);
 	kpreempt_enable();
 	WG_TRACE("softint scheduled");
 	if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) {
 		/* Wait for wg_get_stable_session to drain.  */
 		pserialize_perform(wgp->wgp_psz);
 		/* Transition ESTABLISHED->DESTROYING.  */
 		wgs_prev->wgs_state = WGS_STATE_DESTROYING;
 		/* We can't destroy the old session immediately */
 		wg_schedule_session_dtor_timer(wgp);
 	} else {
 		KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN,
 		    "state=%d", wgs_prev->wgs_state);
+	}
 	mutex_exit(wgs_prev->wgs_lock);
 out:
 	mutex_exit(wgp->wgp_lock);
 	wg_put_session(wgs, &psref);
+}
 static int
 wg_send_handshake_msg_resp(struct wg_softc *wg, struct wg_peer *wgp,
-    const struct wg_msg_init *wgmi)
+    struct wg_session *wgs, const struct wg_msg_init *wgmi)
+{
 	int error;
 	struct mbuf *m;
 	struct wg_msg_resp *wgmr;
 	KASSERT(mutex_owned(wgp->wgp_lock));
 	KASSERT(wgs == wgp->wgp_session_unstable);
 	KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE);
 	m = m_gethdr(M_WAIT, MT_DATA);
 	m->m_pkthdr.len = m->m_len = sizeof(*wgmr);
 	wgmr = mtod(m, struct wg_msg_resp *);
-	wg_fill_msg_resp(wg, wgp, wgmr, wgmi);
+	wg_fill_msg_resp(wg, wgp, wgs, wgmr, wgmi);
 	error = wg->wg_ops->send_hs_msg(wgp, m);
 	if (error == 0)
 		WG_TRACE("resp msg sent");
 	return error;
+}
 static struct wg_peer *
 wg_lookup_peer_by_pubkey(struct wg_softc *wg,
     const uint8_t pubkey[WG_STATIC_KEY_LEN], struct psref *psref)
+{
 	struct wg_peer *wgp;
 @@ -1952,26 +1979,28 @@ wg_lookup_peer_by_pubkey(struct wg_softc
+}
 static void
 wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp,
     struct wg_msg_cookie *wgmc, const uint32_t sender,
     const uint8_t mac1[WG_MAC_LEN], const struct sockaddr *src)
+{
 	uint8_t cookie[WG_COOKIE_LEN];
 	uint8_t key[WG_HASH_LEN];
 	uint8_t addr[sizeof(struct in6_addr)];
 	size_t addrlen;
 	uint16_t uh_sport; /* be */
 	KASSERT(mutex_owned(wgp->wgp_lock));
 	wgmc->wgmc_type = htole32(WG_MSG_TYPE_COOKIE);
 	wgmc->wgmc_receiver = sender;
 	cprng_fast(wgmc->wgmc_salt, sizeof(wgmc->wgmc_salt));
 	/*
 	 * [W] 5.4.7: Under Load: Cookie Reply Message
 	 * "The secret variable, Rm, changes every two minutes to a
 	 * random value"
 	 */
 	if ((time_uptime - wgp->wgp_last_genrandval_time) > WG_RANDVAL_TIME) {
 		wgp->wgp_randval = cprng_strong32();
 		wgp->wgp_last_genrandval_time = time_uptime;
+	}
 @@ -2009,26 +2038,28 @@ wg_fill_msg_cookie(struct wg_softc *wg,
 	memcpy(wgp->wgp_last_sent_cookie, cookie, sizeof(cookie));
 	wgp->wgp_last_sent_cookie_valid = true;
+}
 static int
 wg_send_cookie_msg(struct wg_softc *wg, struct wg_peer *wgp,
     const uint32_t sender, const uint8_t mac1[WG_MAC_LEN],
     const struct sockaddr *src)
+{
 	int error;
 	struct mbuf *m;
 	struct wg_msg_cookie *wgmc;
 	KASSERT(mutex_owned(wgp->wgp_lock));
 	m = m_gethdr(M_WAIT, MT_DATA);
 	m->m_pkthdr.len = m->m_len = sizeof(*wgmc);
 	wgmc = mtod(m, struct wg_msg_cookie *);
 	wg_fill_msg_cookie(wg, wgp, wgmc, sender, mac1, src);
 	error = wg->wg_ops->send_hs_msg(wgp, m);
 	if (error == 0)
 		WG_TRACE("cookie msg sent");
 	return error;
+}
 static bool
 wg_is_underload(struct wg_softc *wg, struct wg_peer *wgp, int msgtype)
 @@ -2043,26 +2074,28 @@ wg_is_underload(struct wg_softc *wg, str
 	 * the mechanism is a DoS mitigation, so we consider frequent handshake
 	 * messages as (a kind of) load; if a message of the same type comes
 	 * to a peer within 1 second, we consider we are under load.
 	 */
 	time_t last = wgp->wgp_last_msg_received_time[msgtype];
 	wgp->wgp_last_msg_received_time[msgtype] = time_uptime;
 	return (time_uptime - last) == 0;
+}
 static void
 wg_calculate_keys(struct wg_session *wgs, const bool initiator)
+{
 	KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock));
 	/*
 	 * [W] 5.4.5: Ti^send = Tr^recv, Ti^recv = Tr^send := KDF2(Ci = Cr, e)
 	 */
 	if (initiator) {
 		wg_algo_kdf(wgs->wgs_tkey_send, wgs->wgs_tkey_recv, NULL,
 		    wgs->wgs_chaining_key, NULL, 0);
 	} else {
 		wg_algo_kdf(wgs->wgs_tkey_recv, wgs->wgs_tkey_send, NULL,
 		    wgs->wgs_chaining_key, NULL, 0);
+	}
 	WG_DUMP_HASH("wgs_tkey_send", wgs->wgs_tkey_send);
 	WG_DUMP_HASH("wgs_tkey_recv", wgs->wgs_tkey_recv);
+}
 @@ -2093,48 +2126,53 @@ wg_session_inc_send_counter(struct wg_se
 	mutex_enter(&wgs->wgs_send_counter_lock);
 	send_counter = wgs->wgs_send_counter++;
 	mutex_exit(&wgs->wgs_send_counter_lock);
 	return send_counter;
 #endif
+}
 static void
 wg_clear_states(struct wg_session *wgs)
+{
 	KASSERT(mutex_owned(wgs->wgs_peer->wgp_lock));
 	wgs->wgs_send_counter = 0;
 	sliwin_reset(&wgs->wgs_recvwin->window);
 #define wgs_clear(v)	explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v))
 	wgs_clear(handshake_hash);
 	wgs_clear(chaining_key);
 	wgs_clear(ephemeral_key_pub);
 	wgs_clear(ephemeral_key_priv);
 	wgs_clear(ephemeral_key_peer);
 #undef wgs_clear
+}
 static struct wg_session *
 wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index,
     struct psref *psref)
+{
 	struct wg_session *wgs;
 	int s = pserialize_read_enter();
 	wgs = thmap_get(wg->wg_sessions_byindex, &index, sizeof index);
-	if (wgs != NULL)
+	if (wgs != NULL) {
 		KASSERT(atomic_load_relaxed(&wgs->wgs_state) !=
 		    WGS_STATE_UNKNOWN);
 		psref_acquire(psref, &wgs->wgs_psref, wg_psref_class);
+	}
 	pserialize_read_exit(s);
 	return wgs;
+}
 static void
 wg_schedule_rekey_timer(struct wg_peer *wgp)
+{
 	int timeout = MIN(wg_rekey_after_time, INT_MAX/hz);
 	callout_schedule(&wgp->wgp_rekey_timer, timeout * hz);
+}
 @@ -2171,39 +2209,36 @@ wg_need_to_send_init_message(struct wg_s
 static void
 wg_schedule_peer_task(struct wg_peer *wgp, int task)
+{
 	atomic_or_uint(&wgp->wgp_tasks, task);
 	WG_DLOG("tasks=%d, task=%d\n", wgp->wgp_tasks, task);
 	wg_wakeup_worker(wgp->wgp_sc->wg_worker, WG_WAKEUP_REASON_PEER);
+}
 static void
 wg_change_endpoint(struct wg_peer *wgp, const struct sockaddr *new)
+{
 	struct wg_sockaddr *wgsa_prev;
 	KASSERT(mutex_owned(wgp->wgp_lock));
 	WG_TRACE("Changing endpoint");
 	memcpy(wgp->wgp_endpoint0, new, new->sa_len);
-#ifndef __HAVE_ATOMIC_AS_MEMBAR	/* store-release */
+	wgsa_prev = wgp->wgp_endpoint;
-	membar_exit();
+	atomic_store_release(&wgp->wgp_endpoint, wgp->wgp_endpoint0);
-#endif
+	wgp->wgp_endpoint0 = wgsa_prev;
-	wgp->wgp_endpoint0 = atomic_swap_ptr(&wgp->wgp_endpoint,
+	atomic_store_release(&wgp->wgp_endpoint_available, true);
 	    wgp->wgp_endpoint0);
 	wgp->wgp_endpoint_available = true;
 	wgp->wgp_endpoint_changing = true;
 	wg_schedule_peer_task(wgp, WGP_TASK_ENDPOINT_CHANGED);
+}
 static bool
 wg_validate_inner_packet(const char *packet, size_t decrypted_len, int *af)
+{
 	uint16_t packet_len;
 	const struct ip *ip;
 	if (__predict_false(decrypted_len < sizeof(struct ip)))
 		return false;
 	ip = (const struct ip *)packet;
 @@ -2271,33 +2306,26 @@ wg_validate_route(struct wg_softc *wg, s
 	if (wgp != NULL)
 		wg_put_peer(wgp, &psref);
 	return ok;
+}
 static void
 wg_session_dtor_timer(void *arg)
+{
 	struct wg_peer *wgp = arg;
 	WG_TRACE("enter");
 	mutex_enter(wgp->wgp_lock);
 	if (__predict_false(wgp->wgp_state == WGP_STATE_DESTROYING)) {
 		mutex_exit(wgp->wgp_lock);
 		return;
 	mutex_exit(wgp->wgp_lock);
 	wg_schedule_peer_task(wgp, WGP_TASK_DESTROY_PREV_SESSION);
+}
 static void
 wg_schedule_session_dtor_timer(struct wg_peer *wgp)
+{
 	/* 1 second grace period */
 	callout_schedule(&wgp->wgp_session_dtor_timer, hz);
+}
 static bool
 sockaddr_port_match(const struct sockaddr *sa1, const struct sockaddr *sa2)
 @@ -2327,200 +2355,218 @@ wg_update_endpoint_if_necessary(struct w
 #ifdef WG_DEBUG_LOG
 	char oldaddr[128], newaddr[128];
 	sockaddr_format(wgsatosa(wgsa), oldaddr, sizeof(oldaddr));
 	sockaddr_format(src, newaddr, sizeof(newaddr));
 	WG_DLOG("old=%s, new=%s\n", oldaddr, newaddr);
 #endif
 	/*
 	 * III: "Since the packet has authenticated correctly, the source IP of
 	 * the outer UDP/IP packet is used to update the endpoint for peer..."
 	 */
 	if (__predict_false(sockaddr_cmp(src, wgsatosa(wgsa)) != 0 ||
 		!sockaddr_port_match(src, wgsatosa(wgsa)))) {
 		mutex_enter(wgp->wgp_lock);
 		/* XXX We can't change the endpoint twice in a short period */
-		if (!wgp->wgp_endpoint_changing) {
+		if (atomic_swap_uint(&wgp->wgp_endpoint_changing, 1) == 0) {
 			wg_change_endpoint(wgp, src);
+		}
 		mutex_exit(wgp->wgp_lock);
+	}
 	wg_put_sa(wgp, wgsa, &psref);
+}
 static void
 wg_handle_msg_data(struct wg_softc *wg, struct mbuf *m,
     const struct sockaddr *src)
+{
 	struct wg_msg_data *wgmd;
 	char *encrypted_buf = NULL, *decrypted_buf;
 	size_t encrypted_len, decrypted_len;
 	struct wg_session *wgs;
 	struct wg_peer *wgp;
 	int state;
 	size_t mlen;
 	struct psref psref;
 	int error, af;
 	bool success, free_encrypted_buf = false, ok;
 	struct mbuf *n;
 	KASSERT(m->m_len >= sizeof(struct wg_msg_data));
 	wgmd = mtod(m, struct wg_msg_data *);
 	KASSERT(wgmd->wgmd_type == htole32(WG_MSG_TYPE_DATA));
 	WG_TRACE("data");
 	/* Find the putative session, or drop.  */
 	wgs = wg_lookup_session_by_index(wg, wgmd->wgmd_receiver, &psref);
 	if (wgs == NULL) {
 		WG_TRACE("No session found");
 		m_freem(m);
 		return;
+	}
 	/*
 	 * We are only ready to handle data when in INIT_PASSIVE,
 	 * ESTABLISHED, or DESTROYING.  All transitions out of that
 	 * state dissociate the session index and drain psrefs.
 	 */
 	state = atomic_load_relaxed(&wgs->wgs_state);
 	switch (state) {
 	case WGS_STATE_UNKNOWN:
 		panic("wg session %p in unknown state has session index %u",
 		    wgs, wgmd->wgmd_receiver);
 	case WGS_STATE_INIT_ACTIVE:
 		WG_TRACE("not yet ready for data");
 		goto out;
 	case WGS_STATE_INIT_PASSIVE:
 	case WGS_STATE_ESTABLISHED:
 	case WGS_STATE_DESTROYING:
 		break;
+	}
 	/*
 	 * Get the peer, for rate-limited logs (XXX MPSAFE, dtrace) and
 	 * to update the endpoint if authentication succeeds.
 	 */
 	wgp = wgs->wgs_peer;
 	/*
 	 * Reject outrageously wrong sequence numbers before doing any
 	 * crypto work or taking any locks.
 	 */
 	error = sliwin_check_fast(&wgs->wgs_recvwin->window,
 	    le64toh(wgmd->wgmd_counter));
 	if (error) {
 		WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
 		    "out-of-window packet: %"PRIu64"\n",
 		    le64toh(wgmd->wgmd_counter));
 		goto out;
+	}
 	/* Ensure the payload and authenticator are contiguous.  */
 	mlen = m_length(m);
 	encrypted_len = mlen - sizeof(*wgmd);
 	if (encrypted_len < WG_AUTHTAG_LEN) {
 		WG_DLOG("Short encrypted_len: %lu\n", encrypted_len);
 		goto out;
+	}
 	success = m_ensure_contig(&m, sizeof(*wgmd) + encrypted_len);
 	if (success) {
 		encrypted_buf = mtod(m, char *) + sizeof(*wgmd);
 	} else {
 		encrypted_buf = kmem_intr_alloc(encrypted_len, KM_NOSLEEP);
 		if (encrypted_buf == NULL) {
 			WG_DLOG("failed to allocate encrypted_buf\n");
 			goto out;
+		}
 		m_copydata(m, sizeof(*wgmd), encrypted_len, encrypted_buf);
 		free_encrypted_buf = true;
+	}
 	/* m_ensure_contig may change m regardless of its result */
 	KASSERT(m->m_len >= sizeof(*wgmd));
 	wgmd = mtod(m, struct wg_msg_data *);
 	/*
 	 * Get a buffer for the plaintext.  Add WG_AUTHTAG_LEN to avoid
 	 * a zero-length buffer (XXX).  Drop if plaintext is longer
 	 * than MCLBYTES (XXX).
 	 */
 	decrypted_len = encrypted_len - WG_AUTHTAG_LEN;
 	if (decrypted_len > MCLBYTES) {
 		/* FIXME handle larger data than MCLBYTES */
 		WG_DLOG("couldn't handle larger data than MCLBYTES\n");
 		goto out;
+	}
 	/* To avoid zero length */
 	n = wg_get_mbuf(0, decrypted_len + WG_AUTHTAG_LEN);
 	if (n == NULL) {
 		WG_DLOG("wg_get_mbuf failed\n");
 		goto out;
+	}
 	decrypted_buf = mtod(n, char *);
 	/* Decrypt and verify the packet.  */
 	WG_DLOG("mlen=%lu, encrypted_len=%lu\n", mlen, encrypted_len);
 	error = wg_algo_aead_dec(decrypted_buf,
 	    encrypted_len - WG_AUTHTAG_LEN /* can be 0 */,
 	    wgs->wgs_tkey_recv, le64toh(wgmd->wgmd_counter), encrypted_buf,
 	    encrypted_len, NULL, 0);
 	if (error != 0) {
 		WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
 		    "failed to wg_algo_aead_dec\n");
 		m_freem(n);
 		goto out;
+	}
 	WG_DLOG("outsize=%u\n", (u_int)decrypted_len);
 	/* Packet is genuine.  Reject it if a replay or just too old.  */
 	mutex_enter(&wgs->wgs_recvwin->lock);
 	error = sliwin_update(&wgs->wgs_recvwin->window,
 	    le64toh(wgmd->wgmd_counter));
 	mutex_exit(&wgs->wgs_recvwin->lock);
 	if (error) {
 		WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
 		    "replay or out-of-window packet: %"PRIu64"\n",
 		    le64toh(wgmd->wgmd_counter));
 		m_freem(n);
 		goto out;
+	}
 	/* We're done with m now; free it and chuck the pointers.  */
 	m_freem(m);
 	m = NULL;
 	wgmd = NULL;
 	/*
 	 * Validate the encapsulated packet header and get the address
 	 * family, or drop.
 	 */
 	ok = wg_validate_inner_packet(decrypted_buf, decrypted_len, &af);
 	if (!ok) {
 		/* something wrong... */
 		m_freem(n);
 		goto out;
+	}
 	/*
 	 * The packet is genuine.  Update the peer's endpoint if the
 	 * source address changed.
+	 *
 	 * XXX How to prevent DoS by replaying genuine packets from the
 	 * wrong source address?
 	 */
 	wg_update_endpoint_if_necessary(wgp, src);
 	/* Submit it into our network stack if routable.  */
 	ok = wg_validate_route(wg, wgp, af, decrypted_buf);
 	if (ok) {
 		wg->wg_ops->input(&wg->wg_if, n, af);
 	} else {
 		WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
 		    "invalid source address\n");
 		m_freem(n);
 		/*
 		 * The inner address is invalid however the session is valid
 		 * so continue the session processing below.
 		 */
+	}
 	n = NULL;
-	if (wgs->wgs_state == WGS_STATE_INIT_PASSIVE) {
+	/* Update the state machine if necessary.  */
-		struct wg_session *wgs_prev;
+	if (__predict_false(state == WGS_STATE_INIT_PASSIVE)) {
 		/*
-		KASSERT(wgs == wgp->wgp_session_unstable);
+		 * We were waiting for the initiator to send their
-		wgs->wgs_state = WGS_STATE_ESTABLISHED;
+		 * first data transport message, and that has happened.
-		wgs->wgs_time_established = time_uptime;
+		 * Schedule a task to establish this session.
-		wgs->wgs_time_last_data_sent = 0;
+		 */
-		wgs->wgs_is_initiator = false;
+		wg_schedule_peer_task(wgp, WGP_TASK_ESTABLISH_SESSION);
 		WG_TRACE("WGS_STATE_ESTABLISHED");
 		mutex_enter(wgp->wgp_lock);
 		wg_swap_sessions(wgp);
 		wgs_prev = wgp->wgp_session_unstable;
 		mutex_enter(wgs_prev->wgs_lock);
 		getnanotime(&wgp->wgp_last_handshake_time);
 		wgp->wgp_handshake_start_time = 0;
 		wgp->wgp_last_sent_mac1_valid = false;
 		wgp->wgp_last_sent_cookie_valid = false;
 		mutex_exit(wgp->wgp_lock);
 		if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) {
 			wgs_prev->wgs_state = WGS_STATE_DESTROYING;
 			/* We can't destroy the old session immediately */
 			wg_schedule_session_dtor_timer(wgp);
 		} else {
 			wg_clear_states(wgs_prev);
 			wgs_prev->wgs_state = WGS_STATE_UNKNOWN;
 		mutex_exit(wgs_prev->wgs_lock);
 		/* Anyway run a softint to flush pending packets */
 		kpreempt_disable();
 		softint_schedule(wgp->wgp_si);
 		kpreempt_enable();
 	} else {
 		if (__predict_false(wg_need_to_send_init_message(wgs))) {
 			wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
+		}
 		/*
 		 * [W] 6.5 Passive Keepalive
 		 * "If a peer has received a validly-authenticated transport
 		 *  data message (section 5.4.6), but does not have any packets
 		 *  itself to send back for KEEPALIVE-TIMEOUT seconds, it sends
 		 *  a keepalive message."
 		 */
 		WG_DLOG("time_uptime=%lu wgs_time_last_data_sent=%lu\n",
 		    time_uptime, wgs->wgs_time_last_data_sent);
 @@ -2545,58 +2591,65 @@ out:
+}
 static void
 wg_handle_msg_cookie(struct wg_softc *wg, const struct wg_msg_cookie *wgmc)
+{
 	struct wg_session *wgs;
 	struct wg_peer *wgp;
 	struct psref psref;
 	int error;
 	uint8_t key[WG_HASH_LEN];
 	uint8_t cookie[WG_COOKIE_LEN];
 	WG_TRACE("cookie msg received");
 	/* Find the putative session.  */
 	wgs = wg_lookup_session_by_index(wg, wgmc->wgmc_receiver, &psref);
 	if (wgs == NULL) {
 		WG_TRACE("No session found");
 		return;
+	}
 	/* Lock the peer so we can update the cookie state.  */
 	wgp = wgs->wgs_peer;
 	mutex_enter(wgp->wgp_lock);
 	if (!wgp->wgp_last_sent_mac1_valid) {
 		WG_TRACE("No valid mac1 sent (or expired)");
 		goto out;
+	}
 	/* Decrypt the cookie and store it for later handshake retry.  */
 	wg_algo_mac_cookie(key, sizeof(key), wgp->wgp_pubkey,
 	    sizeof(wgp->wgp_pubkey));
 	error = wg_algo_xaead_dec(cookie, sizeof(cookie), key,
 	    wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie),
 	    wgp->wgp_last_sent_mac1, sizeof(wgp->wgp_last_sent_mac1),
 	    wgmc->wgmc_salt);
 	if (error != 0) {
 		WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
 		    "wg_algo_aead_dec for cookie failed: error=%d\n", error);
 		goto out;
+	}
 	/*
 	 * [W] 6.6: Interaction with Cookie Reply System
 	 * "it should simply store the decrypted cookie value from the cookie
 	 *  reply message, and wait for the expiration of the REKEY-TIMEOUT
 	 *  timer for retrying a handshake initiation message."
 	 */
 	wgp->wgp_latest_cookie_time = time_uptime;
 	memcpy(wgp->wgp_latest_cookie, cookie, sizeof(wgp->wgp_latest_cookie));
 out:
 	mutex_exit(wgp->wgp_lock);
 	wg_put_session(wgs, &psref);
+}
 static struct mbuf *
 wg_validate_msg_header(struct wg_softc *wg, struct mbuf *m)
+{
 	struct wg_msg wgm;
 	size_t mbuflen;
 	size_t msglen;
 	/*
 	 * Get the mbuf chain length.  It is already guaranteed, by
 	 * wg_overudp_cb, to be large enough for a struct wg_msg.
 @@ -2720,135 +2773,224 @@ wg_get_peer(struct wg_peer *wgp, struct
 	psref_acquire(psref, &wgp->wgp_psref, wg_psref_class);
+}
 static void
 wg_put_peer(struct wg_peer *wgp, struct psref *psref)
+{
 	psref_release(psref, &wgp->wgp_psref, wg_psref_class);
+}
 static void
 wg_task_send_init_message(struct wg_softc *wg, struct wg_peer *wgp)
+{
 	struct psref psref;
 	struct wg_session *wgs;
 	WG_TRACE("WGP_TASK_SEND_INIT_MESSAGE");
-	if (!wgp->wgp_endpoint_available) {
+	KASSERT(mutex_owned(wgp->wgp_lock));
 	if (!atomic_load_acquire(&wgp->wgp_endpoint_available)) {
 		WGLOG(LOG_DEBUG, "No endpoint available\n");
 		/* XXX should do something? */
 		return;
+	}
-	wgs = wg_get_stable_session(wgp, &psref);
+	wgs = wgp->wgp_session_stable;
 	if (wgs->wgs_state == WGS_STATE_UNKNOWN) {
-		wg_put_session(wgs, &psref);
+		/* XXX What if the unstable session is already INIT_ACTIVE?  */
 		wg_send_handshake_msg_init(wg, wgp);
 	} else {
 		wg_put_session(wgs, &psref);
 		/* rekey */
-		wgs = wg_get_unstable_session(wgp, &psref);
+		wgs = wgp->wgp_session_unstable;
 		if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE)
 			wg_send_handshake_msg_init(wg, wgp);
 		wg_put_session(wgs, &psref);
+	}
+}
 static void
 wg_task_retry_handshake(struct wg_softc *wg, struct wg_peer *wgp)
+{
 	struct wg_session *wgs;
 	WG_TRACE("WGP_TASK_RETRY_HANDSHAKE");
 	KASSERT(mutex_owned(wgp->wgp_lock));
 	KASSERT(wgp->wgp_handshake_start_time != 0);
 	wgs = wgp->wgp_session_unstable;
 	if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE)
 		return;
 	/*
 	 * XXX no real need to assign a new index here, but we do need
 	 * to transition to UNKNOWN temporarily
 	 */
 	wg_put_session_index(wg, wgs);
 	/* [W] 6.4 Handshake Initiation Retransmission */
 	if ((time_uptime - wgp->wgp_handshake_start_time) >
 	    wg_rekey_attempt_time) {
 		/* Give up handshaking */
 		wgp->wgp_handshake_start_time = 0;
 		WG_TRACE("give up");
 		/*
 		 * If a new data packet comes, handshaking will be retried
 		 * and a new session would be established at that time,
 		 * however we don't want to send pending packets then.
 		 */
 		wg_purge_pending_packets(wgp);
 		return;
+	}
 	wg_task_send_init_message(wg, wgp);
+}
 static void
 wg_task_establish_session(struct wg_softc *wg, struct wg_peer *wgp)
+{
 	struct wg_session *wgs, *wgs_prev;
 	KASSERT(mutex_owned(wgp->wgp_lock));
 	wgs = wgp->wgp_session_unstable;
 	if (wgs->wgs_state != WGS_STATE_INIT_PASSIVE)
 		/* XXX Can this happen?  */
 		return;
 	wgs->wgs_state = WGS_STATE_ESTABLISHED;
 	wgs->wgs_time_established = time_uptime;
 	wgs->wgs_time_last_data_sent = 0;
 	wgs->wgs_is_initiator = false;
 	WG_TRACE("WGS_STATE_ESTABLISHED");
 	wg_swap_sessions(wgp);
 	KASSERT(wgs == wgp->wgp_session_stable);
 	wgs_prev = wgp->wgp_session_unstable;
 	getnanotime(&wgp->wgp_last_handshake_time);
 	wgp->wgp_handshake_start_time = 0;
 	wgp->wgp_last_sent_mac1_valid = false;
 	wgp->wgp_last_sent_cookie_valid = false;
 	if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) {
 		/* Wait for wg_get_stable_session to drain.  */
 		pserialize_perform(wgp->wgp_psz);
 		/* Transition ESTABLISHED->DESTROYING.  */
 		wgs_prev->wgs_state = WGS_STATE_DESTROYING;
 		/* We can't destroy the old session immediately */
 		wg_schedule_session_dtor_timer(wgp);
 	} else {
 		KASSERTMSG(wgs_prev->wgs_state == WGS_STATE_UNKNOWN,
 		    "state=%d", wgs_prev->wgs_state);
 		wg_clear_states(wgs_prev);
 		wgs_prev->wgs_state = WGS_STATE_UNKNOWN;
+	}
 	/* Anyway run a softint to flush pending packets */
 	kpreempt_disable();
 	softint_schedule(wgp->wgp_si);
 	kpreempt_enable();
+}
 static void
 wg_task_endpoint_changed(struct wg_softc *wg, struct wg_peer *wgp)
+{
 	WG_TRACE("WGP_TASK_ENDPOINT_CHANGED");
-	mutex_enter(wgp->wgp_lock);
+	KASSERT(mutex_owned(wgp->wgp_lock));
 	if (wgp->wgp_endpoint_changing) {
 	if (atomic_load_relaxed(&wgp->wgp_endpoint_changing)) {
 		pserialize_perform(wgp->wgp_psz);
 		psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref,
 		    wg_psref_class);
 		psref_target_init(&wgp->wgp_endpoint0->wgsa_psref,
 		    wg_psref_class);
-		wgp->wgp_endpoint_changing = false;
+		atomic_store_release(&wgp->wgp_endpoint_changing, 0);
+	}
 	mutex_exit(wgp->wgp_lock);
+}
 static void
 wg_task_send_keepalive_message(struct wg_softc *wg, struct wg_peer *wgp)
+{
 	struct psref psref;
 	struct wg_session *wgs;
 	WG_TRACE("WGP_TASK_SEND_KEEPALIVE_MESSAGE");
-	wgs = wg_get_stable_session(wgp, &psref);
+	KASSERT(mutex_owned(wgp->wgp_lock));
 	wgs = wgp->wgp_session_stable;
 	if (wgs->wgs_state != WGS_STATE_ESTABLISHED)
 		return;
 	wg_send_keepalive_msg(wgp, wgs);
 	wg_put_session(wgs, &psref);
+}
 static void
 wg_task_destroy_prev_session(struct wg_softc *wg, struct wg_peer *wgp)
+{
 	struct wg_session *wgs;
 	WG_TRACE("WGP_TASK_DESTROY_PREV_SESSION");
-	mutex_enter(wgp->wgp_lock);
+	KASSERT(mutex_owned(wgp->wgp_lock));
 	wgs = wgp->wgp_session_unstable;
 	mutex_enter(wgs->wgs_lock);
 	if (wgs->wgs_state == WGS_STATE_DESTROYING) {
-		pserialize_perform(wgp->wgp_psz);
+		wg_put_session_index(wg, wgs);
 		psref_target_destroy(&wgs->wgs_psref, wg_psref_class);
 		psref_target_init(&wgs->wgs_psref, wg_psref_class);
 		wg_clear_states(wgs);
 		wgs->wgs_state = WGS_STATE_UNKNOWN;
+	}
 	mutex_exit(wgs->wgs_lock);
 	mutex_exit(wgp->wgp_lock);
+}
 static void
 wg_process_peer_tasks(struct wg_softc *wg)
+{
 	struct wg_peer *wgp;
 	int s;
 	/* XXX should avoid checking all peers */
 	s = pserialize_read_enter();
 	WG_PEER_READER_FOREACH(wgp, wg) {
 		struct psref psref;
 		unsigned int tasks;
 		if (wgp->wgp_tasks == 0)
 			continue;
 		wg_get_peer(wgp, &psref);
 		pserialize_read_exit(s);
 	restart:
 		tasks = atomic_swap_uint(&wgp->wgp_tasks, 0);
 		KASSERT(tasks != 0);
 		WG_DLOG("tasks=%x\n", tasks);
 		mutex_enter(wgp->wgp_lock);
 		if (ISSET(tasks, WGP_TASK_SEND_INIT_MESSAGE))
 			wg_task_send_init_message(wg, wgp);
 		if (ISSET(tasks, WGP_TASK_RETRY_HANDSHAKE))
 			wg_task_retry_handshake(wg, wgp);
 		if (ISSET(tasks, WGP_TASK_ESTABLISH_SESSION))
 			wg_task_establish_session(wg, wgp);
 		if (ISSET(tasks, WGP_TASK_ENDPOINT_CHANGED))
 			wg_task_endpoint_changed(wg, wgp);
 		if (ISSET(tasks, WGP_TASK_SEND_KEEPALIVE_MESSAGE))
 			wg_task_send_keepalive_message(wg, wgp);
 		if (ISSET(tasks, WGP_TASK_DESTROY_PREV_SESSION))
 			wg_task_destroy_prev_session(wg, wgp);
 		mutex_exit(wgp->wgp_lock);
 		/* New tasks may be scheduled during processing tasks */
 		WG_DLOG("wgp_tasks=%d\n", wgp->wgp_tasks);
 		if (wgp->wgp_tasks != 0)
 			goto restart;
 		s = pserialize_read_enter();
 		wg_put_peer(wgp, &psref);
+	}
 	pserialize_read_exit(s);
+}
 static void
 @@ -3026,27 +3168,27 @@ wg_worker_socreate(struct wg_softc *wg,
 	return 0;
+}
 static int
 wg_worker_init(struct wg_softc *wg)
+{
 	int error;
 	struct wg_worker *wgw;
 	const char *ifname = wg->wg_if.if_xname;
 	struct socket *so;
 	wgw = kmem_zalloc(sizeof(struct wg_worker), KM_SLEEP);
-	mutex_init(&wgw->wgw_lock, MUTEX_DEFAULT, IPL_NONE);
+	mutex_init(&wgw->wgw_lock, MUTEX_DEFAULT, IPL_SOFTNET);
 	cv_init(&wgw->wgw_cv, ifname);
 	wgw->wgw_todie = false;
 	wgw->wgw_wakeup_reasons = 0;
 	error = wg_worker_socreate(wg, wgw, AF_INET, &so);
 	if (error != 0)
 		goto error;
 	wgw->wgw_so4 = so;
 #ifdef INET6
 	error = wg_worker_socreate(wg, wgw, AF_INET6, &so);
 	if (error != 0)
 		goto error;
 	wgw->wgw_so6 = so;
 @@ -3119,123 +3261,80 @@ wg_session_hit_limits(struct wg_session
+	}
 	return false;
+}
 static void
 wg_peer_softint(void *arg)
+{
 	struct wg_peer *wgp = arg;
 	struct wg_session *wgs;
 	struct mbuf *m;
 	struct psref psref;
-	wgs = wg_get_stable_session(wgp, &psref);
+	if ((wgs = wg_get_stable_session(wgp, &psref)) == NULL) {
 	if (wgs->wgs_state != WGS_STATE_ESTABLISHED) {
 		/* XXX how to treat? */
 		WG_TRACE("skipped");
-		goto out;
+		return;
+	}
 	if (wg_session_hit_limits(wgs)) {
 		wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
 		goto out;
+	}
 	WG_TRACE("running");
 	while ((m = pcq_get(wgp->wgp_q)) != NULL) {
 		wg_send_data_msg(wgp, wgs, m);
+	}
 out:
 	wg_put_session(wgs, &psref);
+}
 static void
 wg_rekey_timer(void *arg)
+{
 	struct wg_peer *wgp = arg;
-	mutex_enter(wgp->wgp_lock);
+	wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
 	if (__predict_true(wgp->wgp_state != WGP_STATE_DESTROYING)) {
 		wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
 	mutex_exit(wgp->wgp_lock);
+}
 static void
 wg_purge_pending_packets(struct wg_peer *wgp)
+{
 	struct mbuf *m;
 	while ((m = pcq_get(wgp->wgp_q)) != NULL) {
 		m_freem(m);
+	}
+}
 static void
 wg_handshake_timeout_timer(void *arg)
+{
 	struct wg_peer *wgp = arg;
 	struct wg_session *wgs;
 	struct psref psref;
 	WG_TRACE("enter");
-	mutex_enter(wgp->wgp_lock);
+	wg_schedule_peer_task(wgp, WGP_TASK_RETRY_HANDSHAKE);
 	if (__predict_false(wgp->wgp_state == WGP_STATE_DESTROYING)) {
 		mutex_exit(wgp->wgp_lock);
 		return;
 	mutex_exit(wgp->wgp_lock);
 	KASSERT(wgp->wgp_handshake_start_time != 0);
 	wgs = wg_get_unstable_session(wgp, &psref);
 	KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE);
 	/* [W] 6.4 Handshake Initiation Retransmission */
 	if ((time_uptime - wgp->wgp_handshake_start_time) >
 	    wg_rekey_attempt_time) {
 		/* Give up handshaking */
 		wgs->wgs_state = WGS_STATE_UNKNOWN;
 		wg_clear_states(wgs);
 		wgp->wgp_state = WGP_STATE_GIVEUP;
 		wgp->wgp_handshake_start_time = 0;
 		wg_put_session(wgs, &psref);
 		WG_TRACE("give up");
 		/*
 		 * If a new data packet comes, handshaking will be retried
 		 * and a new session would be established at that time,
 		 * however we don't want to send pending packets then.
 		 */
 		wg_purge_pending_packets(wgp);
 		return;
 	/* No response for an initiation message sent, retry handshaking */
 	wgs->wgs_state = WGS_STATE_UNKNOWN;
 	wg_clear_states(wgs);
 	wg_put_session(wgs, &psref);
 	wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
+}
 static struct wg_peer *
 wg_alloc_peer(struct wg_softc *wg)
+{
 	struct wg_peer *wgp;
 	wgp = kmem_zalloc(sizeof(*wgp), KM_SLEEP);
 	wgp->wgp_sc = wg;
 	wgp->wgp_state = WGP_STATE_INIT;
 	wgp->wgp_q = pcq_create(1024, KM_SLEEP);
 	wgp->wgp_si = softint_establish(SOFTINT_NET, wg_peer_softint, wgp);
 	callout_init(&wgp->wgp_rekey_timer, CALLOUT_MPSAFE);
 	callout_setfunc(&wgp->wgp_rekey_timer, wg_rekey_timer, wgp);
 	callout_init(&wgp->wgp_handshake_timeout_timer, CALLOUT_MPSAFE);
 	callout_setfunc(&wgp->wgp_handshake_timeout_timer,
 	    wg_handshake_timeout_timer, wgp);
 	callout_init(&wgp->wgp_session_dtor_timer, CALLOUT_MPSAFE);
 	callout_setfunc(&wgp->wgp_session_dtor_timer,
 	    wg_session_dtor_timer, wgp);
 	PSLIST_ENTRY_INIT(wgp, wgp_peerlist_entry);
 	wgp->wgp_endpoint_changing = false;
 	wgp->wgp_endpoint_available = false;
 @@ -3247,54 +3346,50 @@ wg_alloc_peer(struct wg_softc *wg)
 	wgp->wgp_endpoint0 = kmem_zalloc(sizeof(*wgp->wgp_endpoint0), KM_SLEEP);
 	psref_target_init(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class);
 	psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class);
 	struct wg_session *wgs;
 	wgp->wgp_session_stable =
 	    kmem_zalloc(sizeof(*wgp->wgp_session_stable), KM_SLEEP);
 	wgp->wgp_session_unstable =
 	    kmem_zalloc(sizeof(*wgp->wgp_session_unstable), KM_SLEEP);
 	wgs = wgp->wgp_session_stable;
 	wgs->wgs_peer = wgp;
 	wgs->wgs_state = WGS_STATE_UNKNOWN;
 	psref_target_init(&wgs->wgs_psref, wg_psref_class);
 	wgs->wgs_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
 #ifndef __HAVE_ATOMIC64_LOADSTORE
 	mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET);
 #endif
 	wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP);
-	mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_NONE);
+	mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET);
 	wgs = wgp->wgp_session_unstable;
 	wgs->wgs_peer = wgp;
 	wgs->wgs_state = WGS_STATE_UNKNOWN;
 	psref_target_init(&wgs->wgs_psref, wg_psref_class);
 	wgs->wgs_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
 #ifndef __HAVE_ATOMIC64_LOADSTORE
 	mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET);
 #endif
 	wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP);
-	mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_NONE);
+	mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_SOFTNET);
 	return wgp;
+}
 static void
 wg_destroy_peer(struct wg_peer *wgp)
+{
 	struct wg_session *wgs;
 	struct wg_softc *wg = wgp->wgp_sc;
 	uint32_t index;
 	void *garbage;
 	/* Prevent new packets from this peer on any source address.  */
 	rw_enter(wg->wg_rwlock, RW_WRITER);
 	for (int i = 0; i < wgp->wgp_n_allowedips; i++) {
 		struct wg_allowedip *wga = &wgp->wgp_allowedips[i];
 		struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family);
 		struct radix_node *rn;
 		KASSERT(rnh != NULL);
 		rn = rnh->rnh_deladdr(&wga->wga_sa_addr,
 		    &wga->wga_sa_mask, rnh);
 		if (rn == NULL) {
 			char addrstr[128];
 @@ -3304,56 +3399,45 @@ wg_destroy_peer(struct wg_peer *wgp)
+		}
+	}
 	rw_exit(wg->wg_rwlock);
 	/* Purge pending packets.  */
 	wg_purge_pending_packets(wgp);
 	/* Halt all packet processing and timeouts.  */
 	softint_disestablish(wgp->wgp_si);
 	callout_halt(&wgp->wgp_rekey_timer, NULL);
 	callout_halt(&wgp->wgp_handshake_timeout_timer, NULL);
 	callout_halt(&wgp->wgp_session_dtor_timer, NULL);
 	/* Remove the sessions by index.  */
 	if ((index = wgp->wgp_session_stable->wgs_sender_index) != 0) {
 		thmap_del(wg->wg_sessions_byindex, &index, sizeof index);
 		wgp->wgp_session_stable->wgs_sender_index = 0;
 	if ((index = wgp->wgp_session_unstable->wgs_sender_index) != 0) {
 		thmap_del(wg->wg_sessions_byindex, &index, sizeof index);
 		wgp->wgp_session_unstable->wgs_sender_index = 0;
 	/* Wait for all thmap_gets to complete, and GC.  */
 	garbage = thmap_stage_gc(wg->wg_sessions_byindex);
 	mutex_enter(wgp->wgp_lock);
 	pserialize_perform(wgp->wgp_psz);
 	mutex_exit(wgp->wgp_lock);
 	thmap_gc(wg->wg_sessions_byindex, garbage);
 	wgs = wgp->wgp_session_unstable;
-	psref_target_destroy(&wgs->wgs_psref, wg_psref_class);
+	if (wgs->wgs_state != WGS_STATE_UNKNOWN) {
-	mutex_obj_free(wgs->wgs_lock);
+		mutex_enter(wgp->wgp_lock);
 		wg_destroy_session(wg, wgs);
 		mutex_exit(wgp->wgp_lock);
+	}
 	mutex_destroy(&wgs->wgs_recvwin->lock);
 	kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin));
 #ifndef __HAVE_ATOMIC64_LOADSTORE
 	mutex_destroy(&wgs->wgs_send_counter_lock);
 #endif
 	kmem_free(wgs, sizeof(*wgs));
 	wgs = wgp->wgp_session_stable;
-	psref_target_destroy(&wgs->wgs_psref, wg_psref_class);
+	if (wgs->wgs_state != WGS_STATE_UNKNOWN) {
-	mutex_obj_free(wgs->wgs_lock);
+		mutex_enter(wgp->wgp_lock);
 		wg_destroy_session(wg, wgs);
 		mutex_exit(wgp->wgp_lock);
+	}
 	mutex_destroy(&wgs->wgs_recvwin->lock);
 	kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin));
 #ifndef __HAVE_ATOMIC64_LOADSTORE
 	mutex_destroy(&wgs->wgs_send_counter_lock);
 #endif
 	kmem_free(wgs, sizeof(*wgs));
 	psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class);
 	psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class);
 	kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint));
 	kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0));
 	pserialize_destroy(wgp->wgp_psz);
 @@ -3376,27 +3460,26 @@ restart:
 		if (wgp->wgp_name[0]) {
 			wgp0 = thmap_del(wg->wg_peers_byname, wgp->wgp_name,
 			    strlen(wgp->wgp_name));
 			KASSERT(wgp0 == wgp);
 			garbage_byname = thmap_stage_gc(wg->wg_peers_byname);
+		}
 		wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey,
 		    sizeof(wgp->wgp_pubkey));
 		KASSERT(wgp0 == wgp);
 		garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey);
 		WG_PEER_WRITER_REMOVE(wgp);
 		wg->wg_npeers--;
 		mutex_enter(wgp->wgp_lock);
 		wgp->wgp_state = WGP_STATE_DESTROYING;
 		pserialize_perform(wgp->wgp_psz);
 		mutex_exit(wgp->wgp_lock);
 		PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry);
 		break;
+	}
 	mutex_exit(wg->wg_lock);
 	if (wgp == NULL)
 		return;
 	psref_target_destroy(&wgp->wgp_psref, wg_psref_class);
 	wg_destroy_peer(wgp);
 @@ -3413,27 +3496,26 @@ wg_destroy_peer_name(struct wg_softc *wg
 	void *garbage_byname, *garbage_bypubkey;
 	mutex_enter(wg->wg_lock);
 	wgp = thmap_del(wg->wg_peers_byname, name, strlen(name));
 	if (wgp != NULL) {
 		wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey,
 		    sizeof(wgp->wgp_pubkey));
 		KASSERT(wgp0 == wgp);
 		garbage_byname = thmap_stage_gc(wg->wg_peers_byname);
 		garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey);
 		WG_PEER_WRITER_REMOVE(wgp);
 		wg->wg_npeers--;
 		mutex_enter(wgp->wgp_lock);
 		wgp->wgp_state = WGP_STATE_DESTROYING;
 		pserialize_perform(wgp->wgp_psz);
 		mutex_exit(wgp->wgp_lock);
 		PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry);
+	}
 	mutex_exit(wg->wg_lock);
 	if (wgp == NULL)
 		return ENOENT;
 	psref_target_destroy(&wgp->wgp_psref, wg_psref_class);
 	wg_destroy_peer(wgp);
 	thmap_gc(wg->wg_peers_byname, garbage_byname);
 @@ -3597,95 +3679,93 @@ wg_pick_peer_by_sa(struct wg_softc *wg,
 out:
 	rw_exit(wg->wg_rwlock);
 	return wgp;
+}
 static void
 wg_fill_msg_data(struct wg_softc *wg, struct wg_peer *wgp,
     struct wg_session *wgs, struct wg_msg_data *wgmd)
+{
 	memset(wgmd, 0, sizeof(*wgmd));
 	wgmd->wgmd_type = htole32(WG_MSG_TYPE_DATA);
-	wgmd->wgmd_receiver = wgs->wgs_receiver_index;
+	wgmd->wgmd_receiver = wgs->wgs_remote_index;
 	/* [W] 5.4.6: msg.counter := Nm^send */
 	/* [W] 5.4.6: Nm^send := Nm^send + 1 */
 	wgmd->wgmd_counter = htole64(wg_session_inc_send_counter(wgs));
 	WG_DLOG("counter=%"PRIu64"\n", le64toh(wgmd->wgmd_counter));
+}
 static int
 wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
     const struct rtentry *rt)
+{
 	struct wg_softc *wg = ifp->if_softc;
-	int error = 0;
+	struct wg_peer *wgp = NULL;
 	struct wg_session *wgs = NULL;
 	struct psref wgp_psref, wgs_psref;
 	int bound;
-	struct psref psref;
+	int error;
 	bound = curlwp_bind();
 	/* TODO make the nest limit configurable via sysctl */
 	error = if_tunnel_check_nesting(ifp, m, 1);
-	if (error != 0) {
+	if (error) {
 		m_freem(m);
 		WGLOG(LOG_ERR, "tunneling loop detected and packet dropped\n");
-		return error;
+		goto out;
+	}
 	bound = curlwp_bind();
 	IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family);
 	bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT);
 	m->m_flags &= ~(M_BCAST|M_MCAST);
-	struct wg_peer *wgp = wg_pick_peer_by_sa(wg, dst, &psref);
+	wgp = wg_pick_peer_by_sa(wg, dst, &wgp_psref);
 	if (wgp == NULL) {
 		WG_TRACE("peer not found");
 		error = EHOSTUNREACH;
-		goto error;
+		goto out;
+	}
 	/* Clear checksum-offload flags. */
 	m->m_pkthdr.csum_flags = 0;
 	m->m_pkthdr.csum_data = 0;
 	if (!pcq_put(wgp->wgp_q, m)) {
 		error = ENOBUFS;
-		goto error;
+		goto out;
+	}
 	m = NULL;		/* consumed */
-	struct psref psref_wgs;
+	wgs = wg_get_stable_session(wgp, &wgs_psref);
-	struct wg_session *wgs;
+	if (wgs != NULL && !wg_session_hit_limits(wgs)) {
 	wgs = wg_get_stable_session(wgp, &psref_wgs);
 	if (wgs->wgs_state == WGS_STATE_ESTABLISHED &&
 	    !wg_session_hit_limits(wgs)) {
 		kpreempt_disable();
 		softint_schedule(wgp->wgp_si);
 		kpreempt_enable();
 		WG_TRACE("softint scheduled");
 	} else {
 		wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
 		WG_TRACE("softint NOT scheduled");
+	}
-	wg_put_session(wgs, &psref_wgs);
+	error = 0;
 	wg_put_peer(wgp, &psref);
 	return 0;
-error:
+out:
 	if (wgs != NULL)
 		wg_put_session(wgs, &wgs_psref);
 	if (wgp != NULL)
-		wg_put_peer(wgp, &psref);
+		wg_put_peer(wgp, &wgp_psref);
 	if (m != NULL)
 		m_freem(m);
 	curlwp_bindx(bound);
 	return error;
+}
 static int
 wg_send_udp(struct wg_peer *wgp, struct mbuf *m)
+{
 	struct psref psref;
 	struct wg_sockaddr *wgsa;
 	int error;
 	struct socket *so;