 @@ -1,14 +1,14 @@
-/*	$NetBSD: if_wg.c,v 1.36 2020/08/31 20:21:30 riastradh Exp $	*/
+/*	$NetBSD: if_wg.c,v 1.37 2020/08/31 20:23:56 riastradh Exp $	*/
 /*
  * Copyright (C) Ryota Ozaki <ozaki.ryota@gmail.com>
  * All rights reserved.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
 @@ -31,27 +31,27 @@
 /*
  * This network interface aims to implement the WireGuard protocol.
  * The implementation is based on the paper of WireGuard as of
  * 2018-06-30 [1].  The paper is referred in the source code with label
  * [W].  Also the specification of the Noise protocol framework as of
  * 2018-07-11 [2] is referred with label [N].
+ *
  * [1] https://www.wireguard.com/papers/wireguard.pdf
  * [2] http://noiseprotocol.org/noise.pdf
  */
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.36 2020/08/31 20:21:30 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.37 2020/08/31 20:23:56 riastradh Exp $");
 #ifdef _KERNEL_OPT
 #include "opt_inet.h"
 #endif
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/atomic.h>
 #include <sys/callout.h>
 #include <sys/cprng.h>
 #include <sys/cpu.h>
 #include <sys/device.h>
 @@ -67,26 +67,27 @@ __KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.
 #include <sys/mutex.h>
 #include <sys/pcq.h>
 #include <sys/percpu.h>
 #include <sys/pserialize.h>
 #include <sys/psref.h>
 #include <sys/queue.h>
 #include <sys/rwlock.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/sockio.h>
 #include <sys/sysctl.h>
 #include <sys/syslog.h>
 #include <sys/systm.h>
 #include <sys/thmap.h>
 #include <sys/time.h>
 #include <sys/timespec.h>
 #include <net/bpf.h>
 #include <net/if.h>
 #include <net/if_types.h>
 #include <net/if_wg.h>
 #include <net/route.h>
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
 #include <netinet/in_var.h>
 #include <netinet/ip.h>
 @@ -593,26 +594,29 @@ struct wg_peer {
 struct wg_ops;
 struct wg_softc {
 	struct ifnet	wg_if;
 	LIST_ENTRY(wg_softc) wg_list;
 	kmutex_t	*wg_lock;
 	krwlock_t	*wg_rwlock;
 	uint8_t		wg_privkey[WG_STATIC_KEY_LEN];
 	uint8_t		wg_pubkey[WG_STATIC_KEY_LEN];
 	int		wg_npeers;
 	struct pslist_head	wg_peers;
 	struct thmap	*wg_peers_bypubkey;
 	struct thmap	*wg_peers_byname;
 	struct thmap	*wg_sessions_byindex;
 	uint16_t	wg_listen_port;
 	struct wg_worker	*wg_worker;
 	lwp_t			*wg_worker_lwp;
 	struct radix_node_head	*wg_rtable_ipv4;
 	struct radix_node_head	*wg_rtable_ipv6;
 	struct wg_ppsratecheck	wg_ppsratecheck;
 	struct wg_ops		*wg_ops;
 #ifdef WG_RUMPKERNEL
 @@ -1149,26 +1153,72 @@ wg_lock_unstable_session(struct wg_peer
 	mutex_exit(wgp->wgp_lock);
 	return wgs;
+}
 #if 0
 static void
 wg_unlock_session(struct wg_peer *wgp, struct wg_session *wgs)
+{
 	mutex_exit(wgs->wgs_lock);
+}
 #endif
 static uint32_t
 wg_assign_sender_index(struct wg_softc *wg, struct wg_session *wgs)
+{
 	struct wg_peer *wgp = wgs->wgs_peer;
 	struct wg_session *wgs0;
 	uint32_t index;
 	void *garbage;
 	mutex_enter(wgs->wgs_lock);
 	/* Release the current index, if there is one.  */
 	while ((index = wgs->wgs_sender_index) != 0) {
 		/* Remove the session by index.  */
 		thmap_del(wg->wg_sessions_byindex, &index, sizeof index);
 		wgs->wgs_sender_index = 0;
 		mutex_exit(wgs->wgs_lock);
 		/* Wait for all thmap_gets to complete, and GC.  */
 		garbage = thmap_stage_gc(wg->wg_sessions_byindex);
 		mutex_enter(wgs->wgs_peer->wgp_lock);
 		pserialize_perform(wgp->wgp_psz);
 		mutex_exit(wgs->wgs_peer->wgp_lock);
 		thmap_gc(wg->wg_sessions_byindex, garbage);
 		mutex_enter(wgs->wgs_lock);
+	}
 restart:
 	/* Pick a uniform random nonzero index.  */
 	while (__predict_false((index = cprng_strong32()) == 0))
 		continue;
 	/* Try to take it.  */
 	wgs->wgs_sender_index = index;
 	wgs0 = thmap_put(wg->wg_sessions_byindex,
 	    &wgs->wgs_sender_index, sizeof wgs->wgs_sender_index, wgs);
 	/* If someone else beat us, start over.  */
 	if (__predict_false(wgs0 != wgs))
 		goto restart;
 	mutex_exit(wgs->wgs_lock);
 	return index;
+}
 /*
  * Handshake patterns
+ *
  * [W] 5: "These messages use the "IK" pattern from Noise"
  * [N] 7.5. Interactive handshake patterns (fundamental)
  *     "The first character refers to the initiator’s static key:"
  *     "I = Static key for initiator Immediately transmitted to responder,
  *          despite reduced or absent identity hiding"
  *     "The second character refers to the responder’s static key:"
  *     "K = Static key for responder Known to initiator"
  *     "IK:
  *        <- s
  *        ...
 @@ -1182,27 +1232,27 @@ wg_unlock_session(struct wg_peer *wgp, s
  *        <- e, ee, se, psk"
  */
 static void
 wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp,
     struct wg_session *wgs, struct wg_msg_init *wgmi)
+{
 	uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */
 	uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */
 	uint8_t cipher_key[WG_CIPHER_KEY_LEN];
 	uint8_t pubkey[WG_EPHEMERAL_KEY_LEN];
 	uint8_t privkey[WG_EPHEMERAL_KEY_LEN];
 	wgmi->wgmi_type = WG_MSG_TYPE_INIT;
-	wgmi->wgmi_sender = cprng_strong32();
+	wgmi->wgmi_sender = wg_assign_sender_index(wg, wgs);
 	/* [W] 5.4.2: First Message: Initiator to Responder */
 	/* Ci := HASH(CONSTRUCTION) */
 	/* Hi := HASH(Ci || IDENTIFIER) */
 	wg_init_key_and_hash(ckey, hash);
 	/* Hi := HASH(Hi || Sr^pub) */
 	wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey));
 	WG_DUMP_HASH("hash", hash);
 	/* [N] 2.2: "e" */
 	/* Ei^priv, Ei^pub := DH-GENERATE() */
 @@ -1257,27 +1307,26 @@ wg_fill_msg_init(struct wg_softc *wg, st
 		memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2));
 	else {
 		wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2),
 		    wgp->wgp_latest_cookie, WG_COOKIE_LEN,
 		    (const uint8_t *)wgmi,
 		    offsetof(struct wg_msg_init, wgmi_mac2),
 		    NULL, 0);
+	}
 	memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey));
 	memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey));
 	memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash));
 	memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey));
 	wgs->wgs_sender_index = wgmi->wgmi_sender;
 	WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_sender_index);
+}
 static void
 wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi,
     const struct sockaddr *src)
+{
 	uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */
 	uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */
 	uint8_t cipher_key[WG_CIPHER_KEY_LEN];
 	uint8_t peer_pubkey[WG_STATIC_KEY_LEN];
 	struct wg_peer *wgp;
 	struct wg_session *wgs;
 @@ -1599,27 +1648,27 @@ wg_fill_msg_resp(struct wg_softc *wg, st
 	uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */
 	uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */
 	uint8_t cipher_key[WG_KDF_OUTPUT_LEN];
 	uint8_t pubkey[WG_EPHEMERAL_KEY_LEN];
 	uint8_t privkey[WG_EPHEMERAL_KEY_LEN];
 	struct wg_session *wgs;
 	struct psref psref;
 	wgs = wg_get_unstable_session(wgp, &psref);
 	memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash));
 	memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey));
 	wgmr->wgmr_type = WG_MSG_TYPE_RESP;
-	wgmr->wgmr_sender = cprng_strong32();
+	wgmr->wgmr_sender = wg_assign_sender_index(wg, wgs);
 	wgmr->wgmr_receiver = wgmi->wgmi_sender;
 	/* [W] 5.4.3 Second Message: Responder to Initiator */
 	/* [N] 2.2: "e" */
 	/* Er^priv, Er^pub := DH-GENERATE() */
 	wg_algo_generate_keypair(pubkey, privkey);
 	/* Cr := KDF1(Cr, Er^pub) */
 	wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey));
 	/* msg.ephemeral := Er^pub */
 	memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral));
 	/* Hr := HASH(Hr || msg.ephemeral) */
 	wg_algo_hash(hash, pubkey, sizeof(pubkey));
 @@ -1670,27 +1719,26 @@ wg_fill_msg_resp(struct wg_softc *wg, st
 	else {
 		/* msg.mac2 := MAC(Lm, msg_b) */
 		wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2),
 		    wgp->wgp_latest_cookie, WG_COOKIE_LEN,
 		    (const uint8_t *)wgmr,
 		    offsetof(struct wg_msg_resp, wgmr_mac2),
 		    NULL, 0);
+	}
 	memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash));
 	memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey));
 	memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey));
 	memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey));
 	wgs->wgs_sender_index = wgmr->wgmr_sender;
 	wgs->wgs_receiver_index = wgmi->wgmi_sender;
 	WG_DLOG("sender=%x\n", wgs->wgs_sender_index);
 	WG_DLOG("receiver=%x\n", wgs->wgs_receiver_index);
 	wg_put_session(wgs, &psref);
+}
 static void
 wg_swap_sessions(struct wg_peer *wgp)
+{
 	KASSERT(mutex_owned(wgp->wgp_lock));
 	wgp->wgp_session_unstable = atomic_swap_ptr(&wgp->wgp_session_stable,
 @@ -1896,32 +1944,27 @@ wg_send_handshake_msg_resp(struct wg_sof
 	error = wg->wg_ops->send_hs_msg(wgp, m);
 	if (error == 0)
 		WG_TRACE("resp msg sent");
 	return error;
+}
 static struct wg_peer *
 wg_lookup_peer_by_pubkey(struct wg_softc *wg,
     const uint8_t pubkey[WG_STATIC_KEY_LEN], struct psref *psref)
+{
 	struct wg_peer *wgp;
 	int s = pserialize_read_enter();
-	/* XXX O(n) */
+	wgp = thmap_get(wg->wg_peers_bypubkey, pubkey, WG_STATIC_KEY_LEN);
 	WG_PEER_READER_FOREACH(wgp, wg) {
 		if (consttime_memequal(wgp->wgp_pubkey, pubkey,
 			sizeof(wgp->wgp_pubkey)))
 			break;
 	if (wgp != NULL)
 		wg_get_peer(wgp, psref);
 	pserialize_read_exit(s);
 	return wgp;
+}
 static void
 wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp,
     struct wg_msg_cookie *wgmc, const uint32_t sender,
     const uint8_t mac1[WG_MAC_LEN], const struct sockaddr *src)
+{
 	uint8_t cookie[WG_COOKIE_LEN];
 @@ -2077,44 +2120,30 @@ wg_clear_states(struct wg_session *wgs)
 #define wgs_clear(v)	explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v))
 	wgs_clear(handshake_hash);
 	wgs_clear(chaining_key);
 	wgs_clear(ephemeral_key_pub);
 	wgs_clear(ephemeral_key_priv);
 	wgs_clear(ephemeral_key_peer);
 #undef wgs_clear
+}
 static struct wg_session *
 wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index,
     struct psref *psref)
+{
 	struct wg_peer *wgp;
 	struct wg_session *wgs;
 	int s = pserialize_read_enter();
-	/* XXX O(n) */
+	wgs = thmap_get(wg->wg_sessions_byindex, &index, sizeof index);
 	WG_PEER_READER_FOREACH(wgp, wg) {
 		wgs = wgp->wgp_session_stable;
 		WG_DLOG("index=%x wgs_sender_index=%x\n",
 		    index, wgs->wgs_sender_index);
 		if (wgs->wgs_sender_index == index)
 			break;
 		wgs = wgp->wgp_session_unstable;
 		WG_DLOG("index=%x wgs_sender_index=%x\n",
 		    index, wgs->wgs_sender_index);
 		if (wgs->wgs_sender_index == index)
 			break;
 		wgs = NULL;
 	if (wgs != NULL)
 		psref_acquire(psref, &wgs->wgs_psref, wg_psref_class);
 	pserialize_read_exit(s);
 	return wgs;
+}
 static void
 wg_schedule_rekey_timer(struct wg_peer *wgp)
+{
 	int timeout = MIN(wg_rekey_after_time, INT_MAX/hz);
 	callout_schedule(&wgp->wgp_rekey_timer, timeout * hz);
 @@ -3252,137 +3281,178 @@ wg_alloc_peer(struct wg_softc *wg)
 	mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET);
 #endif
 	wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP);
 	mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_NONE);
 	return wgp;
+}
 static void
 wg_destroy_peer(struct wg_peer *wgp)
+{
 	struct wg_session *wgs;
 	struct wg_softc *wg = wgp->wgp_sc;
 	uint32_t index;
 	void *garbage;
 	/* Prevent new packets from this peer on any source address.  */
 	rw_enter(wg->wg_rwlock, RW_WRITER);
 	for (int i = 0; i < wgp->wgp_n_allowedips; i++) {
 		struct wg_allowedip *wga = &wgp->wgp_allowedips[i];
 		struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family);
 		struct radix_node *rn;
 		KASSERT(rnh != NULL);
 		rn = rnh->rnh_deladdr(&wga->wga_sa_addr,
 		    &wga->wga_sa_mask, rnh);
 		if (rn == NULL) {
 			char addrstr[128];
 			sockaddr_format(&wga->wga_sa_addr, addrstr,
 			    sizeof(addrstr));
 			WGLOG(LOG_WARNING, "Couldn't delete %s", addrstr);
+		}
+	}
 	rw_exit(wg->wg_rwlock);
 	/* Halt all packet processing and timeouts.  */
 	softint_disestablish(wgp->wgp_si);
 	callout_halt(&wgp->wgp_rekey_timer, NULL);
 	callout_halt(&wgp->wgp_handshake_timeout_timer, NULL);
 	callout_halt(&wgp->wgp_session_dtor_timer, NULL);
 	/* Remove the sessions by index.  */
 	if ((index = wgp->wgp_session_stable->wgs_sender_index) != 0) {
 		thmap_del(wg->wg_sessions_byindex, &index, sizeof index);
 		wgp->wgp_session_stable->wgs_sender_index = 0;
+	}
 	if ((index = wgp->wgp_session_unstable->wgs_sender_index) != 0) {
 		thmap_del(wg->wg_sessions_byindex, &index, sizeof index);
 		wgp->wgp_session_unstable->wgs_sender_index = 0;
+	}
 	/* Wait for all thmap_gets to complete, and GC.  */
 	garbage = thmap_stage_gc(wg->wg_sessions_byindex);
 	mutex_enter(wgp->wgp_lock);
 	pserialize_perform(wgp->wgp_psz);
 	mutex_exit(wgp->wgp_lock);
 	thmap_gc(wg->wg_sessions_byindex, garbage);
 	wgs = wgp->wgp_session_unstable;
 	psref_target_destroy(&wgs->wgs_psref, wg_psref_class);
 	mutex_obj_free(wgs->wgs_lock);
 	mutex_destroy(&wgs->wgs_recvwin->lock);
 	kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin));
 #ifndef __HAVE_ATOMIC64_LOADSTORE
 	mutex_destroy(&wgs->wgs_send_counter_lock);
 #endif
 	kmem_free(wgs, sizeof(*wgs));
 	wgs = wgp->wgp_session_stable;
 	psref_target_destroy(&wgs->wgs_psref, wg_psref_class);
 	mutex_obj_free(wgs->wgs_lock);
 	mutex_destroy(&wgs->wgs_recvwin->lock);
 	kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin));
 #ifndef __HAVE_ATOMIC64_LOADSTORE
 	mutex_destroy(&wgs->wgs_send_counter_lock);
 #endif
 	kmem_free(wgs, sizeof(*wgs));
 	psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class);
 	psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class);
 	kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint));
 	kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0));
 	pserialize_destroy(wgp->wgp_psz);
 	pcq_destroy(wgp->wgp_q);
 	mutex_obj_free(wgp->wgp_lock);
 	kmem_free(wgp, sizeof(*wgp));
+}
 static void
 wg_destroy_all_peers(struct wg_softc *wg)
+{
-	struct wg_peer *wgp;
+	struct wg_peer *wgp, *wgp0 __diagused;
 	void *garbage_byname, *garbage_bypubkey;
 restart:
 	garbage_byname = garbage_bypubkey = NULL;
 	mutex_enter(wg->wg_lock);
 	WG_PEER_WRITER_FOREACH(wgp, wg) {
 		if (wgp->wgp_name[0]) {
 			wgp0 = thmap_del(wg->wg_peers_byname, wgp->wgp_name,
 			    strlen(wgp->wgp_name));
 			KASSERT(wgp0 == wgp);
 			garbage_byname = thmap_stage_gc(wg->wg_peers_byname);
+		}
 		wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey,
 		    sizeof(wgp->wgp_pubkey));
 		KASSERT(wgp0 == wgp);
 		garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey);
 		WG_PEER_WRITER_REMOVE(wgp);
 		wg->wg_npeers--;
 		mutex_enter(wgp->wgp_lock);
 		wgp->wgp_state = WGP_STATE_DESTROYING;
 		pserialize_perform(wgp->wgp_psz);
 		mutex_exit(wgp->wgp_lock);
 		PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry);
 		break;
+	}
 	mutex_exit(wg->wg_lock);
 	if (wgp == NULL)
 		return;
 	psref_target_destroy(&wgp->wgp_psref, wg_psref_class);
 	wg_destroy_peer(wgp);
 	thmap_gc(wg->wg_peers_byname, garbage_byname);
 	thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey);
 	goto restart;
+}
 static int
 wg_destroy_peer_name(struct wg_softc *wg, const char *name)
+{
-	struct wg_peer *wgp;
+	struct wg_peer *wgp, *wgp0 __diagused;
 	void *garbage_byname, *garbage_bypubkey;
 	mutex_enter(wg->wg_lock);
-	WG_PEER_WRITER_FOREACH(wgp, wg) {
+	wgp = thmap_del(wg->wg_peers_byname, name, strlen(name));
 		if (strcmp(wgp->wgp_name, name) == 0)
 			break;
 	if (wgp != NULL) {
 		wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey,
 		    sizeof(wgp->wgp_pubkey));
 		KASSERT(wgp0 == wgp);
 		garbage_byname = thmap_stage_gc(wg->wg_peers_byname);
 		garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey);
 		WG_PEER_WRITER_REMOVE(wgp);
 		wg->wg_npeers--;
 		mutex_enter(wgp->wgp_lock);
 		wgp->wgp_state = WGP_STATE_DESTROYING;
 		pserialize_perform(wgp->wgp_psz);
 		mutex_exit(wgp->wgp_lock);
 		PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry);
+	}
 	mutex_exit(wg->wg_lock);
 	if (wgp == NULL)
 		return ENOENT;
 	psref_target_destroy(&wgp->wgp_psref, wg_psref_class);
 	wg_destroy_peer(wgp);
 	thmap_gc(wg->wg_peers_byname, garbage_byname);
 	thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey);
 	return 0;
+}
 static int
 wg_if_attach(struct wg_softc *wg)
+{
 	int error;
 	wg->wg_if.if_addrlen = 0;
 	wg->wg_if.if_mtu = WG_MTU;
 	wg->wg_if.if_flags = IFF_MULTICAST;
 	wg->wg_if.if_extflags = IFEF_NO_LINK_STATE_CHANGE;
 @@ -3422,39 +3492,45 @@ wg_clone_create(struct if_clone *ifc, in
 	if (error != 0) {
 		kmem_free(wg, sizeof(struct wg_softc));
 		return error;
+	}
 	rn_inithead((void **)&wg->wg_rtable_ipv4,
 	    offsetof(struct sockaddr_in, sin_addr) * NBBY);
 #ifdef INET6
 	rn_inithead((void **)&wg->wg_rtable_ipv6,
 	    offsetof(struct sockaddr_in6, sin6_addr) * NBBY);
 #endif
 	PSLIST_INIT(&wg->wg_peers);
 	wg->wg_peers_bypubkey = thmap_create(0, NULL, THMAP_NOCOPY);
 	wg->wg_peers_byname = thmap_create(0, NULL, THMAP_NOCOPY);
 	wg->wg_sessions_byindex = thmap_create(0, NULL, THMAP_NOCOPY);
 	wg->wg_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
 	wg->wg_rwlock = rw_obj_alloc();
 	wg->wg_ops = &wg_ops_rumpkernel;
 	error = wg_if_attach(wg);
 	if (error != 0) {
 		wg_worker_destroy(wg);
 		if (wg->wg_rtable_ipv4 != NULL)
 			free(wg->wg_rtable_ipv4, M_RTABLE);
 		if (wg->wg_rtable_ipv6 != NULL)
 			free(wg->wg_rtable_ipv6, M_RTABLE);
 		PSLIST_DESTROY(&wg->wg_peers);
 		mutex_obj_free(wg->wg_lock);
 		thmap_destroy(wg->wg_sessions_byindex);
 		thmap_destroy(wg->wg_peers_byname);
 		thmap_destroy(wg->wg_peers_bypubkey);
 		kmem_free(wg, sizeof(struct wg_softc));
 		return error;
+	}
 	mutex_enter(&wg_softcs.lock);
 	LIST_INSERT_HEAD(&wg_softcs.list, wg, wg_list);
 	mutex_exit(&wg_softcs.lock);
 	return 0;
+}
 static int
 wg_clone_destroy(struct ifnet *ifp)
 @@ -3472,26 +3548,29 @@ wg_clone_destroy(struct ifnet *ifp)
+	}
 #endif
 	bpf_detach(ifp);
 	if_detach(ifp);
 	wg_worker_destroy(wg);
 	wg_destroy_all_peers(wg);
 	if (wg->wg_rtable_ipv4 != NULL)
 		free(wg->wg_rtable_ipv4, M_RTABLE);
 	if (wg->wg_rtable_ipv6 != NULL)
 		free(wg->wg_rtable_ipv6, M_RTABLE);
 	PSLIST_DESTROY(&wg->wg_peers);
 	thmap_destroy(wg->wg_sessions_byindex);
 	thmap_destroy(wg->wg_peers_byname);
 	thmap_destroy(wg->wg_peers_bypubkey);
 	mutex_obj_free(wg->wg_lock);
 	rw_obj_free(wg->wg_rwlock);
 	kmem_free(wg, sizeof(struct wg_softc));
 	return 0;
+}
 static struct wg_peer *
 wg_pick_peer_by_sa(struct wg_softc *wg, const struct sockaddr *sa,
     struct psref *psref)
+{
 	struct radix_node_head *rnh;
 @@ -4066,41 +4145,59 @@ wg_ioctl_set_listen_port(struct wg_softc
 	error = wg->wg_ops->bind_port(wg, (uint16_t)port);
 out:
 	kmem_free(buf, ifd->ifd_len + 1);
 	return error;
+}
 static int
 wg_ioctl_add_peer(struct wg_softc *wg, struct ifdrv *ifd)
+{
 	int error;
 	prop_dictionary_t prop_dict;
 	char *buf = NULL;
-	struct wg_peer *wgp = NULL;
+	struct wg_peer *wgp = NULL, *wgp0 __diagused;
 	error = wg_alloc_prop_buf(&buf, ifd);
 	if (error != 0)
 		return error;
 	error = EINVAL;
 	prop_dict = prop_dictionary_internalize(buf);
 	if (prop_dict == NULL)
 		goto out;
 	error = wg_handle_prop_peer(wg, prop_dict, &wgp);
 	if (error != 0)
 		goto out;
 	mutex_enter(wg->wg_lock);
 	if (thmap_get(wg->wg_peers_bypubkey, wgp->wgp_pubkey,
 		sizeof(wgp->wgp_pubkey)) != NULL ||
 	    (wgp->wgp_name[0] &&
 		thmap_get(wg->wg_peers_byname, wgp->wgp_name,
 		    strlen(wgp->wgp_name)) != NULL)) {
 		mutex_exit(wg->wg_lock);
 		wg_destroy_peer(wgp);
 		error = EEXIST;
 		goto out;
+	}
 	wgp0 = thmap_put(wg->wg_peers_bypubkey, wgp->wgp_pubkey,
 	    sizeof(wgp->wgp_pubkey), wgp);
 	KASSERT(wgp0 == wgp);
 	if (wgp->wgp_name[0]) {
 		wgp0 = thmap_put(wg->wg_peers_byname, wgp->wgp_name,
 		    strlen(wgp->wgp_name), wgp);
 		KASSERT(wgp0 == wgp);
+	}
 	WG_PEER_WRITER_INSERT_HEAD(wgp, wg);
 	wg->wg_npeers++;
 	mutex_exit(wg->wg_lock);
 out:
 	kmem_free(buf, ifd->ifd_len + 1);
 	return error;
+}
 static int
 wg_ioctl_delete_peer(struct wg_softc *wg, struct ifdrv *ifd)
+{
 	int error;