Mon Aug 31 20:23:56 2020 UTC ()
wg: Use thmap(9) for peer and session lookup.

Make sure we also don't trip over our own shoelaces by choosing the
same session index twice.


(riastradh)
diff -r1.36 -r1.37 src/sys/net/if_wg.c

cvs diff -r1.36 -r1.37 src/sys/net/if_wg.c (switch to unified diff)

--- src/sys/net/if_wg.c 2020/08/31 20:21:30 1.36
+++ src/sys/net/if_wg.c 2020/08/31 20:23:56 1.37
@@ -1,4672 +1,4769 @@ @@ -1,4672 +1,4769 @@
1/* $NetBSD: if_wg.c,v 1.36 2020/08/31 20:21:30 riastradh Exp $ */ 1/* $NetBSD: if_wg.c,v 1.37 2020/08/31 20:23:56 riastradh Exp $ */
2 2
3/* 3/*
4 * Copyright (C) Ryota Ozaki <ozaki.ryota@gmail.com> 4 * Copyright (C) Ryota Ozaki <ozaki.ryota@gmail.com>
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * Redistribution and use in source and binary forms, with or without 7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions 8 * modification, are permitted provided that the following conditions
9 * are met: 9 * are met:
10 * 1. Redistributions of source code must retain the above copyright 10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer. 11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright 12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the 13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution. 14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the project nor the names of its contributors 15 * 3. Neither the name of the project nor the names of its contributors
16 * may be used to endorse or promote products derived from this software 16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission. 17 * without specific prior written permission.
18 * 18 *
19 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 19 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE. 29 * SUCH DAMAGE.
30 */ 30 */
31 31
32/* 32/*
33 * This network interface aims to implement the WireGuard protocol. 33 * This network interface aims to implement the WireGuard protocol.
34 * The implementation is based on the paper of WireGuard as of 34 * The implementation is based on the paper of WireGuard as of
35 * 2018-06-30 [1]. The paper is referred in the source code with label 35 * 2018-06-30 [1]. The paper is referred in the source code with label
36 * [W]. Also the specification of the Noise protocol framework as of 36 * [W]. Also the specification of the Noise protocol framework as of
37 * 2018-07-11 [2] is referred with label [N]. 37 * 2018-07-11 [2] is referred with label [N].
38 * 38 *
39 * [1] https://www.wireguard.com/papers/wireguard.pdf 39 * [1] https://www.wireguard.com/papers/wireguard.pdf
40 * [2] http://noiseprotocol.org/noise.pdf 40 * [2] http://noiseprotocol.org/noise.pdf
41 */ 41 */
42 42
43#include <sys/cdefs.h> 43#include <sys/cdefs.h>
44__KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.36 2020/08/31 20:21:30 riastradh Exp $"); 44__KERNEL_RCSID(0, "$NetBSD: if_wg.c,v 1.37 2020/08/31 20:23:56 riastradh Exp $");
45 45
46#ifdef _KERNEL_OPT 46#ifdef _KERNEL_OPT
47#include "opt_inet.h" 47#include "opt_inet.h"
48#endif 48#endif
49 49
50#include <sys/param.h> 50#include <sys/param.h>
51#include <sys/types.h> 51#include <sys/types.h>
52 52
53#include <sys/atomic.h> 53#include <sys/atomic.h>
54#include <sys/callout.h> 54#include <sys/callout.h>
55#include <sys/cprng.h> 55#include <sys/cprng.h>
56#include <sys/cpu.h> 56#include <sys/cpu.h>
57#include <sys/device.h> 57#include <sys/device.h>
58#include <sys/domain.h> 58#include <sys/domain.h>
59#include <sys/errno.h> 59#include <sys/errno.h>
60#include <sys/intr.h> 60#include <sys/intr.h>
61#include <sys/ioctl.h> 61#include <sys/ioctl.h>
62#include <sys/kernel.h> 62#include <sys/kernel.h>
63#include <sys/kmem.h> 63#include <sys/kmem.h>
64#include <sys/kthread.h> 64#include <sys/kthread.h>
65#include <sys/mbuf.h> 65#include <sys/mbuf.h>
66#include <sys/module.h> 66#include <sys/module.h>
67#include <sys/mutex.h> 67#include <sys/mutex.h>
68#include <sys/pcq.h> 68#include <sys/pcq.h>
69#include <sys/percpu.h> 69#include <sys/percpu.h>
70#include <sys/pserialize.h> 70#include <sys/pserialize.h>
71#include <sys/psref.h> 71#include <sys/psref.h>
72#include <sys/queue.h> 72#include <sys/queue.h>
73#include <sys/rwlock.h> 73#include <sys/rwlock.h>
74#include <sys/socket.h> 74#include <sys/socket.h>
75#include <sys/socketvar.h> 75#include <sys/socketvar.h>
76#include <sys/sockio.h> 76#include <sys/sockio.h>
77#include <sys/sysctl.h> 77#include <sys/sysctl.h>
78#include <sys/syslog.h> 78#include <sys/syslog.h>
79#include <sys/systm.h> 79#include <sys/systm.h>
 80#include <sys/thmap.h>
80#include <sys/time.h> 81#include <sys/time.h>
81#include <sys/timespec.h> 82#include <sys/timespec.h>
82 83
83#include <net/bpf.h> 84#include <net/bpf.h>
84#include <net/if.h> 85#include <net/if.h>
85#include <net/if_types.h> 86#include <net/if_types.h>
86#include <net/if_wg.h> 87#include <net/if_wg.h>
87#include <net/route.h> 88#include <net/route.h>
88 89
89#include <netinet/in.h> 90#include <netinet/in.h>
90#include <netinet/in_pcb.h> 91#include <netinet/in_pcb.h>
91#include <netinet/in_var.h> 92#include <netinet/in_var.h>
92#include <netinet/ip.h> 93#include <netinet/ip.h>
93#include <netinet/ip_var.h> 94#include <netinet/ip_var.h>
94#include <netinet/udp.h> 95#include <netinet/udp.h>
95#include <netinet/udp_var.h> 96#include <netinet/udp_var.h>
96 97
97#ifdef INET6 98#ifdef INET6
98#include <netinet/ip6.h> 99#include <netinet/ip6.h>
99#include <netinet6/in6_pcb.h> 100#include <netinet6/in6_pcb.h>
100#include <netinet6/in6_var.h> 101#include <netinet6/in6_var.h>
101#include <netinet6/ip6_var.h> 102#include <netinet6/ip6_var.h>
102#include <netinet6/udp6_var.h> 103#include <netinet6/udp6_var.h>
103#endif /* INET6 */ 104#endif /* INET6 */
104 105
105#include <prop/proplib.h> 106#include <prop/proplib.h>
106 107
107#include <crypto/blake2/blake2s.h> 108#include <crypto/blake2/blake2s.h>
108#include <crypto/sodium/crypto_aead_chacha20poly1305.h> 109#include <crypto/sodium/crypto_aead_chacha20poly1305.h>
109#include <crypto/sodium/crypto_aead_xchacha20poly1305.h> 110#include <crypto/sodium/crypto_aead_xchacha20poly1305.h>
110#include <crypto/sodium/crypto_scalarmult.h> 111#include <crypto/sodium/crypto_scalarmult.h>
111 112
112#include "ioconf.h" 113#include "ioconf.h"
113 114
114#ifdef WG_RUMPKERNEL 115#ifdef WG_RUMPKERNEL
115#include "wg_user.h" 116#include "wg_user.h"
116#endif 117#endif
117 118
118/* 119/*
119 * Data structures 120 * Data structures
120 * - struct wg_softc is an instance of wg interfaces 121 * - struct wg_softc is an instance of wg interfaces
121 * - It has a list of peers (struct wg_peer) 122 * - It has a list of peers (struct wg_peer)
122 * - It has a kthread that sends/receives handshake messages and 123 * - It has a kthread that sends/receives handshake messages and
123 * runs event handlers 124 * runs event handlers
124 * - It has its own two routing tables: one is for IPv4 and the other IPv6 125 * - It has its own two routing tables: one is for IPv4 and the other IPv6
125 * - struct wg_peer is a representative of a peer 126 * - struct wg_peer is a representative of a peer
126 * - It has a softint that is used to send packets over an wg interface 127 * - It has a softint that is used to send packets over an wg interface
127 * to a peer 128 * to a peer
128 * - It has a pair of session instances (struct wg_session) 129 * - It has a pair of session instances (struct wg_session)
129 * - It has a pair of endpoint instances (struct wg_sockaddr) 130 * - It has a pair of endpoint instances (struct wg_sockaddr)
130 * - Normally one endpoint is used and the second one is used only on 131 * - Normally one endpoint is used and the second one is used only on
131 * a peer migration (a change of peer's IP address) 132 * a peer migration (a change of peer's IP address)
132 * - It has a list of IP addresses and sub networks called allowedips 133 * - It has a list of IP addresses and sub networks called allowedips
133 * (struct wg_allowedip) 134 * (struct wg_allowedip)
134 * - A packets sent over a session is allowed if its destination matches 135 * - A packets sent over a session is allowed if its destination matches
135 * any IP addresses or sub networks of the list 136 * any IP addresses or sub networks of the list
136 * - struct wg_session represents a session of a secure tunnel with a peer 137 * - struct wg_session represents a session of a secure tunnel with a peer
137 * - Two instances of sessions belong to a peer; a stable session and a 138 * - Two instances of sessions belong to a peer; a stable session and a
138 * unstable session 139 * unstable session
139 * - A handshake process of a session always starts with a unstable instace 140 * - A handshake process of a session always starts with a unstable instace
140 * - Once a session is established, its instance becomes stable and the 141 * - Once a session is established, its instance becomes stable and the
141 * other becomes unstable instead 142 * other becomes unstable instead
142 * - Data messages are always sent via a stable session 143 * - Data messages are always sent via a stable session
143 * 144 *
144 * Locking notes: 145 * Locking notes:
145 * - wg interfaces (struct wg_softc, wg) is listed in wg_softcs.list and 146 * - wg interfaces (struct wg_softc, wg) is listed in wg_softcs.list and
146 * protected by wg_softcs.lock 147 * protected by wg_softcs.lock
147 * - Each wg has a mutex(9) and a rwlock(9) 148 * - Each wg has a mutex(9) and a rwlock(9)
148 * - The mutex (wg_lock) protects its peer list (wg_peers) 149 * - The mutex (wg_lock) protects its peer list (wg_peers)
149 * - A peer on the list is also protected by pserialize(9) or psref(9) 150 * - A peer on the list is also protected by pserialize(9) or psref(9)
150 * - The rwlock (wg_rwlock) protects the routing tables (wg_rtable_ipv[46]) 151 * - The rwlock (wg_rwlock) protects the routing tables (wg_rtable_ipv[46])
151 * - Each peer (struct wg_peer, wgp) has a mutex 152 * - Each peer (struct wg_peer, wgp) has a mutex
152 * - The mutex (wgp_lock) protects wgp_session_unstable and wgp_state 153 * - The mutex (wgp_lock) protects wgp_session_unstable and wgp_state
153 * - Each session (struct wg_session, wgs) has a mutex 154 * - Each session (struct wg_session, wgs) has a mutex
154 * - The mutex (wgs_lock) protects its state (wgs_state) and its handshake 155 * - The mutex (wgs_lock) protects its state (wgs_state) and its handshake
155 * states 156 * states
156 * - wgs_state of a unstable session can be changed while it never be 157 * - wgs_state of a unstable session can be changed while it never be
157 * changed on a stable session, so once get a session instace via 158 * changed on a stable session, so once get a session instace via
158 * wgp_session_stable we can safely access wgs_state without 159 * wgp_session_stable we can safely access wgs_state without
159 * holding wgs_lock 160 * holding wgs_lock
160 * - A session is protected by pserialize or psref like wgp 161 * - A session is protected by pserialize or psref like wgp
161 * - On a session swap, we must wait for all readers to release a 162 * - On a session swap, we must wait for all readers to release a
162 * reference to a stable session before changing wgs_state and 163 * reference to a stable session before changing wgs_state and
163 * session states 164 * session states
164 * 165 *
165 * Lock order: wg_lock -> wgp_lock -> wgs_lock 166 * Lock order: wg_lock -> wgp_lock -> wgs_lock
166 */ 167 */
167 168
168 169
169#define WGLOG(level, fmt, args...) \ 170#define WGLOG(level, fmt, args...) \
170 log(level, "%s: " fmt, __func__, ##args) 171 log(level, "%s: " fmt, __func__, ##args)
171 172
172/* Debug options */ 173/* Debug options */
173#ifdef WG_DEBUG 174#ifdef WG_DEBUG
174/* Output debug logs */ 175/* Output debug logs */
175#ifndef WG_DEBUG_LOG 176#ifndef WG_DEBUG_LOG
176#define WG_DEBUG_LOG 177#define WG_DEBUG_LOG
177#endif 178#endif
178/* Output trace logs */ 179/* Output trace logs */
179#ifndef WG_DEBUG_TRACE 180#ifndef WG_DEBUG_TRACE
180#define WG_DEBUG_TRACE 181#define WG_DEBUG_TRACE
181#endif 182#endif
182/* Output hash values, etc. */ 183/* Output hash values, etc. */
183#ifndef WG_DEBUG_DUMP 184#ifndef WG_DEBUG_DUMP
184#define WG_DEBUG_DUMP 185#define WG_DEBUG_DUMP
185#endif 186#endif
186/* Make some internal parameters configurable for testing and debugging */ 187/* Make some internal parameters configurable for testing and debugging */
187#ifndef WG_DEBUG_PARAMS 188#ifndef WG_DEBUG_PARAMS
188#define WG_DEBUG_PARAMS 189#define WG_DEBUG_PARAMS
189#endif 190#endif
190#endif 191#endif
191 192
192#ifdef WG_DEBUG_TRACE 193#ifdef WG_DEBUG_TRACE
193#define WG_TRACE(msg) \ 194#define WG_TRACE(msg) \
194 log(LOG_DEBUG, "%s:%d: %s\n", __func__, __LINE__, (msg)) 195 log(LOG_DEBUG, "%s:%d: %s\n", __func__, __LINE__, (msg))
195#else 196#else
196#define WG_TRACE(msg) __nothing 197#define WG_TRACE(msg) __nothing
197#endif 198#endif
198 199
199#ifdef WG_DEBUG_LOG 200#ifdef WG_DEBUG_LOG
200#define WG_DLOG(fmt, args...) log(LOG_DEBUG, "%s: " fmt, __func__, ##args) 201#define WG_DLOG(fmt, args...) log(LOG_DEBUG, "%s: " fmt, __func__, ##args)
201#else 202#else
202#define WG_DLOG(fmt, args...) __nothing 203#define WG_DLOG(fmt, args...) __nothing
203#endif 204#endif
204 205
205#define WG_LOG_RATECHECK(wgprc, level, fmt, args...) do { \ 206#define WG_LOG_RATECHECK(wgprc, level, fmt, args...) do { \
206 if (ppsratecheck(&(wgprc)->wgprc_lasttime, \ 207 if (ppsratecheck(&(wgprc)->wgprc_lasttime, \
207 &(wgprc)->wgprc_curpps, 1)) { \ 208 &(wgprc)->wgprc_curpps, 1)) { \
208 log(level, fmt, ##args); \ 209 log(level, fmt, ##args); \
209 } \ 210 } \
210} while (0) 211} while (0)
211 212
212#ifdef WG_DEBUG_PARAMS 213#ifdef WG_DEBUG_PARAMS
213static bool wg_force_underload = false; 214static bool wg_force_underload = false;
214#endif 215#endif
215 216
216#ifdef WG_DEBUG_DUMP 217#ifdef WG_DEBUG_DUMP
217 218
218#ifdef WG_RUMPKERNEL 219#ifdef WG_RUMPKERNEL
219static void 220static void
220wg_dump_buf(const char *func, const char *buf, const size_t size) 221wg_dump_buf(const char *func, const char *buf, const size_t size)
221{ 222{
222 223
223 log(LOG_DEBUG, "%s: ", func); 224 log(LOG_DEBUG, "%s: ", func);
224 for (int i = 0; i < size; i++) 225 for (int i = 0; i < size; i++)
225 log(LOG_DEBUG, "%02x ", (int)(0xff & buf[i])); 226 log(LOG_DEBUG, "%02x ", (int)(0xff & buf[i]));
226 log(LOG_DEBUG, "\n"); 227 log(LOG_DEBUG, "\n");
227} 228}
228#endif 229#endif
229 230
230static void 231static void
231wg_dump_hash(const uint8_t *func, const uint8_t *name, const uint8_t *hash, 232wg_dump_hash(const uint8_t *func, const uint8_t *name, const uint8_t *hash,
232 const size_t size) 233 const size_t size)
233{ 234{
234 235
235 log(LOG_DEBUG, "%s: %s: ", func, name); 236 log(LOG_DEBUG, "%s: %s: ", func, name);
236 for (int i = 0; i < size; i++) 237 for (int i = 0; i < size; i++)
237 log(LOG_DEBUG, "%02x ", (int)(0xff & hash[i])); 238 log(LOG_DEBUG, "%02x ", (int)(0xff & hash[i]));
238 log(LOG_DEBUG, "\n"); 239 log(LOG_DEBUG, "\n");
239} 240}
240 241
241#define WG_DUMP_HASH(name, hash) \ 242#define WG_DUMP_HASH(name, hash) \
242 wg_dump_hash(__func__, name, hash, WG_HASH_LEN) 243 wg_dump_hash(__func__, name, hash, WG_HASH_LEN)
243#define WG_DUMP_HASH48(name, hash) \ 244#define WG_DUMP_HASH48(name, hash) \
244 wg_dump_hash(__func__, name, hash, 48) 245 wg_dump_hash(__func__, name, hash, 48)
245#define WG_DUMP_BUF(buf, size) \ 246#define WG_DUMP_BUF(buf, size) \
246 wg_dump_buf(__func__, buf, size) 247 wg_dump_buf(__func__, buf, size)
247#else 248#else
248#define WG_DUMP_HASH(name, hash) __nothing 249#define WG_DUMP_HASH(name, hash) __nothing
249#define WG_DUMP_HASH48(name, hash) __nothing 250#define WG_DUMP_HASH48(name, hash) __nothing
250#define WG_DUMP_BUF(buf, size) __nothing 251#define WG_DUMP_BUF(buf, size) __nothing
251#endif /* WG_DEBUG_DUMP */ 252#endif /* WG_DEBUG_DUMP */
252 253
253#define WG_MTU 1420 254#define WG_MTU 1420
254#define WG_ALLOWEDIPS 16 255#define WG_ALLOWEDIPS 16
255 256
256#define CURVE25519_KEY_LEN 32 257#define CURVE25519_KEY_LEN 32
257#define TAI64N_LEN sizeof(uint32_t) * 3 258#define TAI64N_LEN sizeof(uint32_t) * 3
258#define POLY1305_AUTHTAG_LEN 16 259#define POLY1305_AUTHTAG_LEN 16
259#define HMAC_BLOCK_LEN 64 260#define HMAC_BLOCK_LEN 64
260 261
261/* [N] 4.1: "DHLEN must be 32 or greater." WireGuard chooses 32. */ 262/* [N] 4.1: "DHLEN must be 32 or greater." WireGuard chooses 32. */
262/* [N] 4.3: Hash functions */ 263/* [N] 4.3: Hash functions */
263#define NOISE_DHLEN 32 264#define NOISE_DHLEN 32
264/* [N] 4.3: "Must be 32 or 64." WireGuard chooses 32. */ 265/* [N] 4.3: "Must be 32 or 64." WireGuard chooses 32. */
265#define NOISE_HASHLEN 32 266#define NOISE_HASHLEN 32
266#define NOISE_BLOCKLEN 64 267#define NOISE_BLOCKLEN 64
267#define NOISE_HKDF_OUTPUT_LEN NOISE_HASHLEN 268#define NOISE_HKDF_OUTPUT_LEN NOISE_HASHLEN
268/* [N] 5.1: "k" */ 269/* [N] 5.1: "k" */
269#define NOISE_CIPHER_KEY_LEN 32 270#define NOISE_CIPHER_KEY_LEN 32
270/* 271/*
271 * [N] 9.2: "psk" 272 * [N] 9.2: "psk"
272 * "... psk is a 32-byte secret value provided by the application." 273 * "... psk is a 32-byte secret value provided by the application."
273 */ 274 */
274#define NOISE_PRESHARED_KEY_LEN 32 275#define NOISE_PRESHARED_KEY_LEN 32
275 276
276#define WG_STATIC_KEY_LEN CURVE25519_KEY_LEN 277#define WG_STATIC_KEY_LEN CURVE25519_KEY_LEN
277#define WG_TIMESTAMP_LEN TAI64N_LEN 278#define WG_TIMESTAMP_LEN TAI64N_LEN
278 279
279#define WG_PRESHARED_KEY_LEN NOISE_PRESHARED_KEY_LEN 280#define WG_PRESHARED_KEY_LEN NOISE_PRESHARED_KEY_LEN
280 281
281#define WG_COOKIE_LEN 16 282#define WG_COOKIE_LEN 16
282#define WG_MAC_LEN 16 283#define WG_MAC_LEN 16
283#define WG_RANDVAL_LEN 24 284#define WG_RANDVAL_LEN 24
284 285
285#define WG_EPHEMERAL_KEY_LEN CURVE25519_KEY_LEN 286#define WG_EPHEMERAL_KEY_LEN CURVE25519_KEY_LEN
286/* [N] 5.2: "ck: A chaining key of HASHLEN bytes" */ 287/* [N] 5.2: "ck: A chaining key of HASHLEN bytes" */
287#define WG_CHAINING_KEY_LEN NOISE_HASHLEN 288#define WG_CHAINING_KEY_LEN NOISE_HASHLEN
288/* [N] 5.2: "h: A hash output of HASHLEN bytes" */ 289/* [N] 5.2: "h: A hash output of HASHLEN bytes" */
289#define WG_HASH_LEN NOISE_HASHLEN 290#define WG_HASH_LEN NOISE_HASHLEN
290#define WG_CIPHER_KEY_LEN NOISE_CIPHER_KEY_LEN 291#define WG_CIPHER_KEY_LEN NOISE_CIPHER_KEY_LEN
291#define WG_DH_OUTPUT_LEN NOISE_DHLEN 292#define WG_DH_OUTPUT_LEN NOISE_DHLEN
292#define WG_KDF_OUTPUT_LEN NOISE_HKDF_OUTPUT_LEN 293#define WG_KDF_OUTPUT_LEN NOISE_HKDF_OUTPUT_LEN
293#define WG_AUTHTAG_LEN POLY1305_AUTHTAG_LEN 294#define WG_AUTHTAG_LEN POLY1305_AUTHTAG_LEN
294#define WG_DATA_KEY_LEN 32 295#define WG_DATA_KEY_LEN 32
295#define WG_SALT_LEN 24 296#define WG_SALT_LEN 24
296 297
297/* 298/*
298 * The protocol messages 299 * The protocol messages
299 */ 300 */
300struct wg_msg { 301struct wg_msg {
301 uint32_t wgm_type; 302 uint32_t wgm_type;
302} __packed; 303} __packed;
303 304
304/* [W] 5.4.2 First Message: Initiator to Responder */ 305/* [W] 5.4.2 First Message: Initiator to Responder */
305struct wg_msg_init { 306struct wg_msg_init {
306 uint32_t wgmi_type; 307 uint32_t wgmi_type;
307 uint32_t wgmi_sender; 308 uint32_t wgmi_sender;
308 uint8_t wgmi_ephemeral[WG_EPHEMERAL_KEY_LEN]; 309 uint8_t wgmi_ephemeral[WG_EPHEMERAL_KEY_LEN];
309 uint8_t wgmi_static[WG_STATIC_KEY_LEN + WG_AUTHTAG_LEN]; 310 uint8_t wgmi_static[WG_STATIC_KEY_LEN + WG_AUTHTAG_LEN];
310 uint8_t wgmi_timestamp[WG_TIMESTAMP_LEN + WG_AUTHTAG_LEN]; 311 uint8_t wgmi_timestamp[WG_TIMESTAMP_LEN + WG_AUTHTAG_LEN];
311 uint8_t wgmi_mac1[WG_MAC_LEN]; 312 uint8_t wgmi_mac1[WG_MAC_LEN];
312 uint8_t wgmi_mac2[WG_MAC_LEN]; 313 uint8_t wgmi_mac2[WG_MAC_LEN];
313} __packed; 314} __packed;
314 315
315/* [W] 5.4.3 Second Message: Responder to Initiator */ 316/* [W] 5.4.3 Second Message: Responder to Initiator */
316struct wg_msg_resp { 317struct wg_msg_resp {
317 uint32_t wgmr_type; 318 uint32_t wgmr_type;
318 uint32_t wgmr_sender; 319 uint32_t wgmr_sender;
319 uint32_t wgmr_receiver; 320 uint32_t wgmr_receiver;
320 uint8_t wgmr_ephemeral[WG_EPHEMERAL_KEY_LEN]; 321 uint8_t wgmr_ephemeral[WG_EPHEMERAL_KEY_LEN];
321 uint8_t wgmr_empty[0 + WG_AUTHTAG_LEN]; 322 uint8_t wgmr_empty[0 + WG_AUTHTAG_LEN];
322 uint8_t wgmr_mac1[WG_MAC_LEN]; 323 uint8_t wgmr_mac1[WG_MAC_LEN];
323 uint8_t wgmr_mac2[WG_MAC_LEN]; 324 uint8_t wgmr_mac2[WG_MAC_LEN];
324} __packed; 325} __packed;
325 326
326/* [W] 5.4.6 Subsequent Messages: Transport Data Messages */ 327/* [W] 5.4.6 Subsequent Messages: Transport Data Messages */
327struct wg_msg_data { 328struct wg_msg_data {
328 uint32_t wgmd_type; 329 uint32_t wgmd_type;
329 uint32_t wgmd_receiver; 330 uint32_t wgmd_receiver;
330 uint64_t wgmd_counter; 331 uint64_t wgmd_counter;
331 uint32_t wgmd_packet[0]; 332 uint32_t wgmd_packet[0];
332} __packed; 333} __packed;
333 334
334/* [W] 5.4.7 Under Load: Cookie Reply Message */ 335/* [W] 5.4.7 Under Load: Cookie Reply Message */
335struct wg_msg_cookie { 336struct wg_msg_cookie {
336 uint32_t wgmc_type; 337 uint32_t wgmc_type;
337 uint32_t wgmc_receiver; 338 uint32_t wgmc_receiver;
338 uint8_t wgmc_salt[WG_SALT_LEN]; 339 uint8_t wgmc_salt[WG_SALT_LEN];
339 uint8_t wgmc_cookie[WG_COOKIE_LEN + WG_AUTHTAG_LEN]; 340 uint8_t wgmc_cookie[WG_COOKIE_LEN + WG_AUTHTAG_LEN];
340} __packed; 341} __packed;
341 342
342#define WG_MSG_TYPE_INIT 1 343#define WG_MSG_TYPE_INIT 1
343#define WG_MSG_TYPE_RESP 2 344#define WG_MSG_TYPE_RESP 2
344#define WG_MSG_TYPE_COOKIE 3 345#define WG_MSG_TYPE_COOKIE 3
345#define WG_MSG_TYPE_DATA 4 346#define WG_MSG_TYPE_DATA 4
346#define WG_MSG_TYPE_MAX WG_MSG_TYPE_DATA 347#define WG_MSG_TYPE_MAX WG_MSG_TYPE_DATA
347 348
348/* Sliding windows */ 349/* Sliding windows */
349 350
350#define SLIWIN_BITS 2048u 351#define SLIWIN_BITS 2048u
351#define SLIWIN_TYPE uint32_t 352#define SLIWIN_TYPE uint32_t
352#define SLIWIN_BPW NBBY*sizeof(SLIWIN_TYPE) 353#define SLIWIN_BPW NBBY*sizeof(SLIWIN_TYPE)
353#define SLIWIN_WORDS howmany(SLIWIN_BITS, SLIWIN_BPW) 354#define SLIWIN_WORDS howmany(SLIWIN_BITS, SLIWIN_BPW)
354#define SLIWIN_NPKT (SLIWIN_BITS - NBBY*sizeof(SLIWIN_TYPE)) 355#define SLIWIN_NPKT (SLIWIN_BITS - NBBY*sizeof(SLIWIN_TYPE))
355 356
356struct sliwin { 357struct sliwin {
357 SLIWIN_TYPE B[SLIWIN_WORDS]; 358 SLIWIN_TYPE B[SLIWIN_WORDS];
358 uint64_t T; 359 uint64_t T;
359}; 360};
360 361
361static void 362static void
362sliwin_reset(struct sliwin *W) 363sliwin_reset(struct sliwin *W)
363{ 364{
364 365
365 memset(W, 0, sizeof(*W)); 366 memset(W, 0, sizeof(*W));
366} 367}
367 368
368static int 369static int
369sliwin_check_fast(const volatile struct sliwin *W, uint64_t S) 370sliwin_check_fast(const volatile struct sliwin *W, uint64_t S)
370{ 371{
371 372
372 /* 373 /*
373 * If it's more than one window older than the highest sequence 374 * If it's more than one window older than the highest sequence
374 * number we've seen, reject. 375 * number we've seen, reject.
375 */ 376 */
376#ifdef __HAVE_ATOMIC64_LOADSTORE 377#ifdef __HAVE_ATOMIC64_LOADSTORE
377 if (S + SLIWIN_NPKT < atomic_load_relaxed(&W->T)) 378 if (S + SLIWIN_NPKT < atomic_load_relaxed(&W->T))
378 return EAUTH; 379 return EAUTH;
379#endif 380#endif
380 381
381 /* 382 /*
382 * Otherwise, we need to take the lock to decide, so don't 383 * Otherwise, we need to take the lock to decide, so don't
383 * reject just yet. Caller must serialize a call to 384 * reject just yet. Caller must serialize a call to
384 * sliwin_update in this case. 385 * sliwin_update in this case.
385 */ 386 */
386 return 0; 387 return 0;
387} 388}
388 389
389static int 390static int
390sliwin_update(struct sliwin *W, uint64_t S) 391sliwin_update(struct sliwin *W, uint64_t S)
391{ 392{
392 unsigned word, bit; 393 unsigned word, bit;
393 394
394 /* 395 /*
395 * If it's more than one window older than the highest sequence 396 * If it's more than one window older than the highest sequence
396 * number we've seen, reject. 397 * number we've seen, reject.
397 */ 398 */
398 if (S + SLIWIN_NPKT < W->T) 399 if (S + SLIWIN_NPKT < W->T)
399 return EAUTH; 400 return EAUTH;
400 401
401 /* 402 /*
402 * If it's higher than the highest sequence number we've seen, 403 * If it's higher than the highest sequence number we've seen,
403 * advance the window. 404 * advance the window.
404 */ 405 */
405 if (S > W->T) { 406 if (S > W->T) {
406 uint64_t i = W->T / SLIWIN_BPW; 407 uint64_t i = W->T / SLIWIN_BPW;
407 uint64_t j = S / SLIWIN_BPW; 408 uint64_t j = S / SLIWIN_BPW;
408 unsigned k; 409 unsigned k;
409 410
410 for (k = 0; k < MIN(j - i, SLIWIN_WORDS); k++) 411 for (k = 0; k < MIN(j - i, SLIWIN_WORDS); k++)
411 W->B[(i + k + 1) % SLIWIN_WORDS] = 0; 412 W->B[(i + k + 1) % SLIWIN_WORDS] = 0;
412#ifdef __HAVE_ATOMIC64_LOADSTORE 413#ifdef __HAVE_ATOMIC64_LOADSTORE
413 atomic_store_relaxed(&W->T, S); 414 atomic_store_relaxed(&W->T, S);
414#else 415#else
415 W->T = S; 416 W->T = S;
416#endif 417#endif
417 } 418 }
418 419
419 /* Test and set the bit -- if already set, reject. */ 420 /* Test and set the bit -- if already set, reject. */
420 word = (S / SLIWIN_BPW) % SLIWIN_WORDS; 421 word = (S / SLIWIN_BPW) % SLIWIN_WORDS;
421 bit = S % SLIWIN_BPW; 422 bit = S % SLIWIN_BPW;
422 if (W->B[word] & (1UL << bit)) 423 if (W->B[word] & (1UL << bit))
423 return EAUTH; 424 return EAUTH;
424 W->B[word] |= 1UL << bit; 425 W->B[word] |= 1UL << bit;
425 426
426 /* Accept! */ 427 /* Accept! */
427 return 0; 428 return 0;
428} 429}
429 430
430struct wg_worker { 431struct wg_worker {
431 kmutex_t wgw_lock; 432 kmutex_t wgw_lock;
432 kcondvar_t wgw_cv; 433 kcondvar_t wgw_cv;
433 bool wgw_todie; 434 bool wgw_todie;
434 struct socket *wgw_so4; 435 struct socket *wgw_so4;
435 struct socket *wgw_so6; 436 struct socket *wgw_so6;
436 int wgw_wakeup_reasons; 437 int wgw_wakeup_reasons;
437#define WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4 __BIT(0) 438#define WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4 __BIT(0)
438#define WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6 __BIT(1) 439#define WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6 __BIT(1)
439#define WG_WAKEUP_REASON_PEER __BIT(2) 440#define WG_WAKEUP_REASON_PEER __BIT(2)
440}; 441};
441 442
442struct wg_session { 443struct wg_session {
443 struct wg_peer *wgs_peer; 444 struct wg_peer *wgs_peer;
444 struct psref_target 445 struct psref_target
445 wgs_psref; 446 wgs_psref;
446 kmutex_t *wgs_lock; 447 kmutex_t *wgs_lock;
447 448
448 int wgs_state; 449 int wgs_state;
449#define WGS_STATE_UNKNOWN 0 450#define WGS_STATE_UNKNOWN 0
450#define WGS_STATE_INIT_ACTIVE 1 451#define WGS_STATE_INIT_ACTIVE 1
451#define WGS_STATE_INIT_PASSIVE 2 452#define WGS_STATE_INIT_PASSIVE 2
452#define WGS_STATE_ESTABLISHED 3 453#define WGS_STATE_ESTABLISHED 3
453#define WGS_STATE_DESTROYING 4 454#define WGS_STATE_DESTROYING 4
454 455
455 time_t wgs_time_established; 456 time_t wgs_time_established;
456 time_t wgs_time_last_data_sent; 457 time_t wgs_time_last_data_sent;
457 bool wgs_is_initiator; 458 bool wgs_is_initiator;
458 459
459 uint32_t wgs_sender_index; 460 uint32_t wgs_sender_index;
460 uint32_t wgs_receiver_index; 461 uint32_t wgs_receiver_index;
461#ifdef __HAVE_ATOMIC64_LOADSTORE 462#ifdef __HAVE_ATOMIC64_LOADSTORE
462 volatile uint64_t 463 volatile uint64_t
463 wgs_send_counter; 464 wgs_send_counter;
464#else 465#else
465 kmutex_t wgs_send_counter_lock; 466 kmutex_t wgs_send_counter_lock;
466 uint64_t wgs_send_counter; 467 uint64_t wgs_send_counter;
467#endif 468#endif
468 469
469 struct { 470 struct {
470 kmutex_t lock; 471 kmutex_t lock;
471 struct sliwin window; 472 struct sliwin window;
472 } *wgs_recvwin; 473 } *wgs_recvwin;
473 474
474 uint8_t wgs_handshake_hash[WG_HASH_LEN]; 475 uint8_t wgs_handshake_hash[WG_HASH_LEN];
475 uint8_t wgs_chaining_key[WG_CHAINING_KEY_LEN]; 476 uint8_t wgs_chaining_key[WG_CHAINING_KEY_LEN];
476 uint8_t wgs_ephemeral_key_pub[WG_EPHEMERAL_KEY_LEN]; 477 uint8_t wgs_ephemeral_key_pub[WG_EPHEMERAL_KEY_LEN];
477 uint8_t wgs_ephemeral_key_priv[WG_EPHEMERAL_KEY_LEN]; 478 uint8_t wgs_ephemeral_key_priv[WG_EPHEMERAL_KEY_LEN];
478 uint8_t wgs_ephemeral_key_peer[WG_EPHEMERAL_KEY_LEN]; 479 uint8_t wgs_ephemeral_key_peer[WG_EPHEMERAL_KEY_LEN];
479 uint8_t wgs_tkey_send[WG_DATA_KEY_LEN]; 480 uint8_t wgs_tkey_send[WG_DATA_KEY_LEN];
480 uint8_t wgs_tkey_recv[WG_DATA_KEY_LEN]; 481 uint8_t wgs_tkey_recv[WG_DATA_KEY_LEN];
481}; 482};
482 483
483struct wg_sockaddr { 484struct wg_sockaddr {
484 union { 485 union {
485 struct sockaddr_storage _ss; 486 struct sockaddr_storage _ss;
486 struct sockaddr _sa; 487 struct sockaddr _sa;
487 struct sockaddr_in _sin; 488 struct sockaddr_in _sin;
488 struct sockaddr_in6 _sin6; 489 struct sockaddr_in6 _sin6;
489 }; 490 };
490 struct psref_target wgsa_psref; 491 struct psref_target wgsa_psref;
491}; 492};
492 493
493#define wgsatosa(wgsa) (&(wgsa)->_sa) 494#define wgsatosa(wgsa) (&(wgsa)->_sa)
494#define wgsatosin(wgsa) (&(wgsa)->_sin) 495#define wgsatosin(wgsa) (&(wgsa)->_sin)
495#define wgsatosin6(wgsa) (&(wgsa)->_sin6) 496#define wgsatosin6(wgsa) (&(wgsa)->_sin6)
496 497
497struct wg_peer; 498struct wg_peer;
498struct wg_allowedip { 499struct wg_allowedip {
499 struct radix_node wga_nodes[2]; 500 struct radix_node wga_nodes[2];
500 struct wg_sockaddr _wga_sa_addr; 501 struct wg_sockaddr _wga_sa_addr;
501 struct wg_sockaddr _wga_sa_mask; 502 struct wg_sockaddr _wga_sa_mask;
502#define wga_sa_addr _wga_sa_addr._sa 503#define wga_sa_addr _wga_sa_addr._sa
503#define wga_sa_mask _wga_sa_mask._sa 504#define wga_sa_mask _wga_sa_mask._sa
504 505
505 int wga_family; 506 int wga_family;
506 uint8_t wga_cidr; 507 uint8_t wga_cidr;
507 union { 508 union {
508 struct in_addr _ip4; 509 struct in_addr _ip4;
509 struct in6_addr _ip6; 510 struct in6_addr _ip6;
510 } wga_addr; 511 } wga_addr;
511#define wga_addr4 wga_addr._ip4 512#define wga_addr4 wga_addr._ip4
512#define wga_addr6 wga_addr._ip6 513#define wga_addr6 wga_addr._ip6
513 514
514 struct wg_peer *wga_peer; 515 struct wg_peer *wga_peer;
515}; 516};
516 517
517typedef uint8_t wg_timestamp_t[WG_TIMESTAMP_LEN]; 518typedef uint8_t wg_timestamp_t[WG_TIMESTAMP_LEN];
518 519
519struct wg_ppsratecheck { 520struct wg_ppsratecheck {
520 struct timeval wgprc_lasttime; 521 struct timeval wgprc_lasttime;
521 int wgprc_curpps; 522 int wgprc_curpps;
522}; 523};
523 524
524struct wg_softc; 525struct wg_softc;
525struct wg_peer { 526struct wg_peer {
526 struct wg_softc *wgp_sc; 527 struct wg_softc *wgp_sc;
527 char wgp_name[WG_PEER_NAME_MAXLEN + 1]; 528 char wgp_name[WG_PEER_NAME_MAXLEN + 1];
528 struct pslist_entry wgp_peerlist_entry; 529 struct pslist_entry wgp_peerlist_entry;
529 pserialize_t wgp_psz; 530 pserialize_t wgp_psz;
530 struct psref_target wgp_psref; 531 struct psref_target wgp_psref;
531 kmutex_t *wgp_lock; 532 kmutex_t *wgp_lock;
532 533
533 uint8_t wgp_pubkey[WG_STATIC_KEY_LEN]; 534 uint8_t wgp_pubkey[WG_STATIC_KEY_LEN];
534 struct wg_sockaddr *wgp_endpoint; 535 struct wg_sockaddr *wgp_endpoint;
535#define wgp_ss wgp_endpoint->_ss 536#define wgp_ss wgp_endpoint->_ss
536#define wgp_sa wgp_endpoint->_sa 537#define wgp_sa wgp_endpoint->_sa
537#define wgp_sin wgp_endpoint->_sin 538#define wgp_sin wgp_endpoint->_sin
538#define wgp_sin6 wgp_endpoint->_sin6 539#define wgp_sin6 wgp_endpoint->_sin6
539 struct wg_sockaddr *wgp_endpoint0; 540 struct wg_sockaddr *wgp_endpoint0;
540 bool wgp_endpoint_changing; 541 bool wgp_endpoint_changing;
541 bool wgp_endpoint_available; 542 bool wgp_endpoint_available;
542 543
543 /* The preshared key (optional) */ 544 /* The preshared key (optional) */
544 uint8_t wgp_psk[WG_PRESHARED_KEY_LEN]; 545 uint8_t wgp_psk[WG_PRESHARED_KEY_LEN];
545 546
546 int wgp_state; 547 int wgp_state;
547#define WGP_STATE_INIT 0 548#define WGP_STATE_INIT 0
548#define WGP_STATE_ESTABLISHED 1 549#define WGP_STATE_ESTABLISHED 1
549#define WGP_STATE_GIVEUP 2 550#define WGP_STATE_GIVEUP 2
550#define WGP_STATE_DESTROYING 3 551#define WGP_STATE_DESTROYING 3
551 552
552 void *wgp_si; 553 void *wgp_si;
553 pcq_t *wgp_q; 554 pcq_t *wgp_q;
554 555
555 struct wg_session *wgp_session_stable; 556 struct wg_session *wgp_session_stable;
556 struct wg_session *wgp_session_unstable; 557 struct wg_session *wgp_session_unstable;
557 558
558 /* timestamp in big-endian */ 559 /* timestamp in big-endian */
559 wg_timestamp_t wgp_timestamp_latest_init; 560 wg_timestamp_t wgp_timestamp_latest_init;
560 561
561 struct timespec wgp_last_handshake_time; 562 struct timespec wgp_last_handshake_time;
562 563
563 callout_t wgp_rekey_timer; 564 callout_t wgp_rekey_timer;
564 callout_t wgp_handshake_timeout_timer; 565 callout_t wgp_handshake_timeout_timer;
565 callout_t wgp_session_dtor_timer; 566 callout_t wgp_session_dtor_timer;
566 567
567 time_t wgp_handshake_start_time; 568 time_t wgp_handshake_start_time;
568 569
569 int wgp_n_allowedips; 570 int wgp_n_allowedips;
570 struct wg_allowedip wgp_allowedips[WG_ALLOWEDIPS]; 571 struct wg_allowedip wgp_allowedips[WG_ALLOWEDIPS];
571 572
572 time_t wgp_latest_cookie_time; 573 time_t wgp_latest_cookie_time;
573 uint8_t wgp_latest_cookie[WG_COOKIE_LEN]; 574 uint8_t wgp_latest_cookie[WG_COOKIE_LEN];
574 uint8_t wgp_last_sent_mac1[WG_MAC_LEN]; 575 uint8_t wgp_last_sent_mac1[WG_MAC_LEN];
575 bool wgp_last_sent_mac1_valid; 576 bool wgp_last_sent_mac1_valid;
576 uint8_t wgp_last_sent_cookie[WG_COOKIE_LEN]; 577 uint8_t wgp_last_sent_cookie[WG_COOKIE_LEN];
577 bool wgp_last_sent_cookie_valid; 578 bool wgp_last_sent_cookie_valid;
578 579
579 time_t wgp_last_msg_received_time[WG_MSG_TYPE_MAX]; 580 time_t wgp_last_msg_received_time[WG_MSG_TYPE_MAX];
580 581
581 time_t wgp_last_genrandval_time; 582 time_t wgp_last_genrandval_time;
582 uint32_t wgp_randval; 583 uint32_t wgp_randval;
583 584
584 struct wg_ppsratecheck wgp_ppsratecheck; 585 struct wg_ppsratecheck wgp_ppsratecheck;
585 586
586 volatile unsigned int wgp_tasks; 587 volatile unsigned int wgp_tasks;
587#define WGP_TASK_SEND_INIT_MESSAGE __BIT(0) 588#define WGP_TASK_SEND_INIT_MESSAGE __BIT(0)
588#define WGP_TASK_ENDPOINT_CHANGED __BIT(1) 589#define WGP_TASK_ENDPOINT_CHANGED __BIT(1)
589#define WGP_TASK_SEND_KEEPALIVE_MESSAGE __BIT(2) 590#define WGP_TASK_SEND_KEEPALIVE_MESSAGE __BIT(2)
590#define WGP_TASK_DESTROY_PREV_SESSION __BIT(3) 591#define WGP_TASK_DESTROY_PREV_SESSION __BIT(3)
591}; 592};
592 593
593struct wg_ops; 594struct wg_ops;
594 595
595struct wg_softc { 596struct wg_softc {
596 struct ifnet wg_if; 597 struct ifnet wg_if;
597 LIST_ENTRY(wg_softc) wg_list; 598 LIST_ENTRY(wg_softc) wg_list;
598 kmutex_t *wg_lock; 599 kmutex_t *wg_lock;
599 krwlock_t *wg_rwlock; 600 krwlock_t *wg_rwlock;
600 601
601 uint8_t wg_privkey[WG_STATIC_KEY_LEN]; 602 uint8_t wg_privkey[WG_STATIC_KEY_LEN];
602 uint8_t wg_pubkey[WG_STATIC_KEY_LEN]; 603 uint8_t wg_pubkey[WG_STATIC_KEY_LEN];
603 604
604 int wg_npeers; 605 int wg_npeers;
605 struct pslist_head wg_peers; 606 struct pslist_head wg_peers;
 607 struct thmap *wg_peers_bypubkey;
 608 struct thmap *wg_peers_byname;
 609 struct thmap *wg_sessions_byindex;
606 uint16_t wg_listen_port; 610 uint16_t wg_listen_port;
607 611
608 struct wg_worker *wg_worker; 612 struct wg_worker *wg_worker;
609 lwp_t *wg_worker_lwp; 613 lwp_t *wg_worker_lwp;
610 614
611 struct radix_node_head *wg_rtable_ipv4; 615 struct radix_node_head *wg_rtable_ipv4;
612 struct radix_node_head *wg_rtable_ipv6; 616 struct radix_node_head *wg_rtable_ipv6;
613 617
614 struct wg_ppsratecheck wg_ppsratecheck; 618 struct wg_ppsratecheck wg_ppsratecheck;
615 619
616 struct wg_ops *wg_ops; 620 struct wg_ops *wg_ops;
617 621
618#ifdef WG_RUMPKERNEL 622#ifdef WG_RUMPKERNEL
619 struct wg_user *wg_user; 623 struct wg_user *wg_user;
620#endif 624#endif
621}; 625};
622 626
623/* [W] 6.1 Preliminaries */ 627/* [W] 6.1 Preliminaries */
624#define WG_REKEY_AFTER_MESSAGES (1ULL << 60) 628#define WG_REKEY_AFTER_MESSAGES (1ULL << 60)
625#define WG_REJECT_AFTER_MESSAGES (UINT64_MAX - (1 << 13)) 629#define WG_REJECT_AFTER_MESSAGES (UINT64_MAX - (1 << 13))
626#define WG_REKEY_AFTER_TIME 120 630#define WG_REKEY_AFTER_TIME 120
627#define WG_REJECT_AFTER_TIME 180 631#define WG_REJECT_AFTER_TIME 180
628#define WG_REKEY_ATTEMPT_TIME 90 632#define WG_REKEY_ATTEMPT_TIME 90
629#define WG_REKEY_TIMEOUT 5 633#define WG_REKEY_TIMEOUT 5
630#define WG_KEEPALIVE_TIMEOUT 10 634#define WG_KEEPALIVE_TIMEOUT 10
631 635
632#define WG_COOKIE_TIME 120 636#define WG_COOKIE_TIME 120
633#define WG_RANDVAL_TIME (2 * 60) 637#define WG_RANDVAL_TIME (2 * 60)
634 638
635static uint64_t wg_rekey_after_messages = WG_REKEY_AFTER_MESSAGES; 639static uint64_t wg_rekey_after_messages = WG_REKEY_AFTER_MESSAGES;
636static uint64_t wg_reject_after_messages = WG_REJECT_AFTER_MESSAGES; 640static uint64_t wg_reject_after_messages = WG_REJECT_AFTER_MESSAGES;
637static unsigned wg_rekey_after_time = WG_REKEY_AFTER_TIME; 641static unsigned wg_rekey_after_time = WG_REKEY_AFTER_TIME;
638static unsigned wg_reject_after_time = WG_REJECT_AFTER_TIME; 642static unsigned wg_reject_after_time = WG_REJECT_AFTER_TIME;
639static unsigned wg_rekey_attempt_time = WG_REKEY_ATTEMPT_TIME; 643static unsigned wg_rekey_attempt_time = WG_REKEY_ATTEMPT_TIME;
640static unsigned wg_rekey_timeout = WG_REKEY_TIMEOUT; 644static unsigned wg_rekey_timeout = WG_REKEY_TIMEOUT;
641static unsigned wg_keepalive_timeout = WG_KEEPALIVE_TIMEOUT; 645static unsigned wg_keepalive_timeout = WG_KEEPALIVE_TIMEOUT;
642 646
643static struct mbuf * 647static struct mbuf *
644 wg_get_mbuf(size_t, size_t); 648 wg_get_mbuf(size_t, size_t);
645 649
646static void wg_wakeup_worker(struct wg_worker *, int); 650static void wg_wakeup_worker(struct wg_worker *, int);
647 651
648static int wg_send_data_msg(struct wg_peer *, struct wg_session *, 652static int wg_send_data_msg(struct wg_peer *, struct wg_session *,
649 struct mbuf *); 653 struct mbuf *);
650static int wg_send_cookie_msg(struct wg_softc *, struct wg_peer *, 654static int wg_send_cookie_msg(struct wg_softc *, struct wg_peer *,
651 const uint32_t, const uint8_t [], const struct sockaddr *); 655 const uint32_t, const uint8_t [], const struct sockaddr *);
652static int wg_send_handshake_msg_resp(struct wg_softc *, 656static int wg_send_handshake_msg_resp(struct wg_softc *,
653 struct wg_peer *, const struct wg_msg_init *); 657 struct wg_peer *, const struct wg_msg_init *);
654static void wg_send_keepalive_msg(struct wg_peer *, struct wg_session *); 658static void wg_send_keepalive_msg(struct wg_peer *, struct wg_session *);
655 659
656static struct wg_peer * 660static struct wg_peer *
657 wg_pick_peer_by_sa(struct wg_softc *, const struct sockaddr *, 661 wg_pick_peer_by_sa(struct wg_softc *, const struct sockaddr *,
658 struct psref *); 662 struct psref *);
659static struct wg_peer * 663static struct wg_peer *
660 wg_lookup_peer_by_pubkey(struct wg_softc *, 664 wg_lookup_peer_by_pubkey(struct wg_softc *,
661 const uint8_t [], struct psref *); 665 const uint8_t [], struct psref *);
662 666
663static struct wg_session * 667static struct wg_session *
664 wg_lookup_session_by_index(struct wg_softc *, 668 wg_lookup_session_by_index(struct wg_softc *,
665 const uint32_t, struct psref *); 669 const uint32_t, struct psref *);
666 670
667static void wg_update_endpoint_if_necessary(struct wg_peer *, 671static void wg_update_endpoint_if_necessary(struct wg_peer *,
668 const struct sockaddr *); 672 const struct sockaddr *);
669 673
670static void wg_schedule_rekey_timer(struct wg_peer *); 674static void wg_schedule_rekey_timer(struct wg_peer *);
671static void wg_schedule_session_dtor_timer(struct wg_peer *); 675static void wg_schedule_session_dtor_timer(struct wg_peer *);
672static void wg_stop_session_dtor_timer(struct wg_peer *); 676static void wg_stop_session_dtor_timer(struct wg_peer *);
673 677
674static bool wg_is_underload(struct wg_softc *, struct wg_peer *, int); 678static bool wg_is_underload(struct wg_softc *, struct wg_peer *, int);
675static void wg_calculate_keys(struct wg_session *, const bool); 679static void wg_calculate_keys(struct wg_session *, const bool);
676 680
677static void wg_clear_states(struct wg_session *); 681static void wg_clear_states(struct wg_session *);
678 682
679static void wg_get_peer(struct wg_peer *, struct psref *); 683static void wg_get_peer(struct wg_peer *, struct psref *);
680static void wg_put_peer(struct wg_peer *, struct psref *); 684static void wg_put_peer(struct wg_peer *, struct psref *);
681 685
682static int wg_send_so(struct wg_peer *, struct mbuf *); 686static int wg_send_so(struct wg_peer *, struct mbuf *);
683static int wg_send_udp(struct wg_peer *, struct mbuf *); 687static int wg_send_udp(struct wg_peer *, struct mbuf *);
684static int wg_output(struct ifnet *, struct mbuf *, 688static int wg_output(struct ifnet *, struct mbuf *,
685 const struct sockaddr *, const struct rtentry *); 689 const struct sockaddr *, const struct rtentry *);
686static void wg_input(struct ifnet *, struct mbuf *, const int); 690static void wg_input(struct ifnet *, struct mbuf *, const int);
687static int wg_ioctl(struct ifnet *, u_long, void *); 691static int wg_ioctl(struct ifnet *, u_long, void *);
688static int wg_bind_port(struct wg_softc *, const uint16_t); 692static int wg_bind_port(struct wg_softc *, const uint16_t);
689static int wg_init(struct ifnet *); 693static int wg_init(struct ifnet *);
690static void wg_stop(struct ifnet *, int); 694static void wg_stop(struct ifnet *, int);
691 695
692static int wg_clone_create(struct if_clone *, int); 696static int wg_clone_create(struct if_clone *, int);
693static int wg_clone_destroy(struct ifnet *); 697static int wg_clone_destroy(struct ifnet *);
694 698
695struct wg_ops { 699struct wg_ops {
696 int (*send_hs_msg)(struct wg_peer *, struct mbuf *); 700 int (*send_hs_msg)(struct wg_peer *, struct mbuf *);
697 int (*send_data_msg)(struct wg_peer *, struct mbuf *); 701 int (*send_data_msg)(struct wg_peer *, struct mbuf *);
698 void (*input)(struct ifnet *, struct mbuf *, const int); 702 void (*input)(struct ifnet *, struct mbuf *, const int);
699 int (*bind_port)(struct wg_softc *, const uint16_t); 703 int (*bind_port)(struct wg_softc *, const uint16_t);
700}; 704};
701 705
702struct wg_ops wg_ops_rumpkernel = { 706struct wg_ops wg_ops_rumpkernel = {
703 .send_hs_msg = wg_send_so, 707 .send_hs_msg = wg_send_so,
704 .send_data_msg = wg_send_udp, 708 .send_data_msg = wg_send_udp,
705 .input = wg_input, 709 .input = wg_input,
706 .bind_port = wg_bind_port, 710 .bind_port = wg_bind_port,
707}; 711};
708 712
709#ifdef WG_RUMPKERNEL 713#ifdef WG_RUMPKERNEL
710static bool wg_user_mode(struct wg_softc *); 714static bool wg_user_mode(struct wg_softc *);
711static int wg_ioctl_linkstr(struct wg_softc *, struct ifdrv *); 715static int wg_ioctl_linkstr(struct wg_softc *, struct ifdrv *);
712 716
713static int wg_send_user(struct wg_peer *, struct mbuf *); 717static int wg_send_user(struct wg_peer *, struct mbuf *);
714static void wg_input_user(struct ifnet *, struct mbuf *, const int); 718static void wg_input_user(struct ifnet *, struct mbuf *, const int);
715static int wg_bind_port_user(struct wg_softc *, const uint16_t); 719static int wg_bind_port_user(struct wg_softc *, const uint16_t);
716 720
717struct wg_ops wg_ops_rumpuser = { 721struct wg_ops wg_ops_rumpuser = {
718 .send_hs_msg = wg_send_user, 722 .send_hs_msg = wg_send_user,
719 .send_data_msg = wg_send_user, 723 .send_data_msg = wg_send_user,
720 .input = wg_input_user, 724 .input = wg_input_user,
721 .bind_port = wg_bind_port_user, 725 .bind_port = wg_bind_port_user,
722}; 726};
723#endif 727#endif
724 728
725#define WG_PEER_READER_FOREACH(wgp, wg) \ 729#define WG_PEER_READER_FOREACH(wgp, wg) \
726 PSLIST_READER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 730 PSLIST_READER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \
727 wgp_peerlist_entry) 731 wgp_peerlist_entry)
728#define WG_PEER_WRITER_FOREACH(wgp, wg) \ 732#define WG_PEER_WRITER_FOREACH(wgp, wg) \
729 PSLIST_WRITER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \ 733 PSLIST_WRITER_FOREACH((wgp), &(wg)->wg_peers, struct wg_peer, \
730 wgp_peerlist_entry) 734 wgp_peerlist_entry)
731#define WG_PEER_WRITER_INSERT_HEAD(wgp, wg) \ 735#define WG_PEER_WRITER_INSERT_HEAD(wgp, wg) \
732 PSLIST_WRITER_INSERT_HEAD(&(wg)->wg_peers, (wgp), wgp_peerlist_entry) 736 PSLIST_WRITER_INSERT_HEAD(&(wg)->wg_peers, (wgp), wgp_peerlist_entry)
733#define WG_PEER_WRITER_REMOVE(wgp) \ 737#define WG_PEER_WRITER_REMOVE(wgp) \
734 PSLIST_WRITER_REMOVE((wgp), wgp_peerlist_entry) 738 PSLIST_WRITER_REMOVE((wgp), wgp_peerlist_entry)
735 739
736struct wg_route { 740struct wg_route {
737 struct radix_node wgr_nodes[2]; 741 struct radix_node wgr_nodes[2];
738 struct wg_peer *wgr_peer; 742 struct wg_peer *wgr_peer;
739}; 743};
740 744
741static struct radix_node_head * 745static struct radix_node_head *
742wg_rnh(struct wg_softc *wg, const int family) 746wg_rnh(struct wg_softc *wg, const int family)
743{ 747{
744 748
745 switch (family) { 749 switch (family) {
746 case AF_INET: 750 case AF_INET:
747 return wg->wg_rtable_ipv4; 751 return wg->wg_rtable_ipv4;
748#ifdef INET6 752#ifdef INET6
749 case AF_INET6: 753 case AF_INET6:
750 return wg->wg_rtable_ipv6; 754 return wg->wg_rtable_ipv6;
751#endif 755#endif
752 default: 756 default:
753 return NULL; 757 return NULL;
754 } 758 }
755} 759}
756 760
757 761
758/* 762/*
759 * Global variables 763 * Global variables
760 */ 764 */
761LIST_HEAD(wg_sclist, wg_softc); 765LIST_HEAD(wg_sclist, wg_softc);
762static struct { 766static struct {
763 struct wg_sclist list; 767 struct wg_sclist list;
764 kmutex_t lock; 768 kmutex_t lock;
765} wg_softcs __cacheline_aligned; 769} wg_softcs __cacheline_aligned;
766 770
767struct psref_class *wg_psref_class __read_mostly; 771struct psref_class *wg_psref_class __read_mostly;
768 772
769static struct if_clone wg_cloner = 773static struct if_clone wg_cloner =
770 IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy); 774 IF_CLONE_INITIALIZER("wg", wg_clone_create, wg_clone_destroy);
771 775
772 776
773void wgattach(int); 777void wgattach(int);
774/* ARGSUSED */ 778/* ARGSUSED */
775void 779void
776wgattach(int count) 780wgattach(int count)
777{ 781{
778 /* 782 /*
779 * Nothing to do here, initialization is handled by the 783 * Nothing to do here, initialization is handled by the
780 * module initialization code in wginit() below). 784 * module initialization code in wginit() below).
781 */ 785 */
782} 786}
783 787
784static void 788static void
785wginit(void) 789wginit(void)
786{ 790{
787 791
788 wg_psref_class = psref_class_create("wg", IPL_SOFTNET); 792 wg_psref_class = psref_class_create("wg", IPL_SOFTNET);
789 793
790 mutex_init(&wg_softcs.lock, MUTEX_DEFAULT, IPL_NONE); 794 mutex_init(&wg_softcs.lock, MUTEX_DEFAULT, IPL_NONE);
791 LIST_INIT(&wg_softcs.list); 795 LIST_INIT(&wg_softcs.list);
792 if_clone_attach(&wg_cloner); 796 if_clone_attach(&wg_cloner);
793} 797}
794 798
795static int 799static int
796wgdetach(void) 800wgdetach(void)
797{ 801{
798 int error = 0; 802 int error = 0;
799 803
800 mutex_enter(&wg_softcs.lock); 804 mutex_enter(&wg_softcs.lock);
801 if (!LIST_EMPTY(&wg_softcs.list)) { 805 if (!LIST_EMPTY(&wg_softcs.list)) {
802 mutex_exit(&wg_softcs.lock); 806 mutex_exit(&wg_softcs.lock);
803 error = EBUSY; 807 error = EBUSY;
804 } 808 }
805 809
806 if (error == 0) { 810 if (error == 0) {
807 psref_class_destroy(wg_psref_class); 811 psref_class_destroy(wg_psref_class);
808 812
809 if_clone_detach(&wg_cloner); 813 if_clone_detach(&wg_cloner);
810 } 814 }
811 815
812 return error; 816 return error;
813} 817}
814 818
815static void 819static void
816wg_init_key_and_hash(uint8_t ckey[WG_CHAINING_KEY_LEN], 820wg_init_key_and_hash(uint8_t ckey[WG_CHAINING_KEY_LEN],
817 uint8_t hash[WG_HASH_LEN]) 821 uint8_t hash[WG_HASH_LEN])
818{ 822{
819 /* [W] 5.4: CONSTRUCTION */ 823 /* [W] 5.4: CONSTRUCTION */
820 const char *signature = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; 824 const char *signature = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s";
821 /* [W] 5.4: IDENTIFIER */ 825 /* [W] 5.4: IDENTIFIER */
822 const char *id = "WireGuard v1 zx2c4 Jason@zx2c4.com"; 826 const char *id = "WireGuard v1 zx2c4 Jason@zx2c4.com";
823 struct blake2s state; 827 struct blake2s state;
824 828
825 blake2s(ckey, WG_CHAINING_KEY_LEN, NULL, 0, 829 blake2s(ckey, WG_CHAINING_KEY_LEN, NULL, 0,
826 signature, strlen(signature)); 830 signature, strlen(signature));
827 831
828 CTASSERT(WG_HASH_LEN == WG_CHAINING_KEY_LEN); 832 CTASSERT(WG_HASH_LEN == WG_CHAINING_KEY_LEN);
829 memcpy(hash, ckey, WG_CHAINING_KEY_LEN); 833 memcpy(hash, ckey, WG_CHAINING_KEY_LEN);
830 834
831 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 835 blake2s_init(&state, WG_HASH_LEN, NULL, 0);
832 blake2s_update(&state, ckey, WG_CHAINING_KEY_LEN); 836 blake2s_update(&state, ckey, WG_CHAINING_KEY_LEN);
833 blake2s_update(&state, id, strlen(id)); 837 blake2s_update(&state, id, strlen(id));
834 blake2s_final(&state, hash); 838 blake2s_final(&state, hash);
835 839
836 WG_DUMP_HASH("ckey", ckey); 840 WG_DUMP_HASH("ckey", ckey);
837 WG_DUMP_HASH("hash", hash); 841 WG_DUMP_HASH("hash", hash);
838} 842}
839 843
840static void 844static void
841wg_algo_hash(uint8_t hash[WG_HASH_LEN], const uint8_t input[], 845wg_algo_hash(uint8_t hash[WG_HASH_LEN], const uint8_t input[],
842 const size_t inputsize) 846 const size_t inputsize)
843{ 847{
844 struct blake2s state; 848 struct blake2s state;
845 849
846 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 850 blake2s_init(&state, WG_HASH_LEN, NULL, 0);
847 blake2s_update(&state, hash, WG_HASH_LEN); 851 blake2s_update(&state, hash, WG_HASH_LEN);
848 blake2s_update(&state, input, inputsize); 852 blake2s_update(&state, input, inputsize);
849 blake2s_final(&state, hash); 853 blake2s_final(&state, hash);
850} 854}
851 855
852static void 856static void
853wg_algo_mac(uint8_t out[], const size_t outsize, 857wg_algo_mac(uint8_t out[], const size_t outsize,
854 const uint8_t key[], const size_t keylen, 858 const uint8_t key[], const size_t keylen,
855 const uint8_t input1[], const size_t input1len, 859 const uint8_t input1[], const size_t input1len,
856 const uint8_t input2[], const size_t input2len) 860 const uint8_t input2[], const size_t input2len)
857{ 861{
858 struct blake2s state; 862 struct blake2s state;
859 863
860 blake2s_init(&state, outsize, key, keylen); 864 blake2s_init(&state, outsize, key, keylen);
861 865
862 blake2s_update(&state, input1, input1len); 866 blake2s_update(&state, input1, input1len);
863 if (input2 != NULL) 867 if (input2 != NULL)
864 blake2s_update(&state, input2, input2len); 868 blake2s_update(&state, input2, input2len);
865 blake2s_final(&state, out); 869 blake2s_final(&state, out);
866} 870}
867 871
868static void 872static void
869wg_algo_mac_mac1(uint8_t out[], const size_t outsize, 873wg_algo_mac_mac1(uint8_t out[], const size_t outsize,
870 const uint8_t input1[], const size_t input1len, 874 const uint8_t input1[], const size_t input1len,
871 const uint8_t input2[], const size_t input2len) 875 const uint8_t input2[], const size_t input2len)
872{ 876{
873 struct blake2s state; 877 struct blake2s state;
874 /* [W] 5.4: LABEL-MAC1 */ 878 /* [W] 5.4: LABEL-MAC1 */
875 const char *label = "mac1----"; 879 const char *label = "mac1----";
876 uint8_t key[WG_HASH_LEN]; 880 uint8_t key[WG_HASH_LEN];
877 881
878 blake2s_init(&state, sizeof(key), NULL, 0); 882 blake2s_init(&state, sizeof(key), NULL, 0);
879 blake2s_update(&state, label, strlen(label)); 883 blake2s_update(&state, label, strlen(label));
880 blake2s_update(&state, input1, input1len); 884 blake2s_update(&state, input1, input1len);
881 blake2s_final(&state, key); 885 blake2s_final(&state, key);
882 886
883 blake2s_init(&state, outsize, key, sizeof(key)); 887 blake2s_init(&state, outsize, key, sizeof(key));
884 if (input2 != NULL) 888 if (input2 != NULL)
885 blake2s_update(&state, input2, input2len); 889 blake2s_update(&state, input2, input2len);
886 blake2s_final(&state, out); 890 blake2s_final(&state, out);
887} 891}
888 892
889static void 893static void
890wg_algo_mac_cookie(uint8_t out[], const size_t outsize, 894wg_algo_mac_cookie(uint8_t out[], const size_t outsize,
891 const uint8_t input1[], const size_t input1len) 895 const uint8_t input1[], const size_t input1len)
892{ 896{
893 struct blake2s state; 897 struct blake2s state;
894 /* [W] 5.4: LABEL-COOKIE */ 898 /* [W] 5.4: LABEL-COOKIE */
895 const char *label = "cookie--"; 899 const char *label = "cookie--";
896 900
897 blake2s_init(&state, outsize, NULL, 0); 901 blake2s_init(&state, outsize, NULL, 0);
898 blake2s_update(&state, label, strlen(label)); 902 blake2s_update(&state, label, strlen(label));
899 blake2s_update(&state, input1, input1len); 903 blake2s_update(&state, input1, input1len);
900 blake2s_final(&state, out); 904 blake2s_final(&state, out);
901} 905}
902 906
903static void 907static void
904wg_algo_generate_keypair(uint8_t pubkey[WG_EPHEMERAL_KEY_LEN], 908wg_algo_generate_keypair(uint8_t pubkey[WG_EPHEMERAL_KEY_LEN],
905 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]) 909 uint8_t privkey[WG_EPHEMERAL_KEY_LEN])
906{ 910{
907 911
908 CTASSERT(WG_EPHEMERAL_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 912 CTASSERT(WG_EPHEMERAL_KEY_LEN == crypto_scalarmult_curve25519_BYTES);
909 913
910 cprng_strong(kern_cprng, privkey, WG_EPHEMERAL_KEY_LEN, 0); 914 cprng_strong(kern_cprng, privkey, WG_EPHEMERAL_KEY_LEN, 0);
911 crypto_scalarmult_base(pubkey, privkey); 915 crypto_scalarmult_base(pubkey, privkey);
912} 916}
913 917
914static void 918static void
915wg_algo_dh(uint8_t out[WG_DH_OUTPUT_LEN], 919wg_algo_dh(uint8_t out[WG_DH_OUTPUT_LEN],
916 const uint8_t privkey[WG_STATIC_KEY_LEN], 920 const uint8_t privkey[WG_STATIC_KEY_LEN],
917 const uint8_t pubkey[WG_STATIC_KEY_LEN]) 921 const uint8_t pubkey[WG_STATIC_KEY_LEN])
918{ 922{
919 923
920 CTASSERT(WG_STATIC_KEY_LEN == crypto_scalarmult_curve25519_BYTES); 924 CTASSERT(WG_STATIC_KEY_LEN == crypto_scalarmult_curve25519_BYTES);
921 925
922 int ret __diagused = crypto_scalarmult(out, privkey, pubkey); 926 int ret __diagused = crypto_scalarmult(out, privkey, pubkey);
923 KASSERT(ret == 0); 927 KASSERT(ret == 0);
924} 928}
925 929
926static void 930static void
927wg_algo_hmac(uint8_t out[], const size_t outlen, 931wg_algo_hmac(uint8_t out[], const size_t outlen,
928 const uint8_t key[], const size_t keylen, 932 const uint8_t key[], const size_t keylen,
929 const uint8_t in[], const size_t inlen) 933 const uint8_t in[], const size_t inlen)
930{ 934{
931#define IPAD 0x36 935#define IPAD 0x36
932#define OPAD 0x5c 936#define OPAD 0x5c
933 uint8_t hmackey[HMAC_BLOCK_LEN] = {0}; 937 uint8_t hmackey[HMAC_BLOCK_LEN] = {0};
934 uint8_t ipad[HMAC_BLOCK_LEN]; 938 uint8_t ipad[HMAC_BLOCK_LEN];
935 uint8_t opad[HMAC_BLOCK_LEN]; 939 uint8_t opad[HMAC_BLOCK_LEN];
936 int i; 940 int i;
937 struct blake2s state; 941 struct blake2s state;
938 942
939 KASSERT(outlen == WG_HASH_LEN); 943 KASSERT(outlen == WG_HASH_LEN);
940 KASSERT(keylen <= HMAC_BLOCK_LEN); 944 KASSERT(keylen <= HMAC_BLOCK_LEN);
941 945
942 memcpy(hmackey, key, keylen); 946 memcpy(hmackey, key, keylen);
943 947
944 for (i = 0; i < sizeof(hmackey); i++) { 948 for (i = 0; i < sizeof(hmackey); i++) {
945 ipad[i] = hmackey[i] ^ IPAD; 949 ipad[i] = hmackey[i] ^ IPAD;
946 opad[i] = hmackey[i] ^ OPAD; 950 opad[i] = hmackey[i] ^ OPAD;
947 } 951 }
948 952
949 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 953 blake2s_init(&state, WG_HASH_LEN, NULL, 0);
950 blake2s_update(&state, ipad, sizeof(ipad)); 954 blake2s_update(&state, ipad, sizeof(ipad));
951 blake2s_update(&state, in, inlen); 955 blake2s_update(&state, in, inlen);
952 blake2s_final(&state, out); 956 blake2s_final(&state, out);
953 957
954 blake2s_init(&state, WG_HASH_LEN, NULL, 0); 958 blake2s_init(&state, WG_HASH_LEN, NULL, 0);
955 blake2s_update(&state, opad, sizeof(opad)); 959 blake2s_update(&state, opad, sizeof(opad));
956 blake2s_update(&state, out, WG_HASH_LEN); 960 blake2s_update(&state, out, WG_HASH_LEN);
957 blake2s_final(&state, out); 961 blake2s_final(&state, out);
958#undef IPAD 962#undef IPAD
959#undef OPAD 963#undef OPAD
960} 964}
961 965
962static void 966static void
963wg_algo_kdf(uint8_t out1[WG_KDF_OUTPUT_LEN], uint8_t out2[WG_KDF_OUTPUT_LEN], 967wg_algo_kdf(uint8_t out1[WG_KDF_OUTPUT_LEN], uint8_t out2[WG_KDF_OUTPUT_LEN],
964 uint8_t out3[WG_KDF_OUTPUT_LEN], const uint8_t ckey[WG_CHAINING_KEY_LEN], 968 uint8_t out3[WG_KDF_OUTPUT_LEN], const uint8_t ckey[WG_CHAINING_KEY_LEN],
965 const uint8_t input[], const size_t inputlen) 969 const uint8_t input[], const size_t inputlen)
966{ 970{
967 uint8_t tmp1[WG_KDF_OUTPUT_LEN], tmp2[WG_KDF_OUTPUT_LEN + 1]; 971 uint8_t tmp1[WG_KDF_OUTPUT_LEN], tmp2[WG_KDF_OUTPUT_LEN + 1];
968 uint8_t one[1]; 972 uint8_t one[1];
969 973
970 /* 974 /*
971 * [N] 4.3: "an input_key_material byte sequence with length 975 * [N] 4.3: "an input_key_material byte sequence with length
972 * either zero bytes, 32 bytes, or DHLEN bytes." 976 * either zero bytes, 32 bytes, or DHLEN bytes."
973 */ 977 */
974 KASSERT(inputlen == 0 || inputlen == 32 || inputlen == NOISE_DHLEN); 978 KASSERT(inputlen == 0 || inputlen == 32 || inputlen == NOISE_DHLEN);
975 979
976 WG_DUMP_HASH("ckey", ckey); 980 WG_DUMP_HASH("ckey", ckey);
977 if (input != NULL) 981 if (input != NULL)
978 WG_DUMP_HASH("input", input); 982 WG_DUMP_HASH("input", input);
979 wg_algo_hmac(tmp1, sizeof(tmp1), ckey, WG_CHAINING_KEY_LEN, 983 wg_algo_hmac(tmp1, sizeof(tmp1), ckey, WG_CHAINING_KEY_LEN,
980 input, inputlen); 984 input, inputlen);
981 WG_DUMP_HASH("tmp1", tmp1); 985 WG_DUMP_HASH("tmp1", tmp1);
982 one[0] = 1; 986 one[0] = 1;
983 wg_algo_hmac(out1, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 987 wg_algo_hmac(out1, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1),
984 one, sizeof(one)); 988 one, sizeof(one));
985 WG_DUMP_HASH("out1", out1); 989 WG_DUMP_HASH("out1", out1);
986 if (out2 == NULL) 990 if (out2 == NULL)
987 return; 991 return;
988 memcpy(tmp2, out1, WG_KDF_OUTPUT_LEN); 992 memcpy(tmp2, out1, WG_KDF_OUTPUT_LEN);
989 tmp2[WG_KDF_OUTPUT_LEN] = 2; 993 tmp2[WG_KDF_OUTPUT_LEN] = 2;
990 wg_algo_hmac(out2, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 994 wg_algo_hmac(out2, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1),
991 tmp2, sizeof(tmp2)); 995 tmp2, sizeof(tmp2));
992 WG_DUMP_HASH("out2", out2); 996 WG_DUMP_HASH("out2", out2);
993 if (out3 == NULL) 997 if (out3 == NULL)
994 return; 998 return;
995 memcpy(tmp2, out2, WG_KDF_OUTPUT_LEN); 999 memcpy(tmp2, out2, WG_KDF_OUTPUT_LEN);
996 tmp2[WG_KDF_OUTPUT_LEN] = 3; 1000 tmp2[WG_KDF_OUTPUT_LEN] = 3;
997 wg_algo_hmac(out3, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1), 1001 wg_algo_hmac(out3, WG_KDF_OUTPUT_LEN, tmp1, sizeof(tmp1),
998 tmp2, sizeof(tmp2)); 1002 tmp2, sizeof(tmp2));
999 WG_DUMP_HASH("out3", out3); 1003 WG_DUMP_HASH("out3", out3);
1000} 1004}
1001 1005
1002static void 1006static void
1003wg_algo_dh_kdf(uint8_t ckey[WG_CHAINING_KEY_LEN], 1007wg_algo_dh_kdf(uint8_t ckey[WG_CHAINING_KEY_LEN],
1004 uint8_t cipher_key[WG_CIPHER_KEY_LEN], 1008 uint8_t cipher_key[WG_CIPHER_KEY_LEN],
1005 const uint8_t local_key[WG_STATIC_KEY_LEN], 1009 const uint8_t local_key[WG_STATIC_KEY_LEN],
1006 const uint8_t remote_key[WG_STATIC_KEY_LEN]) 1010 const uint8_t remote_key[WG_STATIC_KEY_LEN])
1007{ 1011{
1008 uint8_t dhout[WG_DH_OUTPUT_LEN]; 1012 uint8_t dhout[WG_DH_OUTPUT_LEN];
1009 1013
1010 wg_algo_dh(dhout, local_key, remote_key); 1014 wg_algo_dh(dhout, local_key, remote_key);
1011 wg_algo_kdf(ckey, cipher_key, NULL, ckey, dhout, sizeof(dhout)); 1015 wg_algo_kdf(ckey, cipher_key, NULL, ckey, dhout, sizeof(dhout));
1012 1016
1013 WG_DUMP_HASH("dhout", dhout); 1017 WG_DUMP_HASH("dhout", dhout);
1014 WG_DUMP_HASH("ckey", ckey); 1018 WG_DUMP_HASH("ckey", ckey);
1015 if (cipher_key != NULL) 1019 if (cipher_key != NULL)
1016 WG_DUMP_HASH("cipher_key", cipher_key); 1020 WG_DUMP_HASH("cipher_key", cipher_key);
1017} 1021}
1018 1022
1019static void 1023static void
1020wg_algo_aead_enc(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1024wg_algo_aead_enc(uint8_t out[], size_t expected_outsize, const uint8_t key[],
1021 const uint64_t counter, const uint8_t plain[], const size_t plainsize, 1025 const uint64_t counter, const uint8_t plain[], const size_t plainsize,
1022 const uint8_t auth[], size_t authlen) 1026 const uint8_t auth[], size_t authlen)
1023{ 1027{
1024 uint8_t nonce[(32 + 64) / 8] = {0}; 1028 uint8_t nonce[(32 + 64) / 8] = {0};
1025 long long unsigned int outsize; 1029 long long unsigned int outsize;
1026 int error __diagused; 1030 int error __diagused;
1027 1031
1028 memcpy(&nonce[4], &counter, sizeof(counter)); 1032 memcpy(&nonce[4], &counter, sizeof(counter));
1029 1033
1030 error = crypto_aead_chacha20poly1305_ietf_encrypt(out, &outsize, plain, 1034 error = crypto_aead_chacha20poly1305_ietf_encrypt(out, &outsize, plain,
1031 plainsize, auth, authlen, NULL, nonce, key); 1035 plainsize, auth, authlen, NULL, nonce, key);
1032 KASSERT(error == 0); 1036 KASSERT(error == 0);
1033 KASSERT(outsize == expected_outsize); 1037 KASSERT(outsize == expected_outsize);
1034} 1038}
1035 1039
1036static int 1040static int
1037wg_algo_aead_dec(uint8_t out[], size_t expected_outsize, const uint8_t key[], 1041wg_algo_aead_dec(uint8_t out[], size_t expected_outsize, const uint8_t key[],
1038 const uint64_t counter, const uint8_t encrypted[], 1042 const uint64_t counter, const uint8_t encrypted[],
1039 const size_t encryptedsize, const uint8_t auth[], size_t authlen) 1043 const size_t encryptedsize, const uint8_t auth[], size_t authlen)
1040{ 1044{
1041 uint8_t nonce[(32 + 64) / 8] = {0}; 1045 uint8_t nonce[(32 + 64) / 8] = {0};
1042 long long unsigned int outsize; 1046 long long unsigned int outsize;
1043 int error; 1047 int error;
1044 1048
1045 memcpy(&nonce[4], &counter, sizeof(counter)); 1049 memcpy(&nonce[4], &counter, sizeof(counter));
1046 1050
1047 error = crypto_aead_chacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1051 error = crypto_aead_chacha20poly1305_ietf_decrypt(out, &outsize, NULL,
1048 encrypted, encryptedsize, auth, authlen, nonce, key); 1052 encrypted, encryptedsize, auth, authlen, nonce, key);
1049 if (error == 0) 1053 if (error == 0)
1050 KASSERT(outsize == expected_outsize); 1054 KASSERT(outsize == expected_outsize);
1051 return error; 1055 return error;
1052} 1056}
1053 1057
1054static void 1058static void
1055wg_algo_xaead_enc(uint8_t out[], const size_t expected_outsize, 1059wg_algo_xaead_enc(uint8_t out[], const size_t expected_outsize,
1056 const uint8_t key[], const uint8_t plain[], const size_t plainsize, 1060 const uint8_t key[], const uint8_t plain[], const size_t plainsize,
1057 const uint8_t auth[], size_t authlen, 1061 const uint8_t auth[], size_t authlen,
1058 const uint8_t nonce[WG_SALT_LEN]) 1062 const uint8_t nonce[WG_SALT_LEN])
1059{ 1063{
1060 long long unsigned int outsize; 1064 long long unsigned int outsize;
1061 int error __diagused; 1065 int error __diagused;
1062 1066
1063 CTASSERT(WG_SALT_LEN == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES); 1067 CTASSERT(WG_SALT_LEN == crypto_aead_xchacha20poly1305_ietf_NPUBBYTES);
1064 error = crypto_aead_xchacha20poly1305_ietf_encrypt(out, &outsize, 1068 error = crypto_aead_xchacha20poly1305_ietf_encrypt(out, &outsize,
1065 plain, plainsize, auth, authlen, NULL, nonce, key); 1069 plain, plainsize, auth, authlen, NULL, nonce, key);
1066 KASSERT(error == 0); 1070 KASSERT(error == 0);
1067 KASSERT(outsize == expected_outsize); 1071 KASSERT(outsize == expected_outsize);
1068} 1072}
1069 1073
1070static int 1074static int
1071wg_algo_xaead_dec(uint8_t out[], const size_t expected_outsize, 1075wg_algo_xaead_dec(uint8_t out[], const size_t expected_outsize,
1072 const uint8_t key[], const uint8_t encrypted[], const size_t encryptedsize, 1076 const uint8_t key[], const uint8_t encrypted[], const size_t encryptedsize,
1073 const uint8_t auth[], size_t authlen, 1077 const uint8_t auth[], size_t authlen,
1074 const uint8_t nonce[WG_SALT_LEN]) 1078 const uint8_t nonce[WG_SALT_LEN])
1075{ 1079{
1076 long long unsigned int outsize; 1080 long long unsigned int outsize;
1077 int error; 1081 int error;
1078 1082
1079 error = crypto_aead_xchacha20poly1305_ietf_decrypt(out, &outsize, NULL, 1083 error = crypto_aead_xchacha20poly1305_ietf_decrypt(out, &outsize, NULL,
1080 encrypted, encryptedsize, auth, authlen, nonce, key); 1084 encrypted, encryptedsize, auth, authlen, nonce, key);
1081 if (error == 0) 1085 if (error == 0)
1082 KASSERT(outsize == expected_outsize); 1086 KASSERT(outsize == expected_outsize);
1083 return error; 1087 return error;
1084} 1088}
1085 1089
1086static void 1090static void
1087wg_algo_tai64n(wg_timestamp_t timestamp) 1091wg_algo_tai64n(wg_timestamp_t timestamp)
1088{ 1092{
1089 struct timespec ts; 1093 struct timespec ts;
1090 1094
1091 /* FIXME strict TAI64N (https://cr.yp.to/libtai/tai64.html) */ 1095 /* FIXME strict TAI64N (https://cr.yp.to/libtai/tai64.html) */
1092 getnanotime(&ts); 1096 getnanotime(&ts);
1093 /* TAI64 label in external TAI64 format */ 1097 /* TAI64 label in external TAI64 format */
1094 be32enc(timestamp, 0x40000000UL + (ts.tv_sec >> 32)); 1098 be32enc(timestamp, 0x40000000UL + (ts.tv_sec >> 32));
1095 /* second beginning from 1970 TAI */ 1099 /* second beginning from 1970 TAI */
1096 be32enc(timestamp + 4, ts.tv_sec & 0xffffffffU); 1100 be32enc(timestamp + 4, ts.tv_sec & 0xffffffffU);
1097 /* nanosecond in big-endian format */ 1101 /* nanosecond in big-endian format */
1098 be32enc(timestamp + 8, ts.tv_nsec); 1102 be32enc(timestamp + 8, ts.tv_nsec);
1099} 1103}
1100 1104
1101static struct wg_session * 1105static struct wg_session *
1102wg_get_unstable_session(struct wg_peer *wgp, struct psref *psref) 1106wg_get_unstable_session(struct wg_peer *wgp, struct psref *psref)
1103{ 1107{
1104 int s; 1108 int s;
1105 struct wg_session *wgs; 1109 struct wg_session *wgs;
1106 1110
1107 s = pserialize_read_enter(); 1111 s = pserialize_read_enter();
1108 wgs = wgp->wgp_session_unstable; 1112 wgs = wgp->wgp_session_unstable;
1109 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1113 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class);
1110 pserialize_read_exit(s); 1114 pserialize_read_exit(s);
1111 return wgs; 1115 return wgs;
1112} 1116}
1113 1117
1114static struct wg_session * 1118static struct wg_session *
1115wg_get_stable_session(struct wg_peer *wgp, struct psref *psref) 1119wg_get_stable_session(struct wg_peer *wgp, struct psref *psref)
1116{ 1120{
1117 int s; 1121 int s;
1118 struct wg_session *wgs; 1122 struct wg_session *wgs;
1119 1123
1120 s = pserialize_read_enter(); 1124 s = pserialize_read_enter();
1121 wgs = wgp->wgp_session_stable; 1125 wgs = wgp->wgp_session_stable;
1122 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1126 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class);
1123 pserialize_read_exit(s); 1127 pserialize_read_exit(s);
1124 return wgs; 1128 return wgs;
1125} 1129}
1126 1130
1127static void 1131static void
1128wg_get_session(struct wg_session *wgs, struct psref *psref) 1132wg_get_session(struct wg_session *wgs, struct psref *psref)
1129{ 1133{
1130 1134
1131 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 1135 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class);
1132} 1136}
1133 1137
1134static void 1138static void
1135wg_put_session(struct wg_session *wgs, struct psref *psref) 1139wg_put_session(struct wg_session *wgs, struct psref *psref)
1136{ 1140{
1137 1141
1138 psref_release(psref, &wgs->wgs_psref, wg_psref_class); 1142 psref_release(psref, &wgs->wgs_psref, wg_psref_class);
1139} 1143}
1140 1144
1141static struct wg_session * 1145static struct wg_session *
1142wg_lock_unstable_session(struct wg_peer *wgp) 1146wg_lock_unstable_session(struct wg_peer *wgp)
1143{ 1147{
1144 struct wg_session *wgs; 1148 struct wg_session *wgs;
1145 1149
1146 mutex_enter(wgp->wgp_lock); 1150 mutex_enter(wgp->wgp_lock);
1147 wgs = wgp->wgp_session_unstable; 1151 wgs = wgp->wgp_session_unstable;
1148 mutex_enter(wgs->wgs_lock); 1152 mutex_enter(wgs->wgs_lock);
1149 mutex_exit(wgp->wgp_lock); 1153 mutex_exit(wgp->wgp_lock);
1150 return wgs; 1154 return wgs;
1151} 1155}
1152 1156
1153#if 0 1157#if 0
1154static void 1158static void
1155wg_unlock_session(struct wg_peer *wgp, struct wg_session *wgs) 1159wg_unlock_session(struct wg_peer *wgp, struct wg_session *wgs)
1156{ 1160{
1157 1161
1158 mutex_exit(wgs->wgs_lock); 1162 mutex_exit(wgs->wgs_lock);
1159} 1163}
1160#endif 1164#endif
1161 1165
 1166static uint32_t
 1167wg_assign_sender_index(struct wg_softc *wg, struct wg_session *wgs)
 1168{
 1169 struct wg_peer *wgp = wgs->wgs_peer;
 1170 struct wg_session *wgs0;
 1171 uint32_t index;
 1172 void *garbage;
 1173
 1174 mutex_enter(wgs->wgs_lock);
 1175
 1176 /* Release the current index, if there is one. */
 1177 while ((index = wgs->wgs_sender_index) != 0) {
 1178 /* Remove the session by index. */
 1179 thmap_del(wg->wg_sessions_byindex, &index, sizeof index);
 1180 wgs->wgs_sender_index = 0;
 1181 mutex_exit(wgs->wgs_lock);
 1182
 1183 /* Wait for all thmap_gets to complete, and GC. */
 1184 garbage = thmap_stage_gc(wg->wg_sessions_byindex);
 1185 mutex_enter(wgs->wgs_peer->wgp_lock);
 1186 pserialize_perform(wgp->wgp_psz);
 1187 mutex_exit(wgs->wgs_peer->wgp_lock);
 1188 thmap_gc(wg->wg_sessions_byindex, garbage);
 1189
 1190 mutex_enter(wgs->wgs_lock);
 1191 }
 1192
 1193restart:
 1194 /* Pick a uniform random nonzero index. */
 1195 while (__predict_false((index = cprng_strong32()) == 0))
 1196 continue;
 1197
 1198 /* Try to take it. */
 1199 wgs->wgs_sender_index = index;
 1200 wgs0 = thmap_put(wg->wg_sessions_byindex,
 1201 &wgs->wgs_sender_index, sizeof wgs->wgs_sender_index, wgs);
 1202
 1203 /* If someone else beat us, start over. */
 1204 if (__predict_false(wgs0 != wgs))
 1205 goto restart;
 1206
 1207 mutex_exit(wgs->wgs_lock);
 1208
 1209 return index;
 1210}
 1211
1162/* 1212/*
1163 * Handshake patterns 1213 * Handshake patterns
1164 * 1214 *
1165 * [W] 5: "These messages use the "IK" pattern from Noise" 1215 * [W] 5: "These messages use the "IK" pattern from Noise"
1166 * [N] 7.5. Interactive handshake patterns (fundamental) 1216 * [N] 7.5. Interactive handshake patterns (fundamental)
1167 * "The first character refers to the initiator’s static key:" 1217 * "The first character refers to the initiator’s static key:"
1168 * "I = Static key for initiator Immediately transmitted to responder, 1218 * "I = Static key for initiator Immediately transmitted to responder,
1169 * despite reduced or absent identity hiding" 1219 * despite reduced or absent identity hiding"
1170 * "The second character refers to the responder’s static key:" 1220 * "The second character refers to the responder’s static key:"
1171 * "K = Static key for responder Known to initiator" 1221 * "K = Static key for responder Known to initiator"
1172 * "IK: 1222 * "IK:
1173 * <- s 1223 * <- s
1174 * ... 1224 * ...
1175 * -> e, es, s, ss 1225 * -> e, es, s, ss
1176 * <- e, ee, se" 1226 * <- e, ee, se"
1177 * [N] 9.4. Pattern modifiers 1227 * [N] 9.4. Pattern modifiers
1178 * "IKpsk2: 1228 * "IKpsk2:
1179 * <- s 1229 * <- s
1180 * ... 1230 * ...
1181 * -> e, es, s, ss 1231 * -> e, es, s, ss
1182 * <- e, ee, se, psk" 1232 * <- e, ee, se, psk"
1183 */ 1233 */
1184static void 1234static void
1185wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp, 1235wg_fill_msg_init(struct wg_softc *wg, struct wg_peer *wgp,
1186 struct wg_session *wgs, struct wg_msg_init *wgmi) 1236 struct wg_session *wgs, struct wg_msg_init *wgmi)
1187{ 1237{
1188 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1238 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */
1189 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1239 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */
1190 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1240 uint8_t cipher_key[WG_CIPHER_KEY_LEN];
1191 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1241 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN];
1192 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1242 uint8_t privkey[WG_EPHEMERAL_KEY_LEN];
1193 1243
1194 wgmi->wgmi_type = WG_MSG_TYPE_INIT; 1244 wgmi->wgmi_type = WG_MSG_TYPE_INIT;
1195 wgmi->wgmi_sender = cprng_strong32(); 1245 wgmi->wgmi_sender = wg_assign_sender_index(wg, wgs);
1196 1246
1197 /* [W] 5.4.2: First Message: Initiator to Responder */ 1247 /* [W] 5.4.2: First Message: Initiator to Responder */
1198 1248
1199 /* Ci := HASH(CONSTRUCTION) */ 1249 /* Ci := HASH(CONSTRUCTION) */
1200 /* Hi := HASH(Ci || IDENTIFIER) */ 1250 /* Hi := HASH(Ci || IDENTIFIER) */
1201 wg_init_key_and_hash(ckey, hash); 1251 wg_init_key_and_hash(ckey, hash);
1202 /* Hi := HASH(Hi || Sr^pub) */ 1252 /* Hi := HASH(Hi || Sr^pub) */
1203 wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey)); 1253 wg_algo_hash(hash, wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey));
1204 1254
1205 WG_DUMP_HASH("hash", hash); 1255 WG_DUMP_HASH("hash", hash);
1206 1256
1207 /* [N] 2.2: "e" */ 1257 /* [N] 2.2: "e" */
1208 /* Ei^priv, Ei^pub := DH-GENERATE() */ 1258 /* Ei^priv, Ei^pub := DH-GENERATE() */
1209 wg_algo_generate_keypair(pubkey, privkey); 1259 wg_algo_generate_keypair(pubkey, privkey);
1210 /* Ci := KDF1(Ci, Ei^pub) */ 1260 /* Ci := KDF1(Ci, Ei^pub) */
1211 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1261 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey));
1212 /* msg.ephemeral := Ei^pub */ 1262 /* msg.ephemeral := Ei^pub */
1213 memcpy(wgmi->wgmi_ephemeral, pubkey, sizeof(wgmi->wgmi_ephemeral)); 1263 memcpy(wgmi->wgmi_ephemeral, pubkey, sizeof(wgmi->wgmi_ephemeral));
1214 /* Hi := HASH(Hi || msg.ephemeral) */ 1264 /* Hi := HASH(Hi || msg.ephemeral) */
1215 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1265 wg_algo_hash(hash, pubkey, sizeof(pubkey));
1216 1266
1217 WG_DUMP_HASH("ckey", ckey); 1267 WG_DUMP_HASH("ckey", ckey);
1218 WG_DUMP_HASH("hash", hash); 1268 WG_DUMP_HASH("hash", hash);
1219 1269
1220 /* [N] 2.2: "es" */ 1270 /* [N] 2.2: "es" */
1221 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1271 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */
1222 wg_algo_dh_kdf(ckey, cipher_key, privkey, wgp->wgp_pubkey); 1272 wg_algo_dh_kdf(ckey, cipher_key, privkey, wgp->wgp_pubkey);
1223 1273
1224 /* [N] 2.2: "s" */ 1274 /* [N] 2.2: "s" */
1225 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1275 /* msg.static := AEAD(k, 0, Si^pub, Hi) */
1226 wg_algo_aead_enc(wgmi->wgmi_static, sizeof(wgmi->wgmi_static), 1276 wg_algo_aead_enc(wgmi->wgmi_static, sizeof(wgmi->wgmi_static),
1227 cipher_key, 0, wg->wg_pubkey, sizeof(wg->wg_pubkey), 1277 cipher_key, 0, wg->wg_pubkey, sizeof(wg->wg_pubkey),
1228 hash, sizeof(hash)); 1278 hash, sizeof(hash));
1229 /* Hi := HASH(Hi || msg.static) */ 1279 /* Hi := HASH(Hi || msg.static) */
1230 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1280 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static));
1231 1281
1232 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1282 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static);
1233 1283
1234 /* [N] 2.2: "ss" */ 1284 /* [N] 2.2: "ss" */
1235 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1285 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */
1236 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1286 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey);
1237 1287
1238 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1288 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */
1239 wg_timestamp_t timestamp; 1289 wg_timestamp_t timestamp;
1240 wg_algo_tai64n(timestamp); 1290 wg_algo_tai64n(timestamp);
1241 wg_algo_aead_enc(wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1291 wg_algo_aead_enc(wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp),
1242 cipher_key, 0, timestamp, sizeof(timestamp), hash, sizeof(hash)); 1292 cipher_key, 0, timestamp, sizeof(timestamp), hash, sizeof(hash));
1243 /* Hi := HASH(Hi || msg.timestamp) */ 1293 /* Hi := HASH(Hi || msg.timestamp) */
1244 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1294 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp));
1245 1295
1246 /* [W] 5.4.4 Cookie MACs */ 1296 /* [W] 5.4.4 Cookie MACs */
1247 wg_algo_mac_mac1(wgmi->wgmi_mac1, sizeof(wgmi->wgmi_mac1), 1297 wg_algo_mac_mac1(wgmi->wgmi_mac1, sizeof(wgmi->wgmi_mac1),
1248 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1298 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey),
1249 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1299 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1));
1250 /* Need mac1 to decrypt a cookie from a cookie message */ 1300 /* Need mac1 to decrypt a cookie from a cookie message */
1251 memcpy(wgp->wgp_last_sent_mac1, wgmi->wgmi_mac1, 1301 memcpy(wgp->wgp_last_sent_mac1, wgmi->wgmi_mac1,
1252 sizeof(wgp->wgp_last_sent_mac1)); 1302 sizeof(wgp->wgp_last_sent_mac1));
1253 wgp->wgp_last_sent_mac1_valid = true; 1303 wgp->wgp_last_sent_mac1_valid = true;
1254 1304
1255 if (wgp->wgp_latest_cookie_time == 0 || 1305 if (wgp->wgp_latest_cookie_time == 0 ||
1256 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1306 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME)
1257 memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2)); 1307 memset(wgmi->wgmi_mac2, 0, sizeof(wgmi->wgmi_mac2));
1258 else { 1308 else {
1259 wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2), 1309 wg_algo_mac(wgmi->wgmi_mac2, sizeof(wgmi->wgmi_mac2),
1260 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1310 wgp->wgp_latest_cookie, WG_COOKIE_LEN,
1261 (const uint8_t *)wgmi, 1311 (const uint8_t *)wgmi,
1262 offsetof(struct wg_msg_init, wgmi_mac2), 1312 offsetof(struct wg_msg_init, wgmi_mac2),
1263 NULL, 0); 1313 NULL, 0);
1264 } 1314 }
1265 1315
1266 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1316 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey));
1267 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1317 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey));
1268 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1318 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash));
1269 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1319 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey));
1270 wgs->wgs_sender_index = wgmi->wgmi_sender; 
1271 WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_sender_index); 1320 WG_DLOG("%s: sender=%x\n", __func__, wgs->wgs_sender_index);
1272} 1321}
1273 1322
1274static void 1323static void
1275wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi, 1324wg_handle_msg_init(struct wg_softc *wg, const struct wg_msg_init *wgmi,
1276 const struct sockaddr *src) 1325 const struct sockaddr *src)
1277{ 1326{
1278 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */ 1327 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.2: Ci */
1279 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */ 1328 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.2: Hi */
1280 uint8_t cipher_key[WG_CIPHER_KEY_LEN]; 1329 uint8_t cipher_key[WG_CIPHER_KEY_LEN];
1281 uint8_t peer_pubkey[WG_STATIC_KEY_LEN]; 1330 uint8_t peer_pubkey[WG_STATIC_KEY_LEN];
1282 struct wg_peer *wgp; 1331 struct wg_peer *wgp;
1283 struct wg_session *wgs; 1332 struct wg_session *wgs;
1284 bool reset_state_on_error = false; 1333 bool reset_state_on_error = false;
1285 int error, ret; 1334 int error, ret;
1286 struct psref psref_peer; 1335 struct psref psref_peer;
1287 struct psref psref_session; 1336 struct psref psref_session;
1288 uint8_t mac1[WG_MAC_LEN]; 1337 uint8_t mac1[WG_MAC_LEN];
1289 1338
1290 WG_TRACE("init msg received"); 1339 WG_TRACE("init msg received");
1291 1340
1292 /* 1341 /*
1293 * [W] 5.4.2: First Message: Initiator to Responder 1342 * [W] 5.4.2: First Message: Initiator to Responder
1294 * "When the responder receives this message, it does the same 1343 * "When the responder receives this message, it does the same
1295 * operations so that its final state variables are identical, 1344 * operations so that its final state variables are identical,
1296 * replacing the operands of the DH function to produce equivalent 1345 * replacing the operands of the DH function to produce equivalent
1297 * values." 1346 * values."
1298 * Note that the following comments of operations are just copies of 1347 * Note that the following comments of operations are just copies of
1299 * the initiator's ones. 1348 * the initiator's ones.
1300 */ 1349 */
1301 1350
1302 /* Ci := HASH(CONSTRUCTION) */ 1351 /* Ci := HASH(CONSTRUCTION) */
1303 /* Hi := HASH(Ci || IDENTIFIER) */ 1352 /* Hi := HASH(Ci || IDENTIFIER) */
1304 wg_init_key_and_hash(ckey, hash); 1353 wg_init_key_and_hash(ckey, hash);
1305 /* Hi := HASH(Hi || Sr^pub) */ 1354 /* Hi := HASH(Hi || Sr^pub) */
1306 wg_algo_hash(hash, wg->wg_pubkey, sizeof(wg->wg_pubkey)); 1355 wg_algo_hash(hash, wg->wg_pubkey, sizeof(wg->wg_pubkey));
1307 1356
1308 /* [N] 2.2: "e" */ 1357 /* [N] 2.2: "e" */
1309 /* Ci := KDF1(Ci, Ei^pub) */ 1358 /* Ci := KDF1(Ci, Ei^pub) */
1310 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmi->wgmi_ephemeral, 1359 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmi->wgmi_ephemeral,
1311 sizeof(wgmi->wgmi_ephemeral)); 1360 sizeof(wgmi->wgmi_ephemeral));
1312 /* Hi := HASH(Hi || msg.ephemeral) */ 1361 /* Hi := HASH(Hi || msg.ephemeral) */
1313 wg_algo_hash(hash, wgmi->wgmi_ephemeral, sizeof(wgmi->wgmi_ephemeral)); 1362 wg_algo_hash(hash, wgmi->wgmi_ephemeral, sizeof(wgmi->wgmi_ephemeral));
1314 1363
1315 WG_DUMP_HASH("ckey", ckey); 1364 WG_DUMP_HASH("ckey", ckey);
1316 1365
1317 /* [N] 2.2: "es" */ 1366 /* [N] 2.2: "es" */
1318 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */ 1367 /* Ci, k := KDF2(Ci, DH(Ei^priv, Sr^pub)) */
1319 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgmi->wgmi_ephemeral); 1368 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgmi->wgmi_ephemeral);
1320 1369
1321 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static); 1370 WG_DUMP_HASH48("wgmi_static", wgmi->wgmi_static);
1322 1371
1323 /* [N] 2.2: "s" */ 1372 /* [N] 2.2: "s" */
1324 /* msg.static := AEAD(k, 0, Si^pub, Hi) */ 1373 /* msg.static := AEAD(k, 0, Si^pub, Hi) */
1325 error = wg_algo_aead_dec(peer_pubkey, WG_STATIC_KEY_LEN, cipher_key, 0, 1374 error = wg_algo_aead_dec(peer_pubkey, WG_STATIC_KEY_LEN, cipher_key, 0,
1326 wgmi->wgmi_static, sizeof(wgmi->wgmi_static), hash, sizeof(hash)); 1375 wgmi->wgmi_static, sizeof(wgmi->wgmi_static), hash, sizeof(hash));
1327 if (error != 0) { 1376 if (error != 0) {
1328 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 1377 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG,
1329 "wg_algo_aead_dec for secret key failed\n"); 1378 "wg_algo_aead_dec for secret key failed\n");
1330 return; 1379 return;
1331 } 1380 }
1332 /* Hi := HASH(Hi || msg.static) */ 1381 /* Hi := HASH(Hi || msg.static) */
1333 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static)); 1382 wg_algo_hash(hash, wgmi->wgmi_static, sizeof(wgmi->wgmi_static));
1334 1383
1335 wgp = wg_lookup_peer_by_pubkey(wg, peer_pubkey, &psref_peer); 1384 wgp = wg_lookup_peer_by_pubkey(wg, peer_pubkey, &psref_peer);
1336 if (wgp == NULL) { 1385 if (wgp == NULL) {
1337 WG_DLOG("peer not found\n"); 1386 WG_DLOG("peer not found\n");
1338 return; 1387 return;
1339 } 1388 }
1340 1389
1341 wgs = wg_lock_unstable_session(wgp); 1390 wgs = wg_lock_unstable_session(wgp);
1342 if (wgs->wgs_state == WGS_STATE_DESTROYING) { 1391 if (wgs->wgs_state == WGS_STATE_DESTROYING) {
1343 /* 1392 /*
1344 * We can assume that the peer doesn't have an established 1393 * We can assume that the peer doesn't have an established
1345 * session, so clear it now. 1394 * session, so clear it now.
1346 */ 1395 */
1347 WG_TRACE("Session destroying, but force to clear"); 1396 WG_TRACE("Session destroying, but force to clear");
1348 wg_stop_session_dtor_timer(wgp); 1397 wg_stop_session_dtor_timer(wgp);
1349 wg_clear_states(wgs); 1398 wg_clear_states(wgs);
1350 wgs->wgs_state = WGS_STATE_UNKNOWN; 1399 wgs->wgs_state = WGS_STATE_UNKNOWN;
1351 } 1400 }
1352 if (wgs->wgs_state == WGS_STATE_INIT_ACTIVE) { 1401 if (wgs->wgs_state == WGS_STATE_INIT_ACTIVE) {
1353 WG_TRACE("Sesssion already initializing, ignoring the message"); 1402 WG_TRACE("Sesssion already initializing, ignoring the message");
1354 mutex_exit(wgs->wgs_lock); 1403 mutex_exit(wgs->wgs_lock);
1355 goto out_wgp; 1404 goto out_wgp;
1356 } 1405 }
1357 if (wgs->wgs_state == WGS_STATE_INIT_PASSIVE) { 1406 if (wgs->wgs_state == WGS_STATE_INIT_PASSIVE) {
1358 WG_TRACE("Sesssion already initializing, destroying old states"); 1407 WG_TRACE("Sesssion already initializing, destroying old states");
1359 wg_clear_states(wgs); 1408 wg_clear_states(wgs);
1360 } 1409 }
1361 wgs->wgs_state = WGS_STATE_INIT_PASSIVE; 1410 wgs->wgs_state = WGS_STATE_INIT_PASSIVE;
1362 reset_state_on_error = true; 1411 reset_state_on_error = true;
1363 wg_get_session(wgs, &psref_session); 1412 wg_get_session(wgs, &psref_session);
1364 mutex_exit(wgs->wgs_lock); 1413 mutex_exit(wgs->wgs_lock);
1365 1414
1366 wg_algo_mac_mac1(mac1, sizeof(mac1), 1415 wg_algo_mac_mac1(mac1, sizeof(mac1),
1367 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1416 wg->wg_pubkey, sizeof(wg->wg_pubkey),
1368 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1)); 1417 (const uint8_t *)wgmi, offsetof(struct wg_msg_init, wgmi_mac1));
1369 1418
1370 /* 1419 /*
1371 * [W] 5.3: Denial of Service Mitigation & Cookies 1420 * [W] 5.3: Denial of Service Mitigation & Cookies
1372 * "the responder, ..., must always reject messages with an invalid 1421 * "the responder, ..., must always reject messages with an invalid
1373 * msg.mac1" 1422 * msg.mac1"
1374 */ 1423 */
1375 if (!consttime_memequal(mac1, wgmi->wgmi_mac1, sizeof(mac1))) { 1424 if (!consttime_memequal(mac1, wgmi->wgmi_mac1, sizeof(mac1))) {
1376 WG_DLOG("mac1 is invalid\n"); 1425 WG_DLOG("mac1 is invalid\n");
1377 goto out; 1426 goto out;
1378 } 1427 }
1379 1428
1380 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_INIT))) { 1429 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_INIT))) {
1381 WG_TRACE("under load"); 1430 WG_TRACE("under load");
1382 /* 1431 /*
1383 * [W] 5.3: Denial of Service Mitigation & Cookies 1432 * [W] 5.3: Denial of Service Mitigation & Cookies
1384 * "the responder, ..., and when under load may reject messages 1433 * "the responder, ..., and when under load may reject messages
1385 * with an invalid msg.mac2. If the responder receives a 1434 * with an invalid msg.mac2. If the responder receives a
1386 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1435 * message with a valid msg.mac1 yet with an invalid msg.mac2,
1387 * and is under load, it may respond with a cookie reply 1436 * and is under load, it may respond with a cookie reply
1388 * message" 1437 * message"
1389 */ 1438 */
1390 uint8_t zero[WG_MAC_LEN] = {0}; 1439 uint8_t zero[WG_MAC_LEN] = {0};
1391 if (consttime_memequal(wgmi->wgmi_mac2, zero, sizeof(zero))) { 1440 if (consttime_memequal(wgmi->wgmi_mac2, zero, sizeof(zero))) {
1392 WG_TRACE("sending a cookie message: no cookie included"); 1441 WG_TRACE("sending a cookie message: no cookie included");
1393 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1442 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender,
1394 wgmi->wgmi_mac1, src); 1443 wgmi->wgmi_mac1, src);
1395 goto out; 1444 goto out;
1396 } 1445 }
1397 if (!wgp->wgp_last_sent_cookie_valid) { 1446 if (!wgp->wgp_last_sent_cookie_valid) {
1398 WG_TRACE("sending a cookie message: no cookie sent ever"); 1447 WG_TRACE("sending a cookie message: no cookie sent ever");
1399 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender, 1448 (void)wg_send_cookie_msg(wg, wgp, wgmi->wgmi_sender,
1400 wgmi->wgmi_mac1, src); 1449 wgmi->wgmi_mac1, src);
1401 goto out; 1450 goto out;
1402 } 1451 }
1403 uint8_t mac2[WG_MAC_LEN]; 1452 uint8_t mac2[WG_MAC_LEN];
1404 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1453 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie,
1405 WG_COOKIE_LEN, (const uint8_t *)wgmi, 1454 WG_COOKIE_LEN, (const uint8_t *)wgmi,
1406 offsetof(struct wg_msg_init, wgmi_mac2), NULL, 0); 1455 offsetof(struct wg_msg_init, wgmi_mac2), NULL, 0);
1407 if (!consttime_memequal(mac2, wgmi->wgmi_mac2, sizeof(mac2))) { 1456 if (!consttime_memequal(mac2, wgmi->wgmi_mac2, sizeof(mac2))) {
1408 WG_DLOG("mac2 is invalid\n"); 1457 WG_DLOG("mac2 is invalid\n");
1409 goto out; 1458 goto out;
1410 } 1459 }
1411 WG_TRACE("under load, but continue to sending"); 1460 WG_TRACE("under load, but continue to sending");
1412 } 1461 }
1413 1462
1414 /* [N] 2.2: "ss" */ 1463 /* [N] 2.2: "ss" */
1415 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */ 1464 /* Ci, k := KDF2(Ci, DH(Si^priv, Sr^pub)) */
1416 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey); 1465 wg_algo_dh_kdf(ckey, cipher_key, wg->wg_privkey, wgp->wgp_pubkey);
1417 1466
1418 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */ 1467 /* msg.timestamp := AEAD(k, TIMESTAMP(), Hi) */
1419 wg_timestamp_t timestamp; 1468 wg_timestamp_t timestamp;
1420 error = wg_algo_aead_dec(timestamp, sizeof(timestamp), cipher_key, 0, 1469 error = wg_algo_aead_dec(timestamp, sizeof(timestamp), cipher_key, 0,
1421 wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp), 1470 wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp),
1422 hash, sizeof(hash)); 1471 hash, sizeof(hash));
1423 if (error != 0) { 1472 if (error != 0) {
1424 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1473 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
1425 "wg_algo_aead_dec for timestamp failed\n"); 1474 "wg_algo_aead_dec for timestamp failed\n");
1426 goto out; 1475 goto out;
1427 } 1476 }
1428 /* Hi := HASH(Hi || msg.timestamp) */ 1477 /* Hi := HASH(Hi || msg.timestamp) */
1429 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp)); 1478 wg_algo_hash(hash, wgmi->wgmi_timestamp, sizeof(wgmi->wgmi_timestamp));
1430 1479
1431 /* 1480 /*
1432 * [W] 5.1 "The responder keeps track of the greatest timestamp 1481 * [W] 5.1 "The responder keeps track of the greatest timestamp
1433 * received per peer and discards packets containing 1482 * received per peer and discards packets containing
1434 * timestamps less than or equal to it." 1483 * timestamps less than or equal to it."
1435 */ 1484 */
1436 ret = memcmp(timestamp, wgp->wgp_timestamp_latest_init, 1485 ret = memcmp(timestamp, wgp->wgp_timestamp_latest_init,
1437 sizeof(timestamp)); 1486 sizeof(timestamp));
1438 if (ret <= 0) { 1487 if (ret <= 0) {
1439 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1488 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
1440 "invalid init msg: timestamp is old\n"); 1489 "invalid init msg: timestamp is old\n");
1441 goto out; 1490 goto out;
1442 } 1491 }
1443 memcpy(wgp->wgp_timestamp_latest_init, timestamp, sizeof(timestamp)); 1492 memcpy(wgp->wgp_timestamp_latest_init, timestamp, sizeof(timestamp));
1444 1493
1445 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1494 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash));
1446 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1495 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey));
1447 memcpy(wgs->wgs_ephemeral_key_peer, wgmi->wgmi_ephemeral, 1496 memcpy(wgs->wgs_ephemeral_key_peer, wgmi->wgmi_ephemeral,
1448 sizeof(wgmi->wgmi_ephemeral)); 1497 sizeof(wgmi->wgmi_ephemeral));
1449 1498
1450 wg_update_endpoint_if_necessary(wgp, src); 1499 wg_update_endpoint_if_necessary(wgp, src);
1451 1500
1452 (void)wg_send_handshake_msg_resp(wg, wgp, wgmi); 1501 (void)wg_send_handshake_msg_resp(wg, wgp, wgmi);
1453 1502
1454 wg_calculate_keys(wgs, false); 1503 wg_calculate_keys(wgs, false);
1455 wg_clear_states(wgs); 1504 wg_clear_states(wgs);
1456 1505
1457 wg_put_session(wgs, &psref_session); 1506 wg_put_session(wgs, &psref_session);
1458 wg_put_peer(wgp, &psref_peer); 1507 wg_put_peer(wgp, &psref_peer);
1459 return; 1508 return;
1460 1509
1461out: 1510out:
1462 if (reset_state_on_error) { 1511 if (reset_state_on_error) {
1463 mutex_enter(wgs->wgs_lock); 1512 mutex_enter(wgs->wgs_lock);
1464 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE); 1513 KASSERT(wgs->wgs_state == WGS_STATE_INIT_PASSIVE);
1465 wgs->wgs_state = WGS_STATE_UNKNOWN; 1514 wgs->wgs_state = WGS_STATE_UNKNOWN;
1466 mutex_exit(wgs->wgs_lock); 1515 mutex_exit(wgs->wgs_lock);
1467 } 1516 }
1468 wg_put_session(wgs, &psref_session); 1517 wg_put_session(wgs, &psref_session);
1469out_wgp: 1518out_wgp:
1470 wg_put_peer(wgp, &psref_peer); 1519 wg_put_peer(wgp, &psref_peer);
1471} 1520}
1472 1521
1473static void 1522static void
1474wg_schedule_handshake_timeout_timer(struct wg_peer *wgp) 1523wg_schedule_handshake_timeout_timer(struct wg_peer *wgp)
1475{ 1524{
1476 1525
1477 mutex_enter(wgp->wgp_lock); 1526 mutex_enter(wgp->wgp_lock);
1478 if (__predict_true(wgp->wgp_state != WGP_STATE_DESTROYING)) { 1527 if (__predict_true(wgp->wgp_state != WGP_STATE_DESTROYING)) {
1479 callout_schedule(&wgp->wgp_handshake_timeout_timer, 1528 callout_schedule(&wgp->wgp_handshake_timeout_timer,
1480 MIN(wg_rekey_timeout, INT_MAX/hz) * hz); 1529 MIN(wg_rekey_timeout, INT_MAX/hz) * hz);
1481 } 1530 }
1482 mutex_exit(wgp->wgp_lock); 1531 mutex_exit(wgp->wgp_lock);
1483} 1532}
1484 1533
1485static void 1534static void
1486wg_stop_handshake_timeout_timer(struct wg_peer *wgp) 1535wg_stop_handshake_timeout_timer(struct wg_peer *wgp)
1487{ 1536{
1488 1537
1489 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 1538 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL);
1490} 1539}
1491 1540
1492static struct socket * 1541static struct socket *
1493wg_get_so_by_af(struct wg_worker *wgw, const int af) 1542wg_get_so_by_af(struct wg_worker *wgw, const int af)
1494{ 1543{
1495 1544
1496 return (af == AF_INET) ? wgw->wgw_so4 : wgw->wgw_so6; 1545 return (af == AF_INET) ? wgw->wgw_so4 : wgw->wgw_so6;
1497} 1546}
1498 1547
1499static struct socket * 1548static struct socket *
1500wg_get_so_by_peer(struct wg_peer *wgp) 1549wg_get_so_by_peer(struct wg_peer *wgp)
1501{ 1550{
1502 1551
1503 return wg_get_so_by_af(wgp->wgp_sc->wg_worker, wgp->wgp_sa.sa_family); 1552 return wg_get_so_by_af(wgp->wgp_sc->wg_worker, wgp->wgp_sa.sa_family);
1504} 1553}
1505 1554
1506static struct wg_sockaddr * 1555static struct wg_sockaddr *
1507wg_get_endpoint_sa(struct wg_peer *wgp, struct psref *psref) 1556wg_get_endpoint_sa(struct wg_peer *wgp, struct psref *psref)
1508{ 1557{
1509 struct wg_sockaddr *wgsa; 1558 struct wg_sockaddr *wgsa;
1510 int s; 1559 int s;
1511 1560
1512 s = pserialize_read_enter(); 1561 s = pserialize_read_enter();
1513 wgsa = wgp->wgp_endpoint; 1562 wgsa = wgp->wgp_endpoint;
1514 psref_acquire(psref, &wgsa->wgsa_psref, wg_psref_class); 1563 psref_acquire(psref, &wgsa->wgsa_psref, wg_psref_class);
1515 pserialize_read_exit(s); 1564 pserialize_read_exit(s);
1516 1565
1517 return wgsa; 1566 return wgsa;
1518} 1567}
1519 1568
1520static void 1569static void
1521wg_put_sa(struct wg_peer *wgp, struct wg_sockaddr *wgsa, struct psref *psref) 1570wg_put_sa(struct wg_peer *wgp, struct wg_sockaddr *wgsa, struct psref *psref)
1522{ 1571{
1523 1572
1524 psref_release(psref, &wgsa->wgsa_psref, wg_psref_class); 1573 psref_release(psref, &wgsa->wgsa_psref, wg_psref_class);
1525} 1574}
1526 1575
1527static int 1576static int
1528wg_send_so(struct wg_peer *wgp, struct mbuf *m) 1577wg_send_so(struct wg_peer *wgp, struct mbuf *m)
1529{ 1578{
1530 int error; 1579 int error;
1531 struct socket *so; 1580 struct socket *so;
1532 struct psref psref; 1581 struct psref psref;
1533 struct wg_sockaddr *wgsa; 1582 struct wg_sockaddr *wgsa;
1534 1583
1535 so = wg_get_so_by_peer(wgp); 1584 so = wg_get_so_by_peer(wgp);
1536 wgsa = wg_get_endpoint_sa(wgp, &psref); 1585 wgsa = wg_get_endpoint_sa(wgp, &psref);
1537 error = sosend(so, wgsatosa(wgsa), NULL, m, NULL, 0, curlwp); 1586 error = sosend(so, wgsatosa(wgsa), NULL, m, NULL, 0, curlwp);
1538 wg_put_sa(wgp, wgsa, &psref); 1587 wg_put_sa(wgp, wgsa, &psref);
1539 1588
1540 return error; 1589 return error;
1541} 1590}
1542 1591
1543static int 1592static int
1544wg_send_handshake_msg_init(struct wg_softc *wg, struct wg_peer *wgp) 1593wg_send_handshake_msg_init(struct wg_softc *wg, struct wg_peer *wgp)
1545{ 1594{
1546 int error; 1595 int error;
1547 struct mbuf *m; 1596 struct mbuf *m;
1548 struct wg_msg_init *wgmi; 1597 struct wg_msg_init *wgmi;
1549 struct wg_session *wgs; 1598 struct wg_session *wgs;
1550 struct psref psref; 1599 struct psref psref;
1551 1600
1552 wgs = wg_lock_unstable_session(wgp); 1601 wgs = wg_lock_unstable_session(wgp);
1553 if (wgs->wgs_state == WGS_STATE_DESTROYING) { 1602 if (wgs->wgs_state == WGS_STATE_DESTROYING) {
1554 WG_TRACE("Session destroying"); 1603 WG_TRACE("Session destroying");
1555 mutex_exit(wgs->wgs_lock); 1604 mutex_exit(wgs->wgs_lock);
1556 /* XXX should wait? */ 1605 /* XXX should wait? */
1557 return EBUSY; 1606 return EBUSY;
1558 } 1607 }
1559 if (wgs->wgs_state == WGS_STATE_INIT_ACTIVE) { 1608 if (wgs->wgs_state == WGS_STATE_INIT_ACTIVE) {
1560 WG_TRACE("Sesssion already initializing, skip starting a new one"); 1609 WG_TRACE("Sesssion already initializing, skip starting a new one");
1561 mutex_exit(wgs->wgs_lock); 1610 mutex_exit(wgs->wgs_lock);
1562 return EBUSY; 1611 return EBUSY;
1563 } 1612 }
1564 if (wgs->wgs_state == WGS_STATE_INIT_PASSIVE) { 1613 if (wgs->wgs_state == WGS_STATE_INIT_PASSIVE) {
1565 WG_TRACE("Sesssion already initializing, destroying old states"); 1614 WG_TRACE("Sesssion already initializing, destroying old states");
1566 wg_clear_states(wgs); 1615 wg_clear_states(wgs);
1567 } 1616 }
1568 wgs->wgs_state = WGS_STATE_INIT_ACTIVE; 1617 wgs->wgs_state = WGS_STATE_INIT_ACTIVE;
1569 wg_get_session(wgs, &psref); 1618 wg_get_session(wgs, &psref);
1570 mutex_exit(wgs->wgs_lock); 1619 mutex_exit(wgs->wgs_lock);
1571 1620
1572 m = m_gethdr(M_WAIT, MT_DATA); 1621 m = m_gethdr(M_WAIT, MT_DATA);
1573 m->m_pkthdr.len = m->m_len = sizeof(*wgmi); 1622 m->m_pkthdr.len = m->m_len = sizeof(*wgmi);
1574 wgmi = mtod(m, struct wg_msg_init *); 1623 wgmi = mtod(m, struct wg_msg_init *);
1575 wg_fill_msg_init(wg, wgp, wgs, wgmi); 1624 wg_fill_msg_init(wg, wgp, wgs, wgmi);
1576 1625
1577 error = wg->wg_ops->send_hs_msg(wgp, m); 1626 error = wg->wg_ops->send_hs_msg(wgp, m);
1578 if (error == 0) { 1627 if (error == 0) {
1579 WG_TRACE("init msg sent"); 1628 WG_TRACE("init msg sent");
1580 1629
1581 if (wgp->wgp_handshake_start_time == 0) 1630 if (wgp->wgp_handshake_start_time == 0)
1582 wgp->wgp_handshake_start_time = time_uptime; 1631 wgp->wgp_handshake_start_time = time_uptime;
1583 wg_schedule_handshake_timeout_timer(wgp); 1632 wg_schedule_handshake_timeout_timer(wgp);
1584 } else { 1633 } else {
1585 mutex_enter(wgs->wgs_lock); 1634 mutex_enter(wgs->wgs_lock);
1586 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 1635 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE);
1587 wgs->wgs_state = WGS_STATE_UNKNOWN; 1636 wgs->wgs_state = WGS_STATE_UNKNOWN;
1588 mutex_exit(wgs->wgs_lock); 1637 mutex_exit(wgs->wgs_lock);
1589 } 1638 }
1590 wg_put_session(wgs, &psref); 1639 wg_put_session(wgs, &psref);
1591 1640
1592 return error; 1641 return error;
1593} 1642}
1594 1643
1595static void 1644static void
1596wg_fill_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1645wg_fill_msg_resp(struct wg_softc *wg, struct wg_peer *wgp,
1597 struct wg_msg_resp *wgmr, const struct wg_msg_init *wgmi) 1646 struct wg_msg_resp *wgmr, const struct wg_msg_init *wgmi)
1598{ 1647{
1599 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1648 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */
1600 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */ 1649 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Hr */
1601 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1650 uint8_t cipher_key[WG_KDF_OUTPUT_LEN];
1602 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN]; 1651 uint8_t pubkey[WG_EPHEMERAL_KEY_LEN];
1603 uint8_t privkey[WG_EPHEMERAL_KEY_LEN]; 1652 uint8_t privkey[WG_EPHEMERAL_KEY_LEN];
1604 struct wg_session *wgs; 1653 struct wg_session *wgs;
1605 struct psref psref; 1654 struct psref psref;
1606 1655
1607 wgs = wg_get_unstable_session(wgp, &psref); 1656 wgs = wg_get_unstable_session(wgp, &psref);
1608 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1657 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash));
1609 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1658 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey));
1610 1659
1611 wgmr->wgmr_type = WG_MSG_TYPE_RESP; 1660 wgmr->wgmr_type = WG_MSG_TYPE_RESP;
1612 wgmr->wgmr_sender = cprng_strong32(); 1661 wgmr->wgmr_sender = wg_assign_sender_index(wg, wgs);
1613 wgmr->wgmr_receiver = wgmi->wgmi_sender; 1662 wgmr->wgmr_receiver = wgmi->wgmi_sender;
1614 1663
1615 /* [W] 5.4.3 Second Message: Responder to Initiator */ 1664 /* [W] 5.4.3 Second Message: Responder to Initiator */
1616 1665
1617 /* [N] 2.2: "e" */ 1666 /* [N] 2.2: "e" */
1618 /* Er^priv, Er^pub := DH-GENERATE() */ 1667 /* Er^priv, Er^pub := DH-GENERATE() */
1619 wg_algo_generate_keypair(pubkey, privkey); 1668 wg_algo_generate_keypair(pubkey, privkey);
1620 /* Cr := KDF1(Cr, Er^pub) */ 1669 /* Cr := KDF1(Cr, Er^pub) */
1621 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey)); 1670 wg_algo_kdf(ckey, NULL, NULL, ckey, pubkey, sizeof(pubkey));
1622 /* msg.ephemeral := Er^pub */ 1671 /* msg.ephemeral := Er^pub */
1623 memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral)); 1672 memcpy(wgmr->wgmr_ephemeral, pubkey, sizeof(wgmr->wgmr_ephemeral));
1624 /* Hr := HASH(Hr || msg.ephemeral) */ 1673 /* Hr := HASH(Hr || msg.ephemeral) */
1625 wg_algo_hash(hash, pubkey, sizeof(pubkey)); 1674 wg_algo_hash(hash, pubkey, sizeof(pubkey));
1626 1675
1627 WG_DUMP_HASH("ckey", ckey); 1676 WG_DUMP_HASH("ckey", ckey);
1628 WG_DUMP_HASH("hash", hash); 1677 WG_DUMP_HASH("hash", hash);
1629 1678
1630 /* [N] 2.2: "ee" */ 1679 /* [N] 2.2: "ee" */
1631 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1680 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */
1632 wg_algo_dh_kdf(ckey, NULL, privkey, wgs->wgs_ephemeral_key_peer); 1681 wg_algo_dh_kdf(ckey, NULL, privkey, wgs->wgs_ephemeral_key_peer);
1633 1682
1634 /* [N] 2.2: "se" */ 1683 /* [N] 2.2: "se" */
1635 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1684 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */
1636 wg_algo_dh_kdf(ckey, NULL, privkey, wgp->wgp_pubkey); 1685 wg_algo_dh_kdf(ckey, NULL, privkey, wgp->wgp_pubkey);
1637 1686
1638 /* [N] 9.2: "psk" */ 1687 /* [N] 9.2: "psk" */
1639 { 1688 {
1640 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1689 uint8_t kdfout[WG_KDF_OUTPUT_LEN];
1641 /* Cr, r, k := KDF3(Cr, Q) */ 1690 /* Cr, r, k := KDF3(Cr, Q) */
1642 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1691 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk,
1643 sizeof(wgp->wgp_psk)); 1692 sizeof(wgp->wgp_psk));
1644 /* Hr := HASH(Hr || r) */ 1693 /* Hr := HASH(Hr || r) */
1645 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1694 wg_algo_hash(hash, kdfout, sizeof(kdfout));
1646 } 1695 }
1647 1696
1648 /* msg.empty := AEAD(k, 0, e, Hr) */ 1697 /* msg.empty := AEAD(k, 0, e, Hr) */
1649 wg_algo_aead_enc(wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty), 1698 wg_algo_aead_enc(wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty),
1650 cipher_key, 0, NULL, 0, hash, sizeof(hash)); 1699 cipher_key, 0, NULL, 0, hash, sizeof(hash));
1651 /* Hr := HASH(Hr || msg.empty) */ 1700 /* Hr := HASH(Hr || msg.empty) */
1652 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1701 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty));
1653 1702
1654 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1703 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty);
1655 1704
1656 /* [W] 5.4.4: Cookie MACs */ 1705 /* [W] 5.4.4: Cookie MACs */
1657 /* msg.mac1 := MAC(HASH(LABEL-MAC1 || Sm'^pub), msg_a) */ 1706 /* msg.mac1 := MAC(HASH(LABEL-MAC1 || Sm'^pub), msg_a) */
1658 wg_algo_mac_mac1(wgmr->wgmr_mac1, sizeof(wgmi->wgmi_mac1), 1707 wg_algo_mac_mac1(wgmr->wgmr_mac1, sizeof(wgmi->wgmi_mac1),
1659 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey), 1708 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey),
1660 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1709 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1));
1661 /* Need mac1 to decrypt a cookie from a cookie message */ 1710 /* Need mac1 to decrypt a cookie from a cookie message */
1662 memcpy(wgp->wgp_last_sent_mac1, wgmr->wgmr_mac1, 1711 memcpy(wgp->wgp_last_sent_mac1, wgmr->wgmr_mac1,
1663 sizeof(wgp->wgp_last_sent_mac1)); 1712 sizeof(wgp->wgp_last_sent_mac1));
1664 wgp->wgp_last_sent_mac1_valid = true; 1713 wgp->wgp_last_sent_mac1_valid = true;
1665 1714
1666 if (wgp->wgp_latest_cookie_time == 0 || 1715 if (wgp->wgp_latest_cookie_time == 0 ||
1667 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME) 1716 (time_uptime - wgp->wgp_latest_cookie_time) >= WG_COOKIE_TIME)
1668 /* msg.mac2 := 0^16 */ 1717 /* msg.mac2 := 0^16 */
1669 memset(wgmr->wgmr_mac2, 0, sizeof(wgmr->wgmr_mac2)); 1718 memset(wgmr->wgmr_mac2, 0, sizeof(wgmr->wgmr_mac2));
1670 else { 1719 else {
1671 /* msg.mac2 := MAC(Lm, msg_b) */ 1720 /* msg.mac2 := MAC(Lm, msg_b) */
1672 wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2), 1721 wg_algo_mac(wgmr->wgmr_mac2, sizeof(wgmi->wgmi_mac2),
1673 wgp->wgp_latest_cookie, WG_COOKIE_LEN, 1722 wgp->wgp_latest_cookie, WG_COOKIE_LEN,
1674 (const uint8_t *)wgmr, 1723 (const uint8_t *)wgmr,
1675 offsetof(struct wg_msg_resp, wgmr_mac2), 1724 offsetof(struct wg_msg_resp, wgmr_mac2),
1676 NULL, 0); 1725 NULL, 0);
1677 } 1726 }
1678 1727
1679 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash)); 1728 memcpy(wgs->wgs_handshake_hash, hash, sizeof(hash));
1680 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey)); 1729 memcpy(wgs->wgs_chaining_key, ckey, sizeof(ckey));
1681 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey)); 1730 memcpy(wgs->wgs_ephemeral_key_pub, pubkey, sizeof(pubkey));
1682 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey)); 1731 memcpy(wgs->wgs_ephemeral_key_priv, privkey, sizeof(privkey));
1683 wgs->wgs_sender_index = wgmr->wgmr_sender; 
1684 wgs->wgs_receiver_index = wgmi->wgmi_sender; 1732 wgs->wgs_receiver_index = wgmi->wgmi_sender;
1685 WG_DLOG("sender=%x\n", wgs->wgs_sender_index); 1733 WG_DLOG("sender=%x\n", wgs->wgs_sender_index);
1686 WG_DLOG("receiver=%x\n", wgs->wgs_receiver_index); 1734 WG_DLOG("receiver=%x\n", wgs->wgs_receiver_index);
1687 wg_put_session(wgs, &psref); 1735 wg_put_session(wgs, &psref);
1688} 1736}
1689 1737
1690static void 1738static void
1691wg_swap_sessions(struct wg_peer *wgp) 1739wg_swap_sessions(struct wg_peer *wgp)
1692{ 1740{
1693 1741
1694 KASSERT(mutex_owned(wgp->wgp_lock)); 1742 KASSERT(mutex_owned(wgp->wgp_lock));
1695 1743
1696 wgp->wgp_session_unstable = atomic_swap_ptr(&wgp->wgp_session_stable, 1744 wgp->wgp_session_unstable = atomic_swap_ptr(&wgp->wgp_session_stable,
1697 wgp->wgp_session_unstable); 1745 wgp->wgp_session_unstable);
1698 KASSERT(wgp->wgp_session_stable->wgs_state == WGS_STATE_ESTABLISHED); 1746 KASSERT(wgp->wgp_session_stable->wgs_state == WGS_STATE_ESTABLISHED);
1699} 1747}
1700 1748
1701static void 1749static void
1702wg_handle_msg_resp(struct wg_softc *wg, const struct wg_msg_resp *wgmr, 1750wg_handle_msg_resp(struct wg_softc *wg, const struct wg_msg_resp *wgmr,
1703 const struct sockaddr *src) 1751 const struct sockaddr *src)
1704{ 1752{
1705 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */ 1753 uint8_t ckey[WG_CHAINING_KEY_LEN]; /* [W] 5.4.3: Cr */
1706 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Kr */ 1754 uint8_t hash[WG_HASH_LEN]; /* [W] 5.4.3: Kr */
1707 uint8_t cipher_key[WG_KDF_OUTPUT_LEN]; 1755 uint8_t cipher_key[WG_KDF_OUTPUT_LEN];
1708 struct wg_peer *wgp; 1756 struct wg_peer *wgp;
1709 struct wg_session *wgs; 1757 struct wg_session *wgs;
1710 struct psref psref; 1758 struct psref psref;
1711 int error; 1759 int error;
1712 uint8_t mac1[WG_MAC_LEN]; 1760 uint8_t mac1[WG_MAC_LEN];
1713 struct wg_session *wgs_prev; 1761 struct wg_session *wgs_prev;
1714 1762
1715 WG_TRACE("resp msg received"); 1763 WG_TRACE("resp msg received");
1716 wgs = wg_lookup_session_by_index(wg, wgmr->wgmr_receiver, &psref); 1764 wgs = wg_lookup_session_by_index(wg, wgmr->wgmr_receiver, &psref);
1717 if (wgs == NULL) { 1765 if (wgs == NULL) {
1718 WG_TRACE("No session found"); 1766 WG_TRACE("No session found");
1719 return; 1767 return;
1720 } 1768 }
1721 1769
1722 wgp = wgs->wgs_peer; 1770 wgp = wgs->wgs_peer;
1723 1771
1724 wg_algo_mac_mac1(mac1, sizeof(mac1), 1772 wg_algo_mac_mac1(mac1, sizeof(mac1),
1725 wg->wg_pubkey, sizeof(wg->wg_pubkey), 1773 wg->wg_pubkey, sizeof(wg->wg_pubkey),
1726 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1)); 1774 (const uint8_t *)wgmr, offsetof(struct wg_msg_resp, wgmr_mac1));
1727 1775
1728 /* 1776 /*
1729 * [W] 5.3: Denial of Service Mitigation & Cookies 1777 * [W] 5.3: Denial of Service Mitigation & Cookies
1730 * "the responder, ..., must always reject messages with an invalid 1778 * "the responder, ..., must always reject messages with an invalid
1731 * msg.mac1" 1779 * msg.mac1"
1732 */ 1780 */
1733 if (!consttime_memequal(mac1, wgmr->wgmr_mac1, sizeof(mac1))) { 1781 if (!consttime_memequal(mac1, wgmr->wgmr_mac1, sizeof(mac1))) {
1734 WG_DLOG("mac1 is invalid\n"); 1782 WG_DLOG("mac1 is invalid\n");
1735 goto out; 1783 goto out;
1736 } 1784 }
1737 1785
1738 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_RESP))) { 1786 if (__predict_false(wg_is_underload(wg, wgp, WG_MSG_TYPE_RESP))) {
1739 WG_TRACE("under load"); 1787 WG_TRACE("under load");
1740 /* 1788 /*
1741 * [W] 5.3: Denial of Service Mitigation & Cookies 1789 * [W] 5.3: Denial of Service Mitigation & Cookies
1742 * "the responder, ..., and when under load may reject messages 1790 * "the responder, ..., and when under load may reject messages
1743 * with an invalid msg.mac2. If the responder receives a 1791 * with an invalid msg.mac2. If the responder receives a
1744 * message with a valid msg.mac1 yet with an invalid msg.mac2, 1792 * message with a valid msg.mac1 yet with an invalid msg.mac2,
1745 * and is under load, it may respond with a cookie reply 1793 * and is under load, it may respond with a cookie reply
1746 * message" 1794 * message"
1747 */ 1795 */
1748 uint8_t zero[WG_MAC_LEN] = {0}; 1796 uint8_t zero[WG_MAC_LEN] = {0};
1749 if (consttime_memequal(wgmr->wgmr_mac2, zero, sizeof(zero))) { 1797 if (consttime_memequal(wgmr->wgmr_mac2, zero, sizeof(zero))) {
1750 WG_TRACE("sending a cookie message: no cookie included"); 1798 WG_TRACE("sending a cookie message: no cookie included");
1751 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1799 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender,
1752 wgmr->wgmr_mac1, src); 1800 wgmr->wgmr_mac1, src);
1753 goto out; 1801 goto out;
1754 } 1802 }
1755 if (!wgp->wgp_last_sent_cookie_valid) { 1803 if (!wgp->wgp_last_sent_cookie_valid) {
1756 WG_TRACE("sending a cookie message: no cookie sent ever"); 1804 WG_TRACE("sending a cookie message: no cookie sent ever");
1757 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender, 1805 (void)wg_send_cookie_msg(wg, wgp, wgmr->wgmr_sender,
1758 wgmr->wgmr_mac1, src); 1806 wgmr->wgmr_mac1, src);
1759 goto out; 1807 goto out;
1760 } 1808 }
1761 uint8_t mac2[WG_MAC_LEN]; 1809 uint8_t mac2[WG_MAC_LEN];
1762 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie, 1810 wg_algo_mac(mac2, sizeof(mac2), wgp->wgp_last_sent_cookie,
1763 WG_COOKIE_LEN, (const uint8_t *)wgmr, 1811 WG_COOKIE_LEN, (const uint8_t *)wgmr,
1764 offsetof(struct wg_msg_resp, wgmr_mac2), NULL, 0); 1812 offsetof(struct wg_msg_resp, wgmr_mac2), NULL, 0);
1765 if (!consttime_memequal(mac2, wgmr->wgmr_mac2, sizeof(mac2))) { 1813 if (!consttime_memequal(mac2, wgmr->wgmr_mac2, sizeof(mac2))) {
1766 WG_DLOG("mac2 is invalid\n"); 1814 WG_DLOG("mac2 is invalid\n");
1767 goto out; 1815 goto out;
1768 } 1816 }
1769 WG_TRACE("under load, but continue to sending"); 1817 WG_TRACE("under load, but continue to sending");
1770 } 1818 }
1771 1819
1772 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash)); 1820 memcpy(hash, wgs->wgs_handshake_hash, sizeof(hash));
1773 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey)); 1821 memcpy(ckey, wgs->wgs_chaining_key, sizeof(ckey));
1774 1822
1775 /* 1823 /*
1776 * [W] 5.4.3 Second Message: Responder to Initiator 1824 * [W] 5.4.3 Second Message: Responder to Initiator
1777 * "When the initiator receives this message, it does the same 1825 * "When the initiator receives this message, it does the same
1778 * operations so that its final state variables are identical, 1826 * operations so that its final state variables are identical,
1779 * replacing the operands of the DH function to produce equivalent 1827 * replacing the operands of the DH function to produce equivalent
1780 * values." 1828 * values."
1781 * Note that the following comments of operations are just copies of 1829 * Note that the following comments of operations are just copies of
1782 * the initiator's ones. 1830 * the initiator's ones.
1783 */ 1831 */
1784 1832
1785 /* [N] 2.2: "e" */ 1833 /* [N] 2.2: "e" */
1786 /* Cr := KDF1(Cr, Er^pub) */ 1834 /* Cr := KDF1(Cr, Er^pub) */
1787 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmr->wgmr_ephemeral, 1835 wg_algo_kdf(ckey, NULL, NULL, ckey, wgmr->wgmr_ephemeral,
1788 sizeof(wgmr->wgmr_ephemeral)); 1836 sizeof(wgmr->wgmr_ephemeral));
1789 /* Hr := HASH(Hr || msg.ephemeral) */ 1837 /* Hr := HASH(Hr || msg.ephemeral) */
1790 wg_algo_hash(hash, wgmr->wgmr_ephemeral, sizeof(wgmr->wgmr_ephemeral)); 1838 wg_algo_hash(hash, wgmr->wgmr_ephemeral, sizeof(wgmr->wgmr_ephemeral));
1791 1839
1792 WG_DUMP_HASH("ckey", ckey); 1840 WG_DUMP_HASH("ckey", ckey);
1793 WG_DUMP_HASH("hash", hash); 1841 WG_DUMP_HASH("hash", hash);
1794 1842
1795 /* [N] 2.2: "ee" */ 1843 /* [N] 2.2: "ee" */
1796 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */ 1844 /* Cr := KDF1(Cr, DH(Er^priv, Ei^pub)) */
1797 wg_algo_dh_kdf(ckey, NULL, wgs->wgs_ephemeral_key_priv, 1845 wg_algo_dh_kdf(ckey, NULL, wgs->wgs_ephemeral_key_priv,
1798 wgmr->wgmr_ephemeral); 1846 wgmr->wgmr_ephemeral);
1799 1847
1800 /* [N] 2.2: "se" */ 1848 /* [N] 2.2: "se" */
1801 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */ 1849 /* Cr := KDF1(Cr, DH(Er^priv, Si^pub)) */
1802 wg_algo_dh_kdf(ckey, NULL, wg->wg_privkey, wgmr->wgmr_ephemeral); 1850 wg_algo_dh_kdf(ckey, NULL, wg->wg_privkey, wgmr->wgmr_ephemeral);
1803 1851
1804 /* [N] 9.2: "psk" */ 1852 /* [N] 9.2: "psk" */
1805 { 1853 {
1806 uint8_t kdfout[WG_KDF_OUTPUT_LEN]; 1854 uint8_t kdfout[WG_KDF_OUTPUT_LEN];
1807 /* Cr, r, k := KDF3(Cr, Q) */ 1855 /* Cr, r, k := KDF3(Cr, Q) */
1808 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk, 1856 wg_algo_kdf(ckey, kdfout, cipher_key, ckey, wgp->wgp_psk,
1809 sizeof(wgp->wgp_psk)); 1857 sizeof(wgp->wgp_psk));
1810 /* Hr := HASH(Hr || r) */ 1858 /* Hr := HASH(Hr || r) */
1811 wg_algo_hash(hash, kdfout, sizeof(kdfout)); 1859 wg_algo_hash(hash, kdfout, sizeof(kdfout));
1812 } 1860 }
1813 1861
1814 { 1862 {
1815 uint8_t out[sizeof(wgmr->wgmr_empty)]; /* for safety */ 1863 uint8_t out[sizeof(wgmr->wgmr_empty)]; /* for safety */
1816 /* msg.empty := AEAD(k, 0, e, Hr) */ 1864 /* msg.empty := AEAD(k, 0, e, Hr) */
1817 error = wg_algo_aead_dec(out, 0, cipher_key, 0, wgmr->wgmr_empty, 1865 error = wg_algo_aead_dec(out, 0, cipher_key, 0, wgmr->wgmr_empty,
1818 sizeof(wgmr->wgmr_empty), hash, sizeof(hash)); 1866 sizeof(wgmr->wgmr_empty), hash, sizeof(hash));
1819 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty); 1867 WG_DUMP_HASH("wgmr_empty", wgmr->wgmr_empty);
1820 if (error != 0) { 1868 if (error != 0) {
1821 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 1869 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
1822 "wg_algo_aead_dec for empty message failed\n"); 1870 "wg_algo_aead_dec for empty message failed\n");
1823 goto out; 1871 goto out;
1824 } 1872 }
1825 /* Hr := HASH(Hr || msg.empty) */ 1873 /* Hr := HASH(Hr || msg.empty) */
1826 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty)); 1874 wg_algo_hash(hash, wgmr->wgmr_empty, sizeof(wgmr->wgmr_empty));
1827 } 1875 }
1828 1876
1829 memcpy(wgs->wgs_handshake_hash, hash, sizeof(wgs->wgs_handshake_hash)); 1877 memcpy(wgs->wgs_handshake_hash, hash, sizeof(wgs->wgs_handshake_hash));
1830 memcpy(wgs->wgs_chaining_key, ckey, sizeof(wgs->wgs_chaining_key)); 1878 memcpy(wgs->wgs_chaining_key, ckey, sizeof(wgs->wgs_chaining_key));
1831 wgs->wgs_receiver_index = wgmr->wgmr_sender; 1879 wgs->wgs_receiver_index = wgmr->wgmr_sender;
1832 WG_DLOG("receiver=%x\n", wgs->wgs_receiver_index); 1880 WG_DLOG("receiver=%x\n", wgs->wgs_receiver_index);
1833 1881
1834 wgs->wgs_state = WGS_STATE_ESTABLISHED; 1882 wgs->wgs_state = WGS_STATE_ESTABLISHED;
1835 wgs->wgs_time_established = time_uptime; 1883 wgs->wgs_time_established = time_uptime;
1836 wgs->wgs_time_last_data_sent = 0; 1884 wgs->wgs_time_last_data_sent = 0;
1837 wgs->wgs_is_initiator = true; 1885 wgs->wgs_is_initiator = true;
1838 wg_calculate_keys(wgs, true); 1886 wg_calculate_keys(wgs, true);
1839 wg_clear_states(wgs); 1887 wg_clear_states(wgs);
1840 WG_TRACE("WGS_STATE_ESTABLISHED"); 1888 WG_TRACE("WGS_STATE_ESTABLISHED");
1841 1889
1842 wg_stop_handshake_timeout_timer(wgp); 1890 wg_stop_handshake_timeout_timer(wgp);
1843 1891
1844 mutex_enter(wgp->wgp_lock); 1892 mutex_enter(wgp->wgp_lock);
1845 wg_swap_sessions(wgp); 1893 wg_swap_sessions(wgp);
1846 wgs_prev = wgp->wgp_session_unstable; 1894 wgs_prev = wgp->wgp_session_unstable;
1847 mutex_enter(wgs_prev->wgs_lock); 1895 mutex_enter(wgs_prev->wgs_lock);
1848 1896
1849 getnanotime(&wgp->wgp_last_handshake_time); 1897 getnanotime(&wgp->wgp_last_handshake_time);
1850 wgp->wgp_handshake_start_time = 0; 1898 wgp->wgp_handshake_start_time = 0;
1851 wgp->wgp_last_sent_mac1_valid = false; 1899 wgp->wgp_last_sent_mac1_valid = false;
1852 wgp->wgp_last_sent_cookie_valid = false; 1900 wgp->wgp_last_sent_cookie_valid = false;
1853 mutex_exit(wgp->wgp_lock); 1901 mutex_exit(wgp->wgp_lock);
1854 1902
1855 wg_schedule_rekey_timer(wgp); 1903 wg_schedule_rekey_timer(wgp);
1856 1904
1857 wg_update_endpoint_if_necessary(wgp, src); 1905 wg_update_endpoint_if_necessary(wgp, src);
1858 1906
1859 /* 1907 /*
1860 * Send something immediately (same as the official implementation) 1908 * Send something immediately (same as the official implementation)
1861 * XXX if there are pending data packets, we don't need to send 1909 * XXX if there are pending data packets, we don't need to send
1862 * a keepalive message. 1910 * a keepalive message.
1863 */ 1911 */
1864 wg_send_keepalive_msg(wgp, wgs); 1912 wg_send_keepalive_msg(wgp, wgs);
1865 1913
1866 /* Anyway run a softint to flush pending packets */ 1914 /* Anyway run a softint to flush pending packets */
1867 kpreempt_disable(); 1915 kpreempt_disable();
1868 softint_schedule(wgp->wgp_si); 1916 softint_schedule(wgp->wgp_si);
1869 kpreempt_enable(); 1917 kpreempt_enable();
1870 WG_TRACE("softint scheduled"); 1918 WG_TRACE("softint scheduled");
1871 1919
1872 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 1920 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) {
1873 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 1921 wgs_prev->wgs_state = WGS_STATE_DESTROYING;
1874 /* We can't destroy the old session immediately */ 1922 /* We can't destroy the old session immediately */
1875 wg_schedule_session_dtor_timer(wgp); 1923 wg_schedule_session_dtor_timer(wgp);
1876 } 1924 }
1877 mutex_exit(wgs_prev->wgs_lock); 1925 mutex_exit(wgs_prev->wgs_lock);
1878 1926
1879out: 1927out:
1880 wg_put_session(wgs, &psref); 1928 wg_put_session(wgs, &psref);
1881} 1929}
1882 1930
1883static int 1931static int
1884wg_send_handshake_msg_resp(struct wg_softc *wg, struct wg_peer *wgp, 1932wg_send_handshake_msg_resp(struct wg_softc *wg, struct wg_peer *wgp,
1885 const struct wg_msg_init *wgmi) 1933 const struct wg_msg_init *wgmi)
1886{ 1934{
1887 int error; 1935 int error;
1888 struct mbuf *m; 1936 struct mbuf *m;
1889 struct wg_msg_resp *wgmr; 1937 struct wg_msg_resp *wgmr;
1890 1938
1891 m = m_gethdr(M_WAIT, MT_DATA); 1939 m = m_gethdr(M_WAIT, MT_DATA);
1892 m->m_pkthdr.len = m->m_len = sizeof(*wgmr); 1940 m->m_pkthdr.len = m->m_len = sizeof(*wgmr);
1893 wgmr = mtod(m, struct wg_msg_resp *); 1941 wgmr = mtod(m, struct wg_msg_resp *);
1894 wg_fill_msg_resp(wg, wgp, wgmr, wgmi); 1942 wg_fill_msg_resp(wg, wgp, wgmr, wgmi);
1895 1943
1896 error = wg->wg_ops->send_hs_msg(wgp, m); 1944 error = wg->wg_ops->send_hs_msg(wgp, m);
1897 if (error == 0) 1945 if (error == 0)
1898 WG_TRACE("resp msg sent"); 1946 WG_TRACE("resp msg sent");
1899 return error; 1947 return error;
1900} 1948}
1901 1949
1902static struct wg_peer * 1950static struct wg_peer *
1903wg_lookup_peer_by_pubkey(struct wg_softc *wg, 1951wg_lookup_peer_by_pubkey(struct wg_softc *wg,
1904 const uint8_t pubkey[WG_STATIC_KEY_LEN], struct psref *psref) 1952 const uint8_t pubkey[WG_STATIC_KEY_LEN], struct psref *psref)
1905{ 1953{
1906 struct wg_peer *wgp; 1954 struct wg_peer *wgp;
1907 1955
1908 int s = pserialize_read_enter(); 1956 int s = pserialize_read_enter();
1909 /* XXX O(n) */ 1957 wgp = thmap_get(wg->wg_peers_bypubkey, pubkey, WG_STATIC_KEY_LEN);
1910 WG_PEER_READER_FOREACH(wgp, wg) { 
1911 if (consttime_memequal(wgp->wgp_pubkey, pubkey, 
1912 sizeof(wgp->wgp_pubkey))) 
1913 break; 
1914 } 
1915 if (wgp != NULL) 1958 if (wgp != NULL)
1916 wg_get_peer(wgp, psref); 1959 wg_get_peer(wgp, psref);
1917 pserialize_read_exit(s); 1960 pserialize_read_exit(s);
1918 1961
1919 return wgp; 1962 return wgp;
1920} 1963}
1921 1964
1922static void 1965static void
1923wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp, 1966wg_fill_msg_cookie(struct wg_softc *wg, struct wg_peer *wgp,
1924 struct wg_msg_cookie *wgmc, const uint32_t sender, 1967 struct wg_msg_cookie *wgmc, const uint32_t sender,
1925 const uint8_t mac1[WG_MAC_LEN], const struct sockaddr *src) 1968 const uint8_t mac1[WG_MAC_LEN], const struct sockaddr *src)
1926{ 1969{
1927 uint8_t cookie[WG_COOKIE_LEN]; 1970 uint8_t cookie[WG_COOKIE_LEN];
1928 uint8_t key[WG_HASH_LEN]; 1971 uint8_t key[WG_HASH_LEN];
1929 uint8_t addr[sizeof(struct in6_addr)]; 1972 uint8_t addr[sizeof(struct in6_addr)];
1930 size_t addrlen; 1973 size_t addrlen;
1931 uint16_t uh_sport; /* be */ 1974 uint16_t uh_sport; /* be */
1932 1975
1933 wgmc->wgmc_type = WG_MSG_TYPE_COOKIE; 1976 wgmc->wgmc_type = WG_MSG_TYPE_COOKIE;
1934 wgmc->wgmc_receiver = sender; 1977 wgmc->wgmc_receiver = sender;
1935 cprng_fast(wgmc->wgmc_salt, sizeof(wgmc->wgmc_salt)); 1978 cprng_fast(wgmc->wgmc_salt, sizeof(wgmc->wgmc_salt));
1936 1979
1937 /* 1980 /*
1938 * [W] 5.4.7: Under Load: Cookie Reply Message 1981 * [W] 5.4.7: Under Load: Cookie Reply Message
1939 * "The secret variable, Rm, changes every two minutes to a 1982 * "The secret variable, Rm, changes every two minutes to a
1940 * random value" 1983 * random value"
1941 */ 1984 */
1942 if ((time_uptime - wgp->wgp_last_genrandval_time) > WG_RANDVAL_TIME) { 1985 if ((time_uptime - wgp->wgp_last_genrandval_time) > WG_RANDVAL_TIME) {
1943 wgp->wgp_randval = cprng_strong32(); 1986 wgp->wgp_randval = cprng_strong32();
1944 wgp->wgp_last_genrandval_time = time_uptime; 1987 wgp->wgp_last_genrandval_time = time_uptime;
1945 } 1988 }
1946 1989
1947 switch (src->sa_family) { 1990 switch (src->sa_family) {
1948 case AF_INET: { 1991 case AF_INET: {
1949 const struct sockaddr_in *sin = satocsin(src); 1992 const struct sockaddr_in *sin = satocsin(src);
1950 addrlen = sizeof(sin->sin_addr); 1993 addrlen = sizeof(sin->sin_addr);
1951 memcpy(addr, &sin->sin_addr, addrlen); 1994 memcpy(addr, &sin->sin_addr, addrlen);
1952 uh_sport = sin->sin_port; 1995 uh_sport = sin->sin_port;
1953 break; 1996 break;
1954 } 1997 }
1955#ifdef INET6 1998#ifdef INET6
1956 case AF_INET6: { 1999 case AF_INET6: {
1957 const struct sockaddr_in6 *sin6 = satocsin6(src); 2000 const struct sockaddr_in6 *sin6 = satocsin6(src);
1958 addrlen = sizeof(sin6->sin6_addr); 2001 addrlen = sizeof(sin6->sin6_addr);
1959 memcpy(addr, &sin6->sin6_addr, addrlen); 2002 memcpy(addr, &sin6->sin6_addr, addrlen);
1960 uh_sport = sin6->sin6_port; 2003 uh_sport = sin6->sin6_port;
1961 break; 2004 break;
1962 } 2005 }
1963#endif 2006#endif
1964 default: 2007 default:
1965 panic("invalid af=%d", wgp->wgp_sa.sa_family); 2008 panic("invalid af=%d", wgp->wgp_sa.sa_family);
1966 } 2009 }
1967 2010
1968 wg_algo_mac(cookie, sizeof(cookie), 2011 wg_algo_mac(cookie, sizeof(cookie),
1969 (const uint8_t *)&wgp->wgp_randval, sizeof(wgp->wgp_randval), 2012 (const uint8_t *)&wgp->wgp_randval, sizeof(wgp->wgp_randval),
1970 addr, addrlen, (const uint8_t *)&uh_sport, sizeof(uh_sport)); 2013 addr, addrlen, (const uint8_t *)&uh_sport, sizeof(uh_sport));
1971 wg_algo_mac_cookie(key, sizeof(key), wg->wg_pubkey, 2014 wg_algo_mac_cookie(key, sizeof(key), wg->wg_pubkey,
1972 sizeof(wg->wg_pubkey)); 2015 sizeof(wg->wg_pubkey));
1973 wg_algo_xaead_enc(wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), key, 2016 wg_algo_xaead_enc(wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), key,
1974 cookie, sizeof(cookie), mac1, WG_MAC_LEN, wgmc->wgmc_salt); 2017 cookie, sizeof(cookie), mac1, WG_MAC_LEN, wgmc->wgmc_salt);
1975 2018
1976 /* Need to store to calculate mac2 */ 2019 /* Need to store to calculate mac2 */
1977 memcpy(wgp->wgp_last_sent_cookie, cookie, sizeof(cookie)); 2020 memcpy(wgp->wgp_last_sent_cookie, cookie, sizeof(cookie));
1978 wgp->wgp_last_sent_cookie_valid = true; 2021 wgp->wgp_last_sent_cookie_valid = true;
1979} 2022}
1980 2023
1981static int 2024static int
1982wg_send_cookie_msg(struct wg_softc *wg, struct wg_peer *wgp, 2025wg_send_cookie_msg(struct wg_softc *wg, struct wg_peer *wgp,
1983 const uint32_t sender, const uint8_t mac1[WG_MAC_LEN], 2026 const uint32_t sender, const uint8_t mac1[WG_MAC_LEN],
1984 const struct sockaddr *src) 2027 const struct sockaddr *src)
1985{ 2028{
1986 int error; 2029 int error;
1987 struct mbuf *m; 2030 struct mbuf *m;
1988 struct wg_msg_cookie *wgmc; 2031 struct wg_msg_cookie *wgmc;
1989 2032
1990 m = m_gethdr(M_WAIT, MT_DATA); 2033 m = m_gethdr(M_WAIT, MT_DATA);
1991 m->m_pkthdr.len = m->m_len = sizeof(*wgmc); 2034 m->m_pkthdr.len = m->m_len = sizeof(*wgmc);
1992 wgmc = mtod(m, struct wg_msg_cookie *); 2035 wgmc = mtod(m, struct wg_msg_cookie *);
1993 wg_fill_msg_cookie(wg, wgp, wgmc, sender, mac1, src); 2036 wg_fill_msg_cookie(wg, wgp, wgmc, sender, mac1, src);
1994 2037
1995 error = wg->wg_ops->send_hs_msg(wgp, m); 2038 error = wg->wg_ops->send_hs_msg(wgp, m);
1996 if (error == 0) 2039 if (error == 0)
1997 WG_TRACE("cookie msg sent"); 2040 WG_TRACE("cookie msg sent");
1998 return error; 2041 return error;
1999} 2042}
2000 2043
2001static bool 2044static bool
2002wg_is_underload(struct wg_softc *wg, struct wg_peer *wgp, int msgtype) 2045wg_is_underload(struct wg_softc *wg, struct wg_peer *wgp, int msgtype)
2003{ 2046{
2004#ifdef WG_DEBUG_PARAMS 2047#ifdef WG_DEBUG_PARAMS
2005 if (wg_force_underload) 2048 if (wg_force_underload)
2006 return true; 2049 return true;
2007#endif 2050#endif
2008 2051
2009 /* 2052 /*
2010 * XXX we don't have a means of a load estimation. The purpose of 2053 * XXX we don't have a means of a load estimation. The purpose of
2011 * the mechanism is a DoS mitigation, so we consider frequent handshake 2054 * the mechanism is a DoS mitigation, so we consider frequent handshake
2012 * messages as (a kind of) load; if a message of the same type comes 2055 * messages as (a kind of) load; if a message of the same type comes
2013 * to a peer within 1 second, we consider we are under load. 2056 * to a peer within 1 second, we consider we are under load.
2014 */ 2057 */
2015 time_t last = wgp->wgp_last_msg_received_time[msgtype]; 2058 time_t last = wgp->wgp_last_msg_received_time[msgtype];
2016 wgp->wgp_last_msg_received_time[msgtype] = time_uptime; 2059 wgp->wgp_last_msg_received_time[msgtype] = time_uptime;
2017 return (time_uptime - last) == 0; 2060 return (time_uptime - last) == 0;
2018} 2061}
2019 2062
2020static void 2063static void
2021wg_calculate_keys(struct wg_session *wgs, const bool initiator) 2064wg_calculate_keys(struct wg_session *wgs, const bool initiator)
2022{ 2065{
2023 2066
2024 /* 2067 /*
2025 * [W] 5.4.5: Ti^send = Tr^recv, Ti^recv = Tr^send := KDF2(Ci = Cr, e) 2068 * [W] 5.4.5: Ti^send = Tr^recv, Ti^recv = Tr^send := KDF2(Ci = Cr, e)
2026 */ 2069 */
2027 if (initiator) { 2070 if (initiator) {
2028 wg_algo_kdf(wgs->wgs_tkey_send, wgs->wgs_tkey_recv, NULL, 2071 wg_algo_kdf(wgs->wgs_tkey_send, wgs->wgs_tkey_recv, NULL,
2029 wgs->wgs_chaining_key, NULL, 0); 2072 wgs->wgs_chaining_key, NULL, 0);
2030 } else { 2073 } else {
2031 wg_algo_kdf(wgs->wgs_tkey_recv, wgs->wgs_tkey_send, NULL, 2074 wg_algo_kdf(wgs->wgs_tkey_recv, wgs->wgs_tkey_send, NULL,
2032 wgs->wgs_chaining_key, NULL, 0); 2075 wgs->wgs_chaining_key, NULL, 0);
2033 } 2076 }
2034 WG_DUMP_HASH("wgs_tkey_send", wgs->wgs_tkey_send); 2077 WG_DUMP_HASH("wgs_tkey_send", wgs->wgs_tkey_send);
2035 WG_DUMP_HASH("wgs_tkey_recv", wgs->wgs_tkey_recv); 2078 WG_DUMP_HASH("wgs_tkey_recv", wgs->wgs_tkey_recv);
2036} 2079}
2037 2080
2038static uint64_t 2081static uint64_t
2039wg_session_get_send_counter(struct wg_session *wgs) 2082wg_session_get_send_counter(struct wg_session *wgs)
2040{ 2083{
2041#ifdef __HAVE_ATOMIC64_LOADSTORE 2084#ifdef __HAVE_ATOMIC64_LOADSTORE
2042 return atomic_load_relaxed(&wgs->wgs_send_counter); 2085 return atomic_load_relaxed(&wgs->wgs_send_counter);
2043#else 2086#else
2044 uint64_t send_counter; 2087 uint64_t send_counter;
2045 2088
2046 mutex_enter(&wgs->wgs_send_counter_lock); 2089 mutex_enter(&wgs->wgs_send_counter_lock);
2047 send_counter = wgs->wgs_send_counter; 2090 send_counter = wgs->wgs_send_counter;
2048 mutex_exit(&wgs->wgs_send_counter_lock); 2091 mutex_exit(&wgs->wgs_send_counter_lock);
2049 2092
2050 return send_counter; 2093 return send_counter;
2051#endif 2094#endif
2052} 2095}
2053 2096
2054static uint64_t 2097static uint64_t
2055wg_session_inc_send_counter(struct wg_session *wgs) 2098wg_session_inc_send_counter(struct wg_session *wgs)
2056{ 2099{
2057#ifdef __HAVE_ATOMIC64_LOADSTORE 2100#ifdef __HAVE_ATOMIC64_LOADSTORE
2058 return atomic_inc_64_nv(&wgs->wgs_send_counter) - 1; 2101 return atomic_inc_64_nv(&wgs->wgs_send_counter) - 1;
2059#else 2102#else
2060 uint64_t send_counter; 2103 uint64_t send_counter;
2061 2104
2062 mutex_enter(&wgs->wgs_send_counter_lock); 2105 mutex_enter(&wgs->wgs_send_counter_lock);
2063 send_counter = wgs->wgs_send_counter++; 2106 send_counter = wgs->wgs_send_counter++;
2064 mutex_exit(&wgs->wgs_send_counter_lock); 2107 mutex_exit(&wgs->wgs_send_counter_lock);
2065 2108
2066 return send_counter; 2109 return send_counter;
2067#endif 2110#endif
2068} 2111}
2069 2112
2070static void 2113static void
2071wg_clear_states(struct wg_session *wgs) 2114wg_clear_states(struct wg_session *wgs)
2072{ 2115{
2073 2116
2074 wgs->wgs_send_counter = 0; 2117 wgs->wgs_send_counter = 0;
2075 sliwin_reset(&wgs->wgs_recvwin->window); 2118 sliwin_reset(&wgs->wgs_recvwin->window);
2076 2119
2077#define wgs_clear(v) explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v)) 2120#define wgs_clear(v) explicit_memset(wgs->wgs_##v, 0, sizeof(wgs->wgs_##v))
2078 wgs_clear(handshake_hash); 2121 wgs_clear(handshake_hash);
2079 wgs_clear(chaining_key); 2122 wgs_clear(chaining_key);
2080 wgs_clear(ephemeral_key_pub); 2123 wgs_clear(ephemeral_key_pub);
2081 wgs_clear(ephemeral_key_priv); 2124 wgs_clear(ephemeral_key_priv);
2082 wgs_clear(ephemeral_key_peer); 2125 wgs_clear(ephemeral_key_peer);
2083#undef wgs_clear 2126#undef wgs_clear
2084} 2127}
2085 2128
2086static struct wg_session * 2129static struct wg_session *
2087wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index, 2130wg_lookup_session_by_index(struct wg_softc *wg, const uint32_t index,
2088 struct psref *psref) 2131 struct psref *psref)
2089{ 2132{
2090 struct wg_peer *wgp; 
2091 struct wg_session *wgs; 2133 struct wg_session *wgs;
2092 2134
2093 int s = pserialize_read_enter(); 2135 int s = pserialize_read_enter();
2094 /* XXX O(n) */ 2136 wgs = thmap_get(wg->wg_sessions_byindex, &index, sizeof index);
2095 WG_PEER_READER_FOREACH(wgp, wg) { 
2096 wgs = wgp->wgp_session_stable; 
2097 WG_DLOG("index=%x wgs_sender_index=%x\n", 
2098 index, wgs->wgs_sender_index); 
2099 if (wgs->wgs_sender_index == index) 
2100 break; 
2101 wgs = wgp->wgp_session_unstable; 
2102 WG_DLOG("index=%x wgs_sender_index=%x\n", 
2103 index, wgs->wgs_sender_index); 
2104 if (wgs->wgs_sender_index == index) 
2105 break; 
2106 wgs = NULL; 
2107 } 
2108 if (wgs != NULL) 2137 if (wgs != NULL)
2109 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class); 2138 psref_acquire(psref, &wgs->wgs_psref, wg_psref_class);
2110 pserialize_read_exit(s); 2139 pserialize_read_exit(s);
2111 2140
2112 return wgs; 2141 return wgs;
2113} 2142}
2114 2143
2115static void 2144static void
2116wg_schedule_rekey_timer(struct wg_peer *wgp) 2145wg_schedule_rekey_timer(struct wg_peer *wgp)
2117{ 2146{
2118 int timeout = MIN(wg_rekey_after_time, INT_MAX/hz); 2147 int timeout = MIN(wg_rekey_after_time, INT_MAX/hz);
2119 2148
2120 callout_schedule(&wgp->wgp_rekey_timer, timeout * hz); 2149 callout_schedule(&wgp->wgp_rekey_timer, timeout * hz);
2121} 2150}
2122 2151
2123static void 2152static void
2124wg_send_keepalive_msg(struct wg_peer *wgp, struct wg_session *wgs) 2153wg_send_keepalive_msg(struct wg_peer *wgp, struct wg_session *wgs)
2125{ 2154{
2126 struct mbuf *m; 2155 struct mbuf *m;
2127 2156
2128 /* 2157 /*
2129 * [W] 6.5 Passive Keepalive 2158 * [W] 6.5 Passive Keepalive
2130 * "A keepalive message is simply a transport data message with 2159 * "A keepalive message is simply a transport data message with
2131 * a zero-length encapsulated encrypted inner-packet." 2160 * a zero-length encapsulated encrypted inner-packet."
2132 */ 2161 */
2133 m = m_gethdr(M_WAIT, MT_DATA); 2162 m = m_gethdr(M_WAIT, MT_DATA);
2134 wg_send_data_msg(wgp, wgs, m); 2163 wg_send_data_msg(wgp, wgs, m);
2135} 2164}
2136 2165
2137static bool 2166static bool
2138wg_need_to_send_init_message(struct wg_session *wgs) 2167wg_need_to_send_init_message(struct wg_session *wgs)
2139{ 2168{
2140 /* 2169 /*
2141 * [W] 6.2 Transport Message Limits 2170 * [W] 6.2 Transport Message Limits
2142 * "if a peer is the initiator of a current secure session, 2171 * "if a peer is the initiator of a current secure session,
2143 * WireGuard will send a handshake initiation message to begin 2172 * WireGuard will send a handshake initiation message to begin
2144 * a new secure session ... if after receiving a transport data 2173 * a new secure session ... if after receiving a transport data
2145 * message, the current secure session is (REJECT-AFTER-TIME − 2174 * message, the current secure session is (REJECT-AFTER-TIME −
2146 * KEEPALIVE-TIMEOUT − REKEY-TIMEOUT) seconds old and it has 2175 * KEEPALIVE-TIMEOUT − REKEY-TIMEOUT) seconds old and it has
2147 * not yet acted upon this event." 2176 * not yet acted upon this event."
2148 */ 2177 */
2149 return wgs->wgs_is_initiator && wgs->wgs_time_last_data_sent == 0 && 2178 return wgs->wgs_is_initiator && wgs->wgs_time_last_data_sent == 0 &&
2150 (time_uptime - wgs->wgs_time_established) >= 2179 (time_uptime - wgs->wgs_time_established) >=
2151 (wg_reject_after_time - wg_keepalive_timeout - wg_rekey_timeout); 2180 (wg_reject_after_time - wg_keepalive_timeout - wg_rekey_timeout);
2152} 2181}
2153 2182
2154static void 2183static void
2155wg_schedule_peer_task(struct wg_peer *wgp, int task) 2184wg_schedule_peer_task(struct wg_peer *wgp, int task)
2156{ 2185{
2157 2186
2158 atomic_or_uint(&wgp->wgp_tasks, task); 2187 atomic_or_uint(&wgp->wgp_tasks, task);
2159 WG_DLOG("tasks=%d, task=%d\n", wgp->wgp_tasks, task); 2188 WG_DLOG("tasks=%d, task=%d\n", wgp->wgp_tasks, task);
2160 wg_wakeup_worker(wgp->wgp_sc->wg_worker, WG_WAKEUP_REASON_PEER); 2189 wg_wakeup_worker(wgp->wgp_sc->wg_worker, WG_WAKEUP_REASON_PEER);
2161} 2190}
2162 2191
2163static void 2192static void
2164wg_change_endpoint(struct wg_peer *wgp, const struct sockaddr *new) 2193wg_change_endpoint(struct wg_peer *wgp, const struct sockaddr *new)
2165{ 2194{
2166 2195
2167 KASSERT(mutex_owned(wgp->wgp_lock)); 2196 KASSERT(mutex_owned(wgp->wgp_lock));
2168 2197
2169 WG_TRACE("Changing endpoint"); 2198 WG_TRACE("Changing endpoint");
2170 2199
2171 memcpy(wgp->wgp_endpoint0, new, new->sa_len); 2200 memcpy(wgp->wgp_endpoint0, new, new->sa_len);
2172 wgp->wgp_endpoint0 = atomic_swap_ptr(&wgp->wgp_endpoint, 2201 wgp->wgp_endpoint0 = atomic_swap_ptr(&wgp->wgp_endpoint,
2173 wgp->wgp_endpoint0); 2202 wgp->wgp_endpoint0);
2174 if (!wgp->wgp_endpoint_available) 2203 if (!wgp->wgp_endpoint_available)
2175 wgp->wgp_endpoint_available = true; 2204 wgp->wgp_endpoint_available = true;
2176 wgp->wgp_endpoint_changing = true; 2205 wgp->wgp_endpoint_changing = true;
2177 wg_schedule_peer_task(wgp, WGP_TASK_ENDPOINT_CHANGED); 2206 wg_schedule_peer_task(wgp, WGP_TASK_ENDPOINT_CHANGED);
2178} 2207}
2179 2208
2180static bool 2209static bool
2181wg_validate_inner_packet(const char *packet, size_t decrypted_len, int *af) 2210wg_validate_inner_packet(const char *packet, size_t decrypted_len, int *af)
2182{ 2211{
2183 uint16_t packet_len; 2212 uint16_t packet_len;
2184 const struct ip *ip; 2213 const struct ip *ip;
2185 2214
2186 if (__predict_false(decrypted_len < sizeof(struct ip))) 2215 if (__predict_false(decrypted_len < sizeof(struct ip)))
2187 return false; 2216 return false;
2188 2217
2189 ip = (const struct ip *)packet; 2218 ip = (const struct ip *)packet;
2190 if (ip->ip_v == 4) 2219 if (ip->ip_v == 4)
2191 *af = AF_INET; 2220 *af = AF_INET;
2192 else if (ip->ip_v == 6) 2221 else if (ip->ip_v == 6)
2193 *af = AF_INET6; 2222 *af = AF_INET6;
2194 else 2223 else
2195 return false; 2224 return false;
2196 2225
2197 WG_DLOG("af=%d\n", *af); 2226 WG_DLOG("af=%d\n", *af);
2198 2227
2199 if (*af == AF_INET) { 2228 if (*af == AF_INET) {
2200 packet_len = ntohs(ip->ip_len); 2229 packet_len = ntohs(ip->ip_len);
2201 } else { 2230 } else {
2202 const struct ip6_hdr *ip6; 2231 const struct ip6_hdr *ip6;
2203 2232
2204 if (__predict_false(decrypted_len < sizeof(struct ip6_hdr))) 2233 if (__predict_false(decrypted_len < sizeof(struct ip6_hdr)))
2205 return false; 2234 return false;
2206 2235
2207 ip6 = (const struct ip6_hdr *)packet; 2236 ip6 = (const struct ip6_hdr *)packet;
2208 packet_len = sizeof(struct ip6_hdr) + ntohs(ip6->ip6_plen); 2237 packet_len = sizeof(struct ip6_hdr) + ntohs(ip6->ip6_plen);
2209 } 2238 }
2210 2239
2211 WG_DLOG("packet_len=%u\n", packet_len); 2240 WG_DLOG("packet_len=%u\n", packet_len);
2212 if (packet_len > decrypted_len) 2241 if (packet_len > decrypted_len)
2213 return false; 2242 return false;
2214 2243
2215 return true; 2244 return true;
2216} 2245}
2217 2246
2218static bool 2247static bool
2219wg_validate_route(struct wg_softc *wg, struct wg_peer *wgp_expected, 2248wg_validate_route(struct wg_softc *wg, struct wg_peer *wgp_expected,
2220 int af, char *packet) 2249 int af, char *packet)
2221{ 2250{
2222 struct sockaddr_storage ss; 2251 struct sockaddr_storage ss;
2223 struct sockaddr *sa; 2252 struct sockaddr *sa;
2224 struct psref psref; 2253 struct psref psref;
2225 struct wg_peer *wgp; 2254 struct wg_peer *wgp;
2226 bool ok; 2255 bool ok;
2227 2256
2228 /* 2257 /*
2229 * II CRYPTOKEY ROUTING 2258 * II CRYPTOKEY ROUTING
2230 * "it will only accept it if its source IP resolves in the 2259 * "it will only accept it if its source IP resolves in the
2231 * table to the public key used in the secure session for 2260 * table to the public key used in the secure session for
2232 * decrypting it." 2261 * decrypting it."
2233 */ 2262 */
2234 2263
2235 if (af == AF_INET) { 2264 if (af == AF_INET) {
2236 const struct ip *ip = (const struct ip *)packet; 2265 const struct ip *ip = (const struct ip *)packet;
2237 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 2266 struct sockaddr_in *sin = (struct sockaddr_in *)&ss;
2238 sockaddr_in_init(sin, &ip->ip_src, 0); 2267 sockaddr_in_init(sin, &ip->ip_src, 0);
2239 sa = sintosa(sin); 2268 sa = sintosa(sin);
2240#ifdef INET6 2269#ifdef INET6
2241 } else { 2270 } else {
2242 const struct ip6_hdr *ip6 = (const struct ip6_hdr *)packet; 2271 const struct ip6_hdr *ip6 = (const struct ip6_hdr *)packet;
2243 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 2272 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss;
2244 sockaddr_in6_init(sin6, &ip6->ip6_src, 0, 0, 0); 2273 sockaddr_in6_init(sin6, &ip6->ip6_src, 0, 0, 0);
2245 sa = sin6tosa(sin6); 2274 sa = sin6tosa(sin6);
2246#endif 2275#endif
2247 } 2276 }
2248 2277
2249 wgp = wg_pick_peer_by_sa(wg, sa, &psref); 2278 wgp = wg_pick_peer_by_sa(wg, sa, &psref);
2250 ok = (wgp == wgp_expected); 2279 ok = (wgp == wgp_expected);
2251 if (wgp != NULL) 2280 if (wgp != NULL)
2252 wg_put_peer(wgp, &psref); 2281 wg_put_peer(wgp, &psref);
2253 2282
2254 return ok; 2283 return ok;
2255} 2284}
2256 2285
2257static void 2286static void
2258wg_session_dtor_timer(void *arg) 2287wg_session_dtor_timer(void *arg)
2259{ 2288{
2260 struct wg_peer *wgp = arg; 2289 struct wg_peer *wgp = arg;
2261 2290
2262 WG_TRACE("enter"); 2291 WG_TRACE("enter");
2263 2292
2264 mutex_enter(wgp->wgp_lock); 2293 mutex_enter(wgp->wgp_lock);
2265 if (__predict_false(wgp->wgp_state == WGP_STATE_DESTROYING)) { 2294 if (__predict_false(wgp->wgp_state == WGP_STATE_DESTROYING)) {
2266 mutex_exit(wgp->wgp_lock); 2295 mutex_exit(wgp->wgp_lock);
2267 return; 2296 return;
2268 } 2297 }
2269 mutex_exit(wgp->wgp_lock); 2298 mutex_exit(wgp->wgp_lock);
2270 2299
2271 wg_schedule_peer_task(wgp, WGP_TASK_DESTROY_PREV_SESSION); 2300 wg_schedule_peer_task(wgp, WGP_TASK_DESTROY_PREV_SESSION);
2272} 2301}
2273 2302
2274static void 2303static void
2275wg_schedule_session_dtor_timer(struct wg_peer *wgp) 2304wg_schedule_session_dtor_timer(struct wg_peer *wgp)
2276{ 2305{
2277 2306
2278 /* 1 second grace period */ 2307 /* 1 second grace period */
2279 callout_schedule(&wgp->wgp_session_dtor_timer, hz); 2308 callout_schedule(&wgp->wgp_session_dtor_timer, hz);
2280} 2309}
2281 2310
2282static void 2311static void
2283wg_stop_session_dtor_timer(struct wg_peer *wgp) 2312wg_stop_session_dtor_timer(struct wg_peer *wgp)
2284{ 2313{
2285 2314
2286 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 2315 callout_halt(&wgp->wgp_session_dtor_timer, NULL);
2287} 2316}
2288 2317
2289static bool 2318static bool
2290sockaddr_port_match(const struct sockaddr *sa1, const struct sockaddr *sa2) 2319sockaddr_port_match(const struct sockaddr *sa1, const struct sockaddr *sa2)
2291{ 2320{
2292 if (sa1->sa_family != sa2->sa_family) 2321 if (sa1->sa_family != sa2->sa_family)
2293 return false; 2322 return false;
2294 2323
2295 switch (sa1->sa_family) { 2324 switch (sa1->sa_family) {
2296 case AF_INET: 2325 case AF_INET:
2297 return satocsin(sa1)->sin_port == satocsin(sa2)->sin_port; 2326 return satocsin(sa1)->sin_port == satocsin(sa2)->sin_port;
2298 case AF_INET6: 2327 case AF_INET6:
2299 return satocsin6(sa1)->sin6_port == satocsin6(sa2)->sin6_port; 2328 return satocsin6(sa1)->sin6_port == satocsin6(sa2)->sin6_port;
2300 default: 2329 default:
2301 return true; 2330 return true;
2302 } 2331 }
2303} 2332}
2304 2333
2305static void 2334static void
2306wg_update_endpoint_if_necessary(struct wg_peer *wgp, 2335wg_update_endpoint_if_necessary(struct wg_peer *wgp,
2307 const struct sockaddr *src) 2336 const struct sockaddr *src)
2308{ 2337{
2309 2338
2310#ifdef WG_DEBUG_LOG 2339#ifdef WG_DEBUG_LOG
2311 char oldaddr[128], newaddr[128]; 2340 char oldaddr[128], newaddr[128];
2312 sockaddr_format(&wgp->wgp_sa, oldaddr, sizeof(oldaddr)); 2341 sockaddr_format(&wgp->wgp_sa, oldaddr, sizeof(oldaddr));
2313 sockaddr_format(src, newaddr, sizeof(newaddr)); 2342 sockaddr_format(src, newaddr, sizeof(newaddr));
2314 WG_DLOG("old=%s, new=%s\n", oldaddr, newaddr); 2343 WG_DLOG("old=%s, new=%s\n", oldaddr, newaddr);
2315#endif 2344#endif
2316 2345
2317 /* 2346 /*
2318 * III: "Since the packet has authenticated correctly, the source IP of 2347 * III: "Since the packet has authenticated correctly, the source IP of
2319 * the outer UDP/IP packet is used to update the endpoint for peer..." 2348 * the outer UDP/IP packet is used to update the endpoint for peer..."
2320 */ 2349 */
2321 if (__predict_false(sockaddr_cmp(src, &wgp->wgp_sa) != 0 || 2350 if (__predict_false(sockaddr_cmp(src, &wgp->wgp_sa) != 0 ||
2322 !sockaddr_port_match(src, &wgp->wgp_sa))) { 2351 !sockaddr_port_match(src, &wgp->wgp_sa))) {
2323 mutex_enter(wgp->wgp_lock); 2352 mutex_enter(wgp->wgp_lock);
2324 /* XXX We can't change the endpoint twice in a short period */ 2353 /* XXX We can't change the endpoint twice in a short period */
2325 if (!wgp->wgp_endpoint_changing) { 2354 if (!wgp->wgp_endpoint_changing) {
2326 wg_change_endpoint(wgp, src); 2355 wg_change_endpoint(wgp, src);
2327 } 2356 }
2328 mutex_exit(wgp->wgp_lock); 2357 mutex_exit(wgp->wgp_lock);
2329 } 2358 }
2330} 2359}
2331 2360
2332static void 2361static void
2333wg_handle_msg_data(struct wg_softc *wg, struct mbuf *m, 2362wg_handle_msg_data(struct wg_softc *wg, struct mbuf *m,
2334 const struct sockaddr *src) 2363 const struct sockaddr *src)
2335{ 2364{
2336 struct wg_msg_data *wgmd; 2365 struct wg_msg_data *wgmd;
2337 char *encrypted_buf = NULL, *decrypted_buf; 2366 char *encrypted_buf = NULL, *decrypted_buf;
2338 size_t encrypted_len, decrypted_len; 2367 size_t encrypted_len, decrypted_len;
2339 struct wg_session *wgs; 2368 struct wg_session *wgs;
2340 struct wg_peer *wgp; 2369 struct wg_peer *wgp;
2341 size_t mlen; 2370 size_t mlen;
2342 struct psref psref; 2371 struct psref psref;
2343 int error, af; 2372 int error, af;
2344 bool success, free_encrypted_buf = false, ok; 2373 bool success, free_encrypted_buf = false, ok;
2345 struct mbuf *n; 2374 struct mbuf *n;
2346 2375
2347 KASSERT(m->m_len >= sizeof(struct wg_msg_data)); 2376 KASSERT(m->m_len >= sizeof(struct wg_msg_data));
2348 wgmd = mtod(m, struct wg_msg_data *); 2377 wgmd = mtod(m, struct wg_msg_data *);
2349 2378
2350 KASSERT(wgmd->wgmd_type == WG_MSG_TYPE_DATA); 2379 KASSERT(wgmd->wgmd_type == WG_MSG_TYPE_DATA);
2351 WG_TRACE("data"); 2380 WG_TRACE("data");
2352 2381
2353 wgs = wg_lookup_session_by_index(wg, wgmd->wgmd_receiver, &psref); 2382 wgs = wg_lookup_session_by_index(wg, wgmd->wgmd_receiver, &psref);
2354 if (wgs == NULL) { 2383 if (wgs == NULL) {
2355 WG_TRACE("No session found"); 2384 WG_TRACE("No session found");
2356 m_freem(m); 2385 m_freem(m);
2357 return; 2386 return;
2358 } 2387 }
2359 wgp = wgs->wgs_peer; 2388 wgp = wgs->wgs_peer;
2360 2389
2361 error = sliwin_check_fast(&wgs->wgs_recvwin->window, 2390 error = sliwin_check_fast(&wgs->wgs_recvwin->window,
2362 wgmd->wgmd_counter); 2391 wgmd->wgmd_counter);
2363 if (error) { 2392 if (error) {
2364 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2393 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
2365 "out-of-window packet: %"PRIu64"\n", 2394 "out-of-window packet: %"PRIu64"\n",
2366 wgmd->wgmd_counter); 2395 wgmd->wgmd_counter);
2367 goto out; 2396 goto out;
2368 } 2397 }
2369 2398
2370 mlen = m_length(m); 2399 mlen = m_length(m);
2371 encrypted_len = mlen - sizeof(*wgmd); 2400 encrypted_len = mlen - sizeof(*wgmd);
2372 2401
2373 if (encrypted_len < WG_AUTHTAG_LEN) { 2402 if (encrypted_len < WG_AUTHTAG_LEN) {
2374 WG_DLOG("Short encrypted_len: %lu\n", encrypted_len); 2403 WG_DLOG("Short encrypted_len: %lu\n", encrypted_len);
2375 goto out; 2404 goto out;
2376 } 2405 }
2377 2406
2378 success = m_ensure_contig(&m, sizeof(*wgmd) + encrypted_len); 2407 success = m_ensure_contig(&m, sizeof(*wgmd) + encrypted_len);
2379 if (success) { 2408 if (success) {
2380 encrypted_buf = mtod(m, char *) + sizeof(*wgmd); 2409 encrypted_buf = mtod(m, char *) + sizeof(*wgmd);
2381 } else { 2410 } else {
2382 encrypted_buf = kmem_intr_alloc(encrypted_len, KM_NOSLEEP); 2411 encrypted_buf = kmem_intr_alloc(encrypted_len, KM_NOSLEEP);
2383 if (encrypted_buf == NULL) { 2412 if (encrypted_buf == NULL) {
2384 WG_DLOG("failed to allocate encrypted_buf\n"); 2413 WG_DLOG("failed to allocate encrypted_buf\n");
2385 goto out; 2414 goto out;
2386 } 2415 }
2387 m_copydata(m, sizeof(*wgmd), encrypted_len, encrypted_buf); 2416 m_copydata(m, sizeof(*wgmd), encrypted_len, encrypted_buf);
2388 free_encrypted_buf = true; 2417 free_encrypted_buf = true;
2389 } 2418 }
2390 /* m_ensure_contig may change m regardless of its result */ 2419 /* m_ensure_contig may change m regardless of its result */
2391 KASSERT(m->m_len >= sizeof(*wgmd)); 2420 KASSERT(m->m_len >= sizeof(*wgmd));
2392 wgmd = mtod(m, struct wg_msg_data *); 2421 wgmd = mtod(m, struct wg_msg_data *);
2393 2422
2394 decrypted_len = encrypted_len - WG_AUTHTAG_LEN; 2423 decrypted_len = encrypted_len - WG_AUTHTAG_LEN;
2395 if (decrypted_len > MCLBYTES) { 2424 if (decrypted_len > MCLBYTES) {
2396 /* FIXME handle larger data than MCLBYTES */ 2425 /* FIXME handle larger data than MCLBYTES */
2397 WG_DLOG("couldn't handle larger data than MCLBYTES\n"); 2426 WG_DLOG("couldn't handle larger data than MCLBYTES\n");
2398 goto out; 2427 goto out;
2399 } 2428 }
2400 2429
2401 /* To avoid zero length */ 2430 /* To avoid zero length */
2402 n = wg_get_mbuf(0, decrypted_len + WG_AUTHTAG_LEN); 2431 n = wg_get_mbuf(0, decrypted_len + WG_AUTHTAG_LEN);
2403 if (n == NULL) { 2432 if (n == NULL) {
2404 WG_DLOG("wg_get_mbuf failed\n"); 2433 WG_DLOG("wg_get_mbuf failed\n");
2405 goto out; 2434 goto out;
2406 } 2435 }
2407 decrypted_buf = mtod(n, char *); 2436 decrypted_buf = mtod(n, char *);
2408 2437
2409 WG_DLOG("mlen=%lu, encrypted_len=%lu\n", mlen, encrypted_len); 2438 WG_DLOG("mlen=%lu, encrypted_len=%lu\n", mlen, encrypted_len);
2410 error = wg_algo_aead_dec(decrypted_buf, 2439 error = wg_algo_aead_dec(decrypted_buf,
2411 encrypted_len - WG_AUTHTAG_LEN /* can be 0 */, 2440 encrypted_len - WG_AUTHTAG_LEN /* can be 0 */,
2412 wgs->wgs_tkey_recv, wgmd->wgmd_counter, encrypted_buf, 2441 wgs->wgs_tkey_recv, wgmd->wgmd_counter, encrypted_buf,
2413 encrypted_len, NULL, 0); 2442 encrypted_len, NULL, 0);
2414 if (error != 0) { 2443 if (error != 0) {
2415 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2444 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
2416 "failed to wg_algo_aead_dec\n"); 2445 "failed to wg_algo_aead_dec\n");
2417 m_freem(n); 2446 m_freem(n);
2418 goto out; 2447 goto out;
2419 } 2448 }
2420 WG_DLOG("outsize=%u\n", (u_int)decrypted_len); 2449 WG_DLOG("outsize=%u\n", (u_int)decrypted_len);
2421 2450
2422 mutex_enter(&wgs->wgs_recvwin->lock); 2451 mutex_enter(&wgs->wgs_recvwin->lock);
2423 error = sliwin_update(&wgs->wgs_recvwin->window, 2452 error = sliwin_update(&wgs->wgs_recvwin->window,
2424 wgmd->wgmd_counter); 2453 wgmd->wgmd_counter);
2425 mutex_exit(&wgs->wgs_recvwin->lock); 2454 mutex_exit(&wgs->wgs_recvwin->lock);
2426 if (error) { 2455 if (error) {
2427 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2456 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
2428 "replay or out-of-window packet: %"PRIu64"\n", 2457 "replay or out-of-window packet: %"PRIu64"\n",
2429 wgmd->wgmd_counter); 2458 wgmd->wgmd_counter);
2430 m_freem(n); 2459 m_freem(n);
2431 goto out; 2460 goto out;
2432 } 2461 }
2433 2462
2434 m_freem(m); 2463 m_freem(m);
2435 m = NULL; 2464 m = NULL;
2436 wgmd = NULL; 2465 wgmd = NULL;
2437 2466
2438 ok = wg_validate_inner_packet(decrypted_buf, decrypted_len, &af); 2467 ok = wg_validate_inner_packet(decrypted_buf, decrypted_len, &af);
2439 if (!ok) { 2468 if (!ok) {
2440 /* something wrong... */ 2469 /* something wrong... */
2441 m_freem(n); 2470 m_freem(n);
2442 goto out; 2471 goto out;
2443 } 2472 }
2444 2473
2445 wg_update_endpoint_if_necessary(wgp, src); 2474 wg_update_endpoint_if_necessary(wgp, src);
2446 2475
2447 ok = wg_validate_route(wg, wgp, af, decrypted_buf); 2476 ok = wg_validate_route(wg, wgp, af, decrypted_buf);
2448 if (ok) { 2477 if (ok) {
2449 wg->wg_ops->input(&wg->wg_if, n, af); 2478 wg->wg_ops->input(&wg->wg_if, n, af);
2450 } else { 2479 } else {
2451 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2480 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
2452 "invalid source address\n"); 2481 "invalid source address\n");
2453 m_freem(n); 2482 m_freem(n);
2454 /* 2483 /*
2455 * The inner address is invalid however the session is valid 2484 * The inner address is invalid however the session is valid
2456 * so continue the session processing below. 2485 * so continue the session processing below.
2457 */ 2486 */
2458 } 2487 }
2459 n = NULL; 2488 n = NULL;
2460 2489
2461 if (wgs->wgs_state == WGS_STATE_INIT_PASSIVE) { 2490 if (wgs->wgs_state == WGS_STATE_INIT_PASSIVE) {
2462 struct wg_session *wgs_prev; 2491 struct wg_session *wgs_prev;
2463 2492
2464 KASSERT(wgs == wgp->wgp_session_unstable); 2493 KASSERT(wgs == wgp->wgp_session_unstable);
2465 wgs->wgs_state = WGS_STATE_ESTABLISHED; 2494 wgs->wgs_state = WGS_STATE_ESTABLISHED;
2466 wgs->wgs_time_established = time_uptime; 2495 wgs->wgs_time_established = time_uptime;
2467 wgs->wgs_time_last_data_sent = 0; 2496 wgs->wgs_time_last_data_sent = 0;
2468 wgs->wgs_is_initiator = false; 2497 wgs->wgs_is_initiator = false;
2469 WG_TRACE("WGS_STATE_ESTABLISHED"); 2498 WG_TRACE("WGS_STATE_ESTABLISHED");
2470 2499
2471 mutex_enter(wgp->wgp_lock); 2500 mutex_enter(wgp->wgp_lock);
2472 wg_swap_sessions(wgp); 2501 wg_swap_sessions(wgp);
2473 wgs_prev = wgp->wgp_session_unstable; 2502 wgs_prev = wgp->wgp_session_unstable;
2474 mutex_enter(wgs_prev->wgs_lock); 2503 mutex_enter(wgs_prev->wgs_lock);
2475 getnanotime(&wgp->wgp_last_handshake_time); 2504 getnanotime(&wgp->wgp_last_handshake_time);
2476 wgp->wgp_handshake_start_time = 0; 2505 wgp->wgp_handshake_start_time = 0;
2477 wgp->wgp_last_sent_mac1_valid = false; 2506 wgp->wgp_last_sent_mac1_valid = false;
2478 wgp->wgp_last_sent_cookie_valid = false; 2507 wgp->wgp_last_sent_cookie_valid = false;
2479 mutex_exit(wgp->wgp_lock); 2508 mutex_exit(wgp->wgp_lock);
2480 2509
2481 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) { 2510 if (wgs_prev->wgs_state == WGS_STATE_ESTABLISHED) {
2482 wgs_prev->wgs_state = WGS_STATE_DESTROYING; 2511 wgs_prev->wgs_state = WGS_STATE_DESTROYING;
2483 /* We can't destroy the old session immediately */ 2512 /* We can't destroy the old session immediately */
2484 wg_schedule_session_dtor_timer(wgp); 2513 wg_schedule_session_dtor_timer(wgp);
2485 } else { 2514 } else {
2486 wg_clear_states(wgs_prev); 2515 wg_clear_states(wgs_prev);
2487 wgs_prev->wgs_state = WGS_STATE_UNKNOWN; 2516 wgs_prev->wgs_state = WGS_STATE_UNKNOWN;
2488 } 2517 }
2489 mutex_exit(wgs_prev->wgs_lock); 2518 mutex_exit(wgs_prev->wgs_lock);
2490 2519
2491 /* Anyway run a softint to flush pending packets */ 2520 /* Anyway run a softint to flush pending packets */
2492 kpreempt_disable(); 2521 kpreempt_disable();
2493 softint_schedule(wgp->wgp_si); 2522 softint_schedule(wgp->wgp_si);
2494 kpreempt_enable(); 2523 kpreempt_enable();
2495 } else { 2524 } else {
2496 if (__predict_false(wg_need_to_send_init_message(wgs))) { 2525 if (__predict_false(wg_need_to_send_init_message(wgs))) {
2497 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 2526 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
2498 } 2527 }
2499 /* 2528 /*
2500 * [W] 6.5 Passive Keepalive 2529 * [W] 6.5 Passive Keepalive
2501 * "If a peer has received a validly-authenticated transport 2530 * "If a peer has received a validly-authenticated transport
2502 * data message (section 5.4.6), but does not have any packets 2531 * data message (section 5.4.6), but does not have any packets
2503 * itself to send back for KEEPALIVE-TIMEOUT seconds, it sends 2532 * itself to send back for KEEPALIVE-TIMEOUT seconds, it sends
2504 * a keepalive message." 2533 * a keepalive message."
2505 */ 2534 */
2506 WG_DLOG("time_uptime=%lu wgs_time_last_data_sent=%lu\n", 2535 WG_DLOG("time_uptime=%lu wgs_time_last_data_sent=%lu\n",
2507 time_uptime, wgs->wgs_time_last_data_sent); 2536 time_uptime, wgs->wgs_time_last_data_sent);
2508 if ((time_uptime - wgs->wgs_time_last_data_sent) >= 2537 if ((time_uptime - wgs->wgs_time_last_data_sent) >=
2509 wg_keepalive_timeout) { 2538 wg_keepalive_timeout) {
2510 WG_TRACE("Schedule sending keepalive message"); 2539 WG_TRACE("Schedule sending keepalive message");
2511 /* 2540 /*
2512 * We can't send a keepalive message here to avoid 2541 * We can't send a keepalive message here to avoid
2513 * a deadlock; we already hold the solock of a socket 2542 * a deadlock; we already hold the solock of a socket
2514 * that is used to send the message. 2543 * that is used to send the message.
2515 */ 2544 */
2516 wg_schedule_peer_task(wgp, 2545 wg_schedule_peer_task(wgp,
2517 WGP_TASK_SEND_KEEPALIVE_MESSAGE); 2546 WGP_TASK_SEND_KEEPALIVE_MESSAGE);
2518 } 2547 }
2519 } 2548 }
2520out: 2549out:
2521 wg_put_session(wgs, &psref); 2550 wg_put_session(wgs, &psref);
2522 if (m != NULL) 2551 if (m != NULL)
2523 m_freem(m); 2552 m_freem(m);
2524 if (free_encrypted_buf) 2553 if (free_encrypted_buf)
2525 kmem_intr_free(encrypted_buf, encrypted_len); 2554 kmem_intr_free(encrypted_buf, encrypted_len);
2526} 2555}
2527 2556
2528static void 2557static void
2529wg_handle_msg_cookie(struct wg_softc *wg, const struct wg_msg_cookie *wgmc) 2558wg_handle_msg_cookie(struct wg_softc *wg, const struct wg_msg_cookie *wgmc)
2530{ 2559{
2531 struct wg_session *wgs; 2560 struct wg_session *wgs;
2532 struct wg_peer *wgp; 2561 struct wg_peer *wgp;
2533 struct psref psref; 2562 struct psref psref;
2534 int error; 2563 int error;
2535 uint8_t key[WG_HASH_LEN]; 2564 uint8_t key[WG_HASH_LEN];
2536 uint8_t cookie[WG_COOKIE_LEN]; 2565 uint8_t cookie[WG_COOKIE_LEN];
2537 2566
2538 WG_TRACE("cookie msg received"); 2567 WG_TRACE("cookie msg received");
2539 wgs = wg_lookup_session_by_index(wg, wgmc->wgmc_receiver, &psref); 2568 wgs = wg_lookup_session_by_index(wg, wgmc->wgmc_receiver, &psref);
2540 if (wgs == NULL) { 2569 if (wgs == NULL) {
2541 WG_TRACE("No session found"); 2570 WG_TRACE("No session found");
2542 return; 2571 return;
2543 } 2572 }
2544 wgp = wgs->wgs_peer; 2573 wgp = wgs->wgs_peer;
2545 2574
2546 if (!wgp->wgp_last_sent_mac1_valid) { 2575 if (!wgp->wgp_last_sent_mac1_valid) {
2547 WG_TRACE("No valid mac1 sent (or expired)"); 2576 WG_TRACE("No valid mac1 sent (or expired)");
2548 goto out; 2577 goto out;
2549 } 2578 }
2550 2579
2551 wg_algo_mac_cookie(key, sizeof(key), wgp->wgp_pubkey, 2580 wg_algo_mac_cookie(key, sizeof(key), wgp->wgp_pubkey,
2552 sizeof(wgp->wgp_pubkey)); 2581 sizeof(wgp->wgp_pubkey));
2553 error = wg_algo_xaead_dec(cookie, sizeof(cookie), key, 2582 error = wg_algo_xaead_dec(cookie, sizeof(cookie), key,
2554 wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie), 2583 wgmc->wgmc_cookie, sizeof(wgmc->wgmc_cookie),
2555 wgp->wgp_last_sent_mac1, sizeof(wgp->wgp_last_sent_mac1), 2584 wgp->wgp_last_sent_mac1, sizeof(wgp->wgp_last_sent_mac1),
2556 wgmc->wgmc_salt); 2585 wgmc->wgmc_salt);
2557 if (error != 0) { 2586 if (error != 0) {
2558 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG, 2587 WG_LOG_RATECHECK(&wgp->wgp_ppsratecheck, LOG_DEBUG,
2559 "wg_algo_aead_dec for cookie failed: error=%d\n", error); 2588 "wg_algo_aead_dec for cookie failed: error=%d\n", error);
2560 goto out; 2589 goto out;
2561 } 2590 }
2562 /* 2591 /*
2563 * [W] 6.6: Interaction with Cookie Reply System 2592 * [W] 6.6: Interaction with Cookie Reply System
2564 * "it should simply store the decrypted cookie value from the cookie 2593 * "it should simply store the decrypted cookie value from the cookie
2565 * reply message, and wait for the expiration of the REKEY-TIMEOUT 2594 * reply message, and wait for the expiration of the REKEY-TIMEOUT
2566 * timer for retrying a handshake initiation message." 2595 * timer for retrying a handshake initiation message."
2567 */ 2596 */
2568 wgp->wgp_latest_cookie_time = time_uptime; 2597 wgp->wgp_latest_cookie_time = time_uptime;
2569 memcpy(wgp->wgp_latest_cookie, cookie, sizeof(wgp->wgp_latest_cookie)); 2598 memcpy(wgp->wgp_latest_cookie, cookie, sizeof(wgp->wgp_latest_cookie));
2570out: 2599out:
2571 wg_put_session(wgs, &psref); 2600 wg_put_session(wgs, &psref);
2572} 2601}
2573 2602
2574static struct mbuf * 2603static struct mbuf *
2575wg_validate_msg_header(struct wg_softc *wg, struct mbuf *m) 2604wg_validate_msg_header(struct wg_softc *wg, struct mbuf *m)
2576{ 2605{
2577 struct wg_msg wgm; 2606 struct wg_msg wgm;
2578 size_t mbuflen; 2607 size_t mbuflen;
2579 size_t msglen; 2608 size_t msglen;
2580 2609
2581 /* 2610 /*
2582 * Get the mbuf chain length. It is already guaranteed, by 2611 * Get the mbuf chain length. It is already guaranteed, by
2583 * wg_overudp_cb, to be large enough for a struct wg_msg. 2612 * wg_overudp_cb, to be large enough for a struct wg_msg.
2584 */ 2613 */
2585 mbuflen = m_length(m); 2614 mbuflen = m_length(m);
2586 KASSERT(mbuflen >= sizeof(struct wg_msg)); 2615 KASSERT(mbuflen >= sizeof(struct wg_msg));
2587 2616
2588 /* 2617 /*
2589 * Copy the message header (32-bit message type) out -- we'll 2618 * Copy the message header (32-bit message type) out -- we'll
2590 * worry about contiguity and alignment later. 2619 * worry about contiguity and alignment later.
2591 */ 2620 */
2592 m_copydata(m, 0, sizeof(wgm), &wgm); 2621 m_copydata(m, 0, sizeof(wgm), &wgm);
2593 switch (wgm.wgm_type) { 2622 switch (wgm.wgm_type) {
2594 case WG_MSG_TYPE_INIT: 2623 case WG_MSG_TYPE_INIT:
2595 msglen = sizeof(struct wg_msg_init); 2624 msglen = sizeof(struct wg_msg_init);
2596 break; 2625 break;
2597 case WG_MSG_TYPE_RESP: 2626 case WG_MSG_TYPE_RESP:
2598 msglen = sizeof(struct wg_msg_resp); 2627 msglen = sizeof(struct wg_msg_resp);
2599 break; 2628 break;
2600 case WG_MSG_TYPE_COOKIE: 2629 case WG_MSG_TYPE_COOKIE:
2601 msglen = sizeof(struct wg_msg_cookie); 2630 msglen = sizeof(struct wg_msg_cookie);
2602 break; 2631 break;
2603 case WG_MSG_TYPE_DATA: 2632 case WG_MSG_TYPE_DATA:
2604 msglen = sizeof(struct wg_msg_data); 2633 msglen = sizeof(struct wg_msg_data);
2605 break; 2634 break;
2606 default: 2635 default:
2607 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG, 2636 WG_LOG_RATECHECK(&wg->wg_ppsratecheck, LOG_DEBUG,
2608 "Unexpected msg type: %u\n", wgm.wgm_type); 2637 "Unexpected msg type: %u\n", wgm.wgm_type);
2609 goto error; 2638 goto error;
2610 } 2639 }
2611 2640
2612 /* Verify the mbuf chain is long enough for this type of message. */ 2641 /* Verify the mbuf chain is long enough for this type of message. */
2613 if (__predict_false(mbuflen < msglen)) { 2642 if (__predict_false(mbuflen < msglen)) {
2614 WG_DLOG("Invalid msg size: mbuflen=%lu type=%u\n", mbuflen, 2643 WG_DLOG("Invalid msg size: mbuflen=%lu type=%u\n", mbuflen,
2615 wgm.wgm_type); 2644 wgm.wgm_type);
2616 goto error; 2645 goto error;
2617 } 2646 }
2618 2647
2619 /* Make the message header contiguous if necessary. */ 2648 /* Make the message header contiguous if necessary. */
2620 if (__predict_false(m->m_len < msglen)) { 2649 if (__predict_false(m->m_len < msglen)) {
2621 m = m_pullup(m, msglen); 2650 m = m_pullup(m, msglen);
2622 if (m == NULL) 2651 if (m == NULL)
2623 return NULL; 2652 return NULL;
2624 } 2653 }
2625 2654
2626 return m; 2655 return m;
2627 2656
2628error: 2657error:
2629 m_freem(m); 2658 m_freem(m);
2630 return NULL; 2659 return NULL;
2631} 2660}
2632 2661
2633static void 2662static void
2634wg_handle_packet(struct wg_softc *wg, struct mbuf *m, 2663wg_handle_packet(struct wg_softc *wg, struct mbuf *m,
2635 const struct sockaddr *src) 2664 const struct sockaddr *src)
2636{ 2665{
2637 struct wg_msg *wgm; 2666 struct wg_msg *wgm;
2638 2667
2639 m = wg_validate_msg_header(wg, m); 2668 m = wg_validate_msg_header(wg, m);
2640 if (__predict_false(m == NULL)) 2669 if (__predict_false(m == NULL))
2641 return; 2670 return;
2642 2671
2643 KASSERT(m->m_len >= sizeof(struct wg_msg)); 2672 KASSERT(m->m_len >= sizeof(struct wg_msg));
2644 wgm = mtod(m, struct wg_msg *); 2673 wgm = mtod(m, struct wg_msg *);
2645 switch (wgm->wgm_type) { 2674 switch (wgm->wgm_type) {
2646 case WG_MSG_TYPE_INIT: 2675 case WG_MSG_TYPE_INIT:
2647 wg_handle_msg_init(wg, (struct wg_msg_init *)wgm, src); 2676 wg_handle_msg_init(wg, (struct wg_msg_init *)wgm, src);
2648 break; 2677 break;
2649 case WG_MSG_TYPE_RESP: 2678 case WG_MSG_TYPE_RESP:
2650 wg_handle_msg_resp(wg, (struct wg_msg_resp *)wgm, src); 2679 wg_handle_msg_resp(wg, (struct wg_msg_resp *)wgm, src);
2651 break; 2680 break;
2652 case WG_MSG_TYPE_COOKIE: 2681 case WG_MSG_TYPE_COOKIE:
2653 wg_handle_msg_cookie(wg, (struct wg_msg_cookie *)wgm); 2682 wg_handle_msg_cookie(wg, (struct wg_msg_cookie *)wgm);
2654 break; 2683 break;
2655 case WG_MSG_TYPE_DATA: 2684 case WG_MSG_TYPE_DATA:
2656 wg_handle_msg_data(wg, m, src); 2685 wg_handle_msg_data(wg, m, src);
2657 break; 2686 break;
2658 default: 2687 default:
2659 /* wg_validate_msg_header should already reject this case */ 2688 /* wg_validate_msg_header should already reject this case */
2660 break; 2689 break;
2661 } 2690 }
2662} 2691}
2663 2692
2664static void 2693static void
2665wg_receive_packets(struct wg_softc *wg, const int af) 2694wg_receive_packets(struct wg_softc *wg, const int af)
2666{ 2695{
2667 2696
2668 for (;;) { 2697 for (;;) {
2669 int error, flags; 2698 int error, flags;
2670 struct socket *so; 2699 struct socket *so;
2671 struct mbuf *m = NULL; 2700 struct mbuf *m = NULL;
2672 struct uio dummy_uio; 2701 struct uio dummy_uio;
2673 struct mbuf *paddr = NULL; 2702 struct mbuf *paddr = NULL;
2674 struct sockaddr *src; 2703 struct sockaddr *src;
2675 2704
2676 so = wg_get_so_by_af(wg->wg_worker, af); 2705 so = wg_get_so_by_af(wg->wg_worker, af);
2677 flags = MSG_DONTWAIT; 2706 flags = MSG_DONTWAIT;
2678 dummy_uio.uio_resid = 1000000000; 2707 dummy_uio.uio_resid = 1000000000;
2679 2708
2680 error = so->so_receive(so, &paddr, &dummy_uio, &m, NULL, 2709 error = so->so_receive(so, &paddr, &dummy_uio, &m, NULL,
2681 &flags); 2710 &flags);
2682 if (error || m == NULL) { 2711 if (error || m == NULL) {
2683 //if (error == EWOULDBLOCK) 2712 //if (error == EWOULDBLOCK)
2684 return; 2713 return;
2685 } 2714 }
2686 2715
2687 KASSERT(paddr != NULL); 2716 KASSERT(paddr != NULL);
2688 KASSERT(paddr->m_len >= sizeof(struct sockaddr)); 2717 KASSERT(paddr->m_len >= sizeof(struct sockaddr));
2689 src = mtod(paddr, struct sockaddr *); 2718 src = mtod(paddr, struct sockaddr *);
2690 2719
2691 wg_handle_packet(wg, m, src); 2720 wg_handle_packet(wg, m, src);
2692 } 2721 }
2693} 2722}
2694 2723
2695static void 2724static void
2696wg_get_peer(struct wg_peer *wgp, struct psref *psref) 2725wg_get_peer(struct wg_peer *wgp, struct psref *psref)
2697{ 2726{
2698 2727
2699 psref_acquire(psref, &wgp->wgp_psref, wg_psref_class); 2728 psref_acquire(psref, &wgp->wgp_psref, wg_psref_class);
2700} 2729}
2701 2730
2702static void 2731static void
2703wg_put_peer(struct wg_peer *wgp, struct psref *psref) 2732wg_put_peer(struct wg_peer *wgp, struct psref *psref)
2704{ 2733{
2705 2734
2706 psref_release(psref, &wgp->wgp_psref, wg_psref_class); 2735 psref_release(psref, &wgp->wgp_psref, wg_psref_class);
2707} 2736}
2708 2737
2709static void 2738static void
2710wg_task_send_init_message(struct wg_softc *wg, struct wg_peer *wgp) 2739wg_task_send_init_message(struct wg_softc *wg, struct wg_peer *wgp)
2711{ 2740{
2712 struct psref psref; 2741 struct psref psref;
2713 struct wg_session *wgs; 2742 struct wg_session *wgs;
2714 2743
2715 WG_TRACE("WGP_TASK_SEND_INIT_MESSAGE"); 2744 WG_TRACE("WGP_TASK_SEND_INIT_MESSAGE");
2716 2745
2717 if (!wgp->wgp_endpoint_available) { 2746 if (!wgp->wgp_endpoint_available) {
2718 WGLOG(LOG_DEBUG, "No endpoint available\n"); 2747 WGLOG(LOG_DEBUG, "No endpoint available\n");
2719 /* XXX should do something? */ 2748 /* XXX should do something? */
2720 return; 2749 return;
2721 } 2750 }
2722 2751
2723 wgs = wg_get_stable_session(wgp, &psref); 2752 wgs = wg_get_stable_session(wgp, &psref);
2724 if (wgs->wgs_state == WGS_STATE_UNKNOWN) { 2753 if (wgs->wgs_state == WGS_STATE_UNKNOWN) {
2725 wg_put_session(wgs, &psref); 2754 wg_put_session(wgs, &psref);
2726 wg_send_handshake_msg_init(wg, wgp); 2755 wg_send_handshake_msg_init(wg, wgp);
2727 } else { 2756 } else {
2728 wg_put_session(wgs, &psref); 2757 wg_put_session(wgs, &psref);
2729 /* rekey */ 2758 /* rekey */
2730 wgs = wg_get_unstable_session(wgp, &psref); 2759 wgs = wg_get_unstable_session(wgp, &psref);
2731 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE) 2760 if (wgs->wgs_state != WGS_STATE_INIT_ACTIVE)
2732 wg_send_handshake_msg_init(wg, wgp); 2761 wg_send_handshake_msg_init(wg, wgp);
2733 wg_put_session(wgs, &psref); 2762 wg_put_session(wgs, &psref);
2734 } 2763 }
2735} 2764}
2736 2765
2737static void 2766static void
2738wg_task_endpoint_changed(struct wg_softc *wg, struct wg_peer *wgp) 2767wg_task_endpoint_changed(struct wg_softc *wg, struct wg_peer *wgp)
2739{ 2768{
2740 2769
2741 WG_TRACE("WGP_TASK_ENDPOINT_CHANGED"); 2770 WG_TRACE("WGP_TASK_ENDPOINT_CHANGED");
2742 2771
2743 mutex_enter(wgp->wgp_lock); 2772 mutex_enter(wgp->wgp_lock);
2744 if (wgp->wgp_endpoint_changing) { 2773 if (wgp->wgp_endpoint_changing) {
2745 pserialize_perform(wgp->wgp_psz); 2774 pserialize_perform(wgp->wgp_psz);
2746 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, 2775 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref,
2747 wg_psref_class); 2776 wg_psref_class);
2748 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, 2777 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref,
2749 wg_psref_class); 2778 wg_psref_class);
2750 wgp->wgp_endpoint_changing = false; 2779 wgp->wgp_endpoint_changing = false;
2751 } 2780 }
2752 mutex_exit(wgp->wgp_lock); 2781 mutex_exit(wgp->wgp_lock);
2753} 2782}
2754 2783
2755static void 2784static void
2756wg_task_send_keepalive_message(struct wg_softc *wg, struct wg_peer *wgp) 2785wg_task_send_keepalive_message(struct wg_softc *wg, struct wg_peer *wgp)
2757{ 2786{
2758 struct psref psref; 2787 struct psref psref;
2759 struct wg_session *wgs; 2788 struct wg_session *wgs;
2760 2789
2761 WG_TRACE("WGP_TASK_SEND_KEEPALIVE_MESSAGE"); 2790 WG_TRACE("WGP_TASK_SEND_KEEPALIVE_MESSAGE");
2762 2791
2763 wgs = wg_get_stable_session(wgp, &psref); 2792 wgs = wg_get_stable_session(wgp, &psref);
2764 wg_send_keepalive_msg(wgp, wgs); 2793 wg_send_keepalive_msg(wgp, wgs);
2765 wg_put_session(wgs, &psref); 2794 wg_put_session(wgs, &psref);
2766} 2795}
2767 2796
2768static void 2797static void
2769wg_task_destroy_prev_session(struct wg_softc *wg, struct wg_peer *wgp) 2798wg_task_destroy_prev_session(struct wg_softc *wg, struct wg_peer *wgp)
2770{ 2799{
2771 struct wg_session *wgs; 2800 struct wg_session *wgs;
2772 2801
2773 WG_TRACE("WGP_TASK_DESTROY_PREV_SESSION"); 2802 WG_TRACE("WGP_TASK_DESTROY_PREV_SESSION");
2774 2803
2775 mutex_enter(wgp->wgp_lock); 2804 mutex_enter(wgp->wgp_lock);
2776 wgs = wgp->wgp_session_unstable; 2805 wgs = wgp->wgp_session_unstable;
2777 mutex_enter(wgs->wgs_lock); 2806 mutex_enter(wgs->wgs_lock);
2778 if (wgs->wgs_state == WGS_STATE_DESTROYING) { 2807 if (wgs->wgs_state == WGS_STATE_DESTROYING) {
2779 pserialize_perform(wgp->wgp_psz); 2808 pserialize_perform(wgp->wgp_psz);
2780 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 2809 psref_target_destroy(&wgs->wgs_psref, wg_psref_class);
2781 psref_target_init(&wgs->wgs_psref, wg_psref_class); 2810 psref_target_init(&wgs->wgs_psref, wg_psref_class);
2782 wg_clear_states(wgs); 2811 wg_clear_states(wgs);
2783 wgs->wgs_state = WGS_STATE_UNKNOWN; 2812 wgs->wgs_state = WGS_STATE_UNKNOWN;
2784 } 2813 }
2785 mutex_exit(wgs->wgs_lock); 2814 mutex_exit(wgs->wgs_lock);
2786 mutex_exit(wgp->wgp_lock); 2815 mutex_exit(wgp->wgp_lock);
2787} 2816}
2788 2817
2789static void 2818static void
2790wg_process_peer_tasks(struct wg_softc *wg) 2819wg_process_peer_tasks(struct wg_softc *wg)
2791{ 2820{
2792 struct wg_peer *wgp; 2821 struct wg_peer *wgp;
2793 int s; 2822 int s;
2794 2823
2795 /* XXX should avoid checking all peers */ 2824 /* XXX should avoid checking all peers */
2796 s = pserialize_read_enter(); 2825 s = pserialize_read_enter();
2797 WG_PEER_READER_FOREACH(wgp, wg) { 2826 WG_PEER_READER_FOREACH(wgp, wg) {
2798 struct psref psref; 2827 struct psref psref;
2799 unsigned int tasks; 2828 unsigned int tasks;
2800 2829
2801 if (wgp->wgp_tasks == 0) 2830 if (wgp->wgp_tasks == 0)
2802 continue; 2831 continue;
2803 2832
2804 wg_get_peer(wgp, &psref); 2833 wg_get_peer(wgp, &psref);
2805 pserialize_read_exit(s); 2834 pserialize_read_exit(s);
2806 2835
2807 restart: 2836 restart:
2808 tasks = atomic_swap_uint(&wgp->wgp_tasks, 0); 2837 tasks = atomic_swap_uint(&wgp->wgp_tasks, 0);
2809 KASSERT(tasks != 0); 2838 KASSERT(tasks != 0);
2810 2839
2811 WG_DLOG("tasks=%x\n", tasks); 2840 WG_DLOG("tasks=%x\n", tasks);
2812 2841
2813 if (ISSET(tasks, WGP_TASK_SEND_INIT_MESSAGE)) 2842 if (ISSET(tasks, WGP_TASK_SEND_INIT_MESSAGE))
2814 wg_task_send_init_message(wg, wgp); 2843 wg_task_send_init_message(wg, wgp);
2815 if (ISSET(tasks, WGP_TASK_ENDPOINT_CHANGED)) 2844 if (ISSET(tasks, WGP_TASK_ENDPOINT_CHANGED))
2816 wg_task_endpoint_changed(wg, wgp); 2845 wg_task_endpoint_changed(wg, wgp);
2817 if (ISSET(tasks, WGP_TASK_SEND_KEEPALIVE_MESSAGE)) 2846 if (ISSET(tasks, WGP_TASK_SEND_KEEPALIVE_MESSAGE))
2818 wg_task_send_keepalive_message(wg, wgp); 2847 wg_task_send_keepalive_message(wg, wgp);
2819 if (ISSET(tasks, WGP_TASK_DESTROY_PREV_SESSION)) 2848 if (ISSET(tasks, WGP_TASK_DESTROY_PREV_SESSION))
2820 wg_task_destroy_prev_session(wg, wgp); 2849 wg_task_destroy_prev_session(wg, wgp);
2821 2850
2822 /* New tasks may be scheduled during processing tasks */ 2851 /* New tasks may be scheduled during processing tasks */
2823 WG_DLOG("wgp_tasks=%d\n", wgp->wgp_tasks); 2852 WG_DLOG("wgp_tasks=%d\n", wgp->wgp_tasks);
2824 if (wgp->wgp_tasks != 0) 2853 if (wgp->wgp_tasks != 0)
2825 goto restart; 2854 goto restart;
2826 2855
2827 s = pserialize_read_enter(); 2856 s = pserialize_read_enter();
2828 wg_put_peer(wgp, &psref); 2857 wg_put_peer(wgp, &psref);
2829 } 2858 }
2830 pserialize_read_exit(s); 2859 pserialize_read_exit(s);
2831} 2860}
2832 2861
2833static void 2862static void
2834wg_worker(void *arg) 2863wg_worker(void *arg)
2835{ 2864{
2836 struct wg_softc *wg = arg; 2865 struct wg_softc *wg = arg;
2837 struct wg_worker *wgw = wg->wg_worker; 2866 struct wg_worker *wgw = wg->wg_worker;
2838 bool todie = false; 2867 bool todie = false;
2839 2868
2840 KASSERT(wg != NULL); 2869 KASSERT(wg != NULL);
2841 KASSERT(wgw != NULL); 2870 KASSERT(wgw != NULL);
2842 2871
2843 while (!todie) { 2872 while (!todie) {
2844 int reasons; 2873 int reasons;
2845 int bound; 2874 int bound;
2846 2875
2847 mutex_enter(&wgw->wgw_lock); 2876 mutex_enter(&wgw->wgw_lock);
2848 /* New tasks may come during task handling */ 2877 /* New tasks may come during task handling */
2849 while ((reasons = wgw->wgw_wakeup_reasons) == 0 && 2878 while ((reasons = wgw->wgw_wakeup_reasons) == 0 &&
2850 !(todie = wgw->wgw_todie)) 2879 !(todie = wgw->wgw_todie))
2851 cv_wait(&wgw->wgw_cv, &wgw->wgw_lock); 2880 cv_wait(&wgw->wgw_cv, &wgw->wgw_lock);
2852 wgw->wgw_wakeup_reasons = 0; 2881 wgw->wgw_wakeup_reasons = 0;
2853 mutex_exit(&wgw->wgw_lock); 2882 mutex_exit(&wgw->wgw_lock);
2854 2883
2855 bound = curlwp_bind(); 2884 bound = curlwp_bind();
2856 if (ISSET(reasons, WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4)) 2885 if (ISSET(reasons, WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4))
2857 wg_receive_packets(wg, AF_INET); 2886 wg_receive_packets(wg, AF_INET);
2858 if (ISSET(reasons, WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6)) 2887 if (ISSET(reasons, WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6))
2859 wg_receive_packets(wg, AF_INET6); 2888 wg_receive_packets(wg, AF_INET6);
2860 if (ISSET(reasons, WG_WAKEUP_REASON_PEER)) 2889 if (ISSET(reasons, WG_WAKEUP_REASON_PEER))
2861 wg_process_peer_tasks(wg); 2890 wg_process_peer_tasks(wg);
2862 curlwp_bindx(bound); 2891 curlwp_bindx(bound);
2863 } 2892 }
2864 kthread_exit(0); 2893 kthread_exit(0);
2865} 2894}
2866 2895
2867static void 2896static void
2868wg_wakeup_worker(struct wg_worker *wgw, const int reason) 2897wg_wakeup_worker(struct wg_worker *wgw, const int reason)
2869{ 2898{
2870 2899
2871 mutex_enter(&wgw->wgw_lock); 2900 mutex_enter(&wgw->wgw_lock);
2872 wgw->wgw_wakeup_reasons |= reason; 2901 wgw->wgw_wakeup_reasons |= reason;
2873 cv_broadcast(&wgw->wgw_cv); 2902 cv_broadcast(&wgw->wgw_cv);
2874 mutex_exit(&wgw->wgw_lock); 2903 mutex_exit(&wgw->wgw_lock);
2875} 2904}
2876 2905
2877static int 2906static int
2878wg_bind_port(struct wg_softc *wg, const uint16_t port) 2907wg_bind_port(struct wg_softc *wg, const uint16_t port)
2879{ 2908{
2880 int error; 2909 int error;
2881 struct wg_worker *wgw = wg->wg_worker; 2910 struct wg_worker *wgw = wg->wg_worker;
2882 uint16_t old_port = wg->wg_listen_port; 2911 uint16_t old_port = wg->wg_listen_port;
2883 2912
2884 if (port != 0 && old_port == port) 2913 if (port != 0 && old_port == port)
2885 return 0; 2914 return 0;
2886 2915
2887 struct sockaddr_in _sin, *sin = &_sin; 2916 struct sockaddr_in _sin, *sin = &_sin;
2888 sin->sin_len = sizeof(*sin); 2917 sin->sin_len = sizeof(*sin);
2889 sin->sin_family = AF_INET; 2918 sin->sin_family = AF_INET;
2890 sin->sin_addr.s_addr = INADDR_ANY; 2919 sin->sin_addr.s_addr = INADDR_ANY;
2891 sin->sin_port = htons(port); 2920 sin->sin_port = htons(port);
2892 2921
2893 error = sobind(wgw->wgw_so4, sintosa(sin), curlwp); 2922 error = sobind(wgw->wgw_so4, sintosa(sin), curlwp);
2894 if (error != 0) 2923 if (error != 0)
2895 return error; 2924 return error;
2896 2925
2897#ifdef INET6 2926#ifdef INET6
2898 struct sockaddr_in6 _sin6, *sin6 = &_sin6; 2927 struct sockaddr_in6 _sin6, *sin6 = &_sin6;
2899 sin6->sin6_len = sizeof(*sin6); 2928 sin6->sin6_len = sizeof(*sin6);
2900 sin6->sin6_family = AF_INET6; 2929 sin6->sin6_family = AF_INET6;
2901 sin6->sin6_addr = in6addr_any; 2930 sin6->sin6_addr = in6addr_any;
2902 sin6->sin6_port = htons(port); 2931 sin6->sin6_port = htons(port);
2903 2932
2904 error = sobind(wgw->wgw_so6, sin6tosa(sin6), curlwp); 2933 error = sobind(wgw->wgw_so6, sin6tosa(sin6), curlwp);
2905 if (error != 0) 2934 if (error != 0)
2906 return error; 2935 return error;
2907#endif 2936#endif
2908 2937
2909 wg->wg_listen_port = port; 2938 wg->wg_listen_port = port;
2910 2939
2911 return 0; 2940 return 0;
2912} 2941}
2913 2942
2914static void 2943static void
2915wg_so_upcall(struct socket *so, void *arg, int events, int waitflag) 2944wg_so_upcall(struct socket *so, void *arg, int events, int waitflag)
2916{ 2945{
2917 struct wg_worker *wgw = arg; 2946 struct wg_worker *wgw = arg;
2918 int reason; 2947 int reason;
2919 2948
2920 reason = (so->so_proto->pr_domain->dom_family == AF_INET) ? 2949 reason = (so->so_proto->pr_domain->dom_family == AF_INET) ?
2921 WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4 : 2950 WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV4 :
2922 WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6; 2951 WG_WAKEUP_REASON_RECEIVE_PACKETS_IPV6;
2923 wg_wakeup_worker(wgw, reason); 2952 wg_wakeup_worker(wgw, reason);
2924} 2953}
2925 2954
2926static int 2955static int
2927wg_overudp_cb(struct mbuf **mp, int offset, struct socket *so, 2956wg_overudp_cb(struct mbuf **mp, int offset, struct socket *so,
2928 struct sockaddr *src, void *arg) 2957 struct sockaddr *src, void *arg)
2929{ 2958{
2930 struct wg_softc *wg = arg; 2959 struct wg_softc *wg = arg;
2931 struct wg_msg wgm; 2960 struct wg_msg wgm;
2932 struct mbuf *m = *mp; 2961 struct mbuf *m = *mp;
2933 2962
2934 WG_TRACE("enter"); 2963 WG_TRACE("enter");
2935 2964
2936 /* Verify the mbuf chain is long enough to have a wg msg header. */ 2965 /* Verify the mbuf chain is long enough to have a wg msg header. */
2937 KASSERT(offset <= m_length(m)); 2966 KASSERT(offset <= m_length(m));
2938 if (__predict_false(m_length(m) - offset < sizeof(struct wg_msg))) { 2967 if (__predict_false(m_length(m) - offset < sizeof(struct wg_msg))) {
2939 /* drop on the floor */ 2968 /* drop on the floor */
2940 m_freem(m); 2969 m_freem(m);
2941 return -1; 2970 return -1;
2942 } 2971 }
2943 2972
2944 /* 2973 /*
2945 * Copy the message header (32-bit message type) out -- we'll 2974 * Copy the message header (32-bit message type) out -- we'll
2946 * worry about contiguity and alignment later. 2975 * worry about contiguity and alignment later.
2947 */ 2976 */
2948 m_copydata(m, offset, sizeof(struct wg_msg), &wgm); 2977 m_copydata(m, offset, sizeof(struct wg_msg), &wgm);
2949 WG_DLOG("type=%d\n", wgm.wgm_type); 2978 WG_DLOG("type=%d\n", wgm.wgm_type);
2950 2979
2951 /* 2980 /*
2952 * Handle DATA packets promptly as they arrive. Other packets 2981 * Handle DATA packets promptly as they arrive. Other packets
2953 * may require expensive public-key crypto and are not as 2982 * may require expensive public-key crypto and are not as
2954 * sensitive to latency, so defer them to the worker thread. 2983 * sensitive to latency, so defer them to the worker thread.
2955 */ 2984 */
2956 switch (wgm.wgm_type) { 2985 switch (wgm.wgm_type) {
2957 case WG_MSG_TYPE_DATA: 2986 case WG_MSG_TYPE_DATA:
2958 /* handle immediately */ 2987 /* handle immediately */
2959 m_adj(m, offset); 2988 m_adj(m, offset);
2960 if (__predict_false(m->m_len < sizeof(struct wg_msg_data))) { 2989 if (__predict_false(m->m_len < sizeof(struct wg_msg_data))) {
2961 m = m_pullup(m, sizeof(struct wg_msg_data)); 2990 m = m_pullup(m, sizeof(struct wg_msg_data));
2962 if (m == NULL) 2991 if (m == NULL)
2963 return -1; 2992 return -1;
2964 } 2993 }
2965 wg_handle_msg_data(wg, m, src); 2994 wg_handle_msg_data(wg, m, src);
2966 *mp = NULL; 2995 *mp = NULL;
2967 return 1; 2996 return 1;
2968 case WG_MSG_TYPE_INIT: 2997 case WG_MSG_TYPE_INIT:
2969 case WG_MSG_TYPE_RESP: 2998 case WG_MSG_TYPE_RESP:
2970 case WG_MSG_TYPE_COOKIE: 2999 case WG_MSG_TYPE_COOKIE:
2971 /* pass through to so_receive in wg_receive_packets */ 3000 /* pass through to so_receive in wg_receive_packets */
2972 return 0; 3001 return 0;
2973 default: 3002 default:
2974 /* drop on the floor */ 3003 /* drop on the floor */
2975 m_freem(m); 3004 m_freem(m);
2976 return -1; 3005 return -1;
2977 } 3006 }
2978} 3007}
2979 3008
2980static int 3009static int
2981wg_worker_socreate(struct wg_softc *wg, struct wg_worker *wgw, const int af, 3010wg_worker_socreate(struct wg_softc *wg, struct wg_worker *wgw, const int af,
2982 struct socket **sop) 3011 struct socket **sop)
2983{ 3012{
2984 int error; 3013 int error;
2985 struct socket *so; 3014 struct socket *so;
2986 3015
2987 error = socreate(af, &so, SOCK_DGRAM, 0, curlwp, NULL); 3016 error = socreate(af, &so, SOCK_DGRAM, 0, curlwp, NULL);
2988 if (error != 0) 3017 if (error != 0)
2989 return error; 3018 return error;
2990 3019
2991 solock(so); 3020 solock(so);
2992 so->so_upcallarg = wgw; 3021 so->so_upcallarg = wgw;
2993 so->so_upcall = wg_so_upcall; 3022 so->so_upcall = wg_so_upcall;
2994 so->so_rcv.sb_flags |= SB_UPCALL; 3023 so->so_rcv.sb_flags |= SB_UPCALL;
2995 if (af == AF_INET) 3024 if (af == AF_INET)
2996 in_pcb_register_overudp_cb(sotoinpcb(so), wg_overudp_cb, wg); 3025 in_pcb_register_overudp_cb(sotoinpcb(so), wg_overudp_cb, wg);
2997#if INET6 3026#if INET6
2998 else 3027 else
2999 in6_pcb_register_overudp_cb(sotoin6pcb(so), wg_overudp_cb, wg); 3028 in6_pcb_register_overudp_cb(sotoin6pcb(so), wg_overudp_cb, wg);
3000#endif 3029#endif
3001 sounlock(so); 3030 sounlock(so);
3002 3031
3003 *sop = so; 3032 *sop = so;
3004 3033
3005 return 0; 3034 return 0;
3006} 3035}
3007 3036
3008static int 3037static int
3009wg_worker_init(struct wg_softc *wg) 3038wg_worker_init(struct wg_softc *wg)
3010{ 3039{
3011 int error; 3040 int error;
3012 struct wg_worker *wgw; 3041 struct wg_worker *wgw;
3013 const char *ifname = wg->wg_if.if_xname; 3042 const char *ifname = wg->wg_if.if_xname;
3014 struct socket *so; 3043 struct socket *so;
3015 3044
3016 wgw = kmem_zalloc(sizeof(struct wg_worker), KM_SLEEP); 3045 wgw = kmem_zalloc(sizeof(struct wg_worker), KM_SLEEP);
3017 3046
3018 mutex_init(&wgw->wgw_lock, MUTEX_DEFAULT, IPL_NONE); 3047 mutex_init(&wgw->wgw_lock, MUTEX_DEFAULT, IPL_NONE);
3019 cv_init(&wgw->wgw_cv, ifname); 3048 cv_init(&wgw->wgw_cv, ifname);
3020 wgw->wgw_todie = false; 3049 wgw->wgw_todie = false;
3021 wgw->wgw_wakeup_reasons = 0; 3050 wgw->wgw_wakeup_reasons = 0;
3022 3051
3023 error = wg_worker_socreate(wg, wgw, AF_INET, &so); 3052 error = wg_worker_socreate(wg, wgw, AF_INET, &so);
3024 if (error != 0) 3053 if (error != 0)
3025 goto error; 3054 goto error;
3026 wgw->wgw_so4 = so; 3055 wgw->wgw_so4 = so;
3027#ifdef INET6 3056#ifdef INET6
3028 error = wg_worker_socreate(wg, wgw, AF_INET6, &so); 3057 error = wg_worker_socreate(wg, wgw, AF_INET6, &so);
3029 if (error != 0) 3058 if (error != 0)
3030 goto error; 3059 goto error;
3031 wgw->wgw_so6 = so; 3060 wgw->wgw_so6 = so;
3032#endif 3061#endif
3033 3062
3034 wg->wg_worker = wgw; 3063 wg->wg_worker = wgw;
3035 3064
3036 error = kthread_create(PRI_NONE, KTHREAD_MPSAFE | KTHREAD_MUSTJOIN, 3065 error = kthread_create(PRI_NONE, KTHREAD_MPSAFE | KTHREAD_MUSTJOIN,
3037 NULL, wg_worker, wg, &wg->wg_worker_lwp, "%s", ifname); 3066 NULL, wg_worker, wg, &wg->wg_worker_lwp, "%s", ifname);
3038 if (error != 0) 3067 if (error != 0)
3039 goto error; 3068 goto error;
3040 3069
3041 return 0; 3070 return 0;
3042 3071
3043error: 3072error:
3044#ifdef INET6 3073#ifdef INET6
3045 if (wgw->wgw_so6 != NULL) 3074 if (wgw->wgw_so6 != NULL)
3046 soclose(wgw->wgw_so6); 3075 soclose(wgw->wgw_so6);
3047#endif 3076#endif
3048 if (wgw->wgw_so4 != NULL) 3077 if (wgw->wgw_so4 != NULL)
3049 soclose(wgw->wgw_so4); 3078 soclose(wgw->wgw_so4);
3050 cv_destroy(&wgw->wgw_cv); 3079 cv_destroy(&wgw->wgw_cv);
3051 mutex_destroy(&wgw->wgw_lock); 3080 mutex_destroy(&wgw->wgw_lock);
3052 3081
3053 return error; 3082 return error;
3054} 3083}
3055 3084
3056static void 3085static void
3057wg_worker_destroy(struct wg_softc *wg) 3086wg_worker_destroy(struct wg_softc *wg)
3058{ 3087{
3059 struct wg_worker *wgw = wg->wg_worker; 3088 struct wg_worker *wgw = wg->wg_worker;
3060 3089
3061 mutex_enter(&wgw->wgw_lock); 3090 mutex_enter(&wgw->wgw_lock);
3062 wgw->wgw_todie = true; 3091 wgw->wgw_todie = true;
3063 wgw->wgw_wakeup_reasons = 0; 3092 wgw->wgw_wakeup_reasons = 0;
3064 cv_broadcast(&wgw->wgw_cv); 3093 cv_broadcast(&wgw->wgw_cv);
3065 mutex_exit(&wgw->wgw_lock); 3094 mutex_exit(&wgw->wgw_lock);
3066 3095
3067 kthread_join(wg->wg_worker_lwp); 3096 kthread_join(wg->wg_worker_lwp);
3068 3097
3069#ifdef INET6 3098#ifdef INET6
3070 soclose(wgw->wgw_so6); 3099 soclose(wgw->wgw_so6);
3071#endif 3100#endif
3072 soclose(wgw->wgw_so4); 3101 soclose(wgw->wgw_so4);
3073 cv_destroy(&wgw->wgw_cv); 3102 cv_destroy(&wgw->wgw_cv);
3074 mutex_destroy(&wgw->wgw_lock); 3103 mutex_destroy(&wgw->wgw_lock);
3075 kmem_free(wg->wg_worker, sizeof(struct wg_worker)); 3104 kmem_free(wg->wg_worker, sizeof(struct wg_worker));
3076 wg->wg_worker = NULL; 3105 wg->wg_worker = NULL;
3077} 3106}
3078 3107
3079static bool 3108static bool
3080wg_session_hit_limits(struct wg_session *wgs) 3109wg_session_hit_limits(struct wg_session *wgs)
3081{ 3110{
3082 3111
3083 /* 3112 /*
3084 * [W] 6.2: Transport Message Limits 3113 * [W] 6.2: Transport Message Limits
3085 * "After REJECT-AFTER-MESSAGES transport data messages or after the 3114 * "After REJECT-AFTER-MESSAGES transport data messages or after the
3086 * current secure session is REJECT-AFTER-TIME seconds old, whichever 3115 * current secure session is REJECT-AFTER-TIME seconds old, whichever
3087 * comes first, WireGuard will refuse to send any more transport data 3116 * comes first, WireGuard will refuse to send any more transport data
3088 * messages using the current secure session, ..." 3117 * messages using the current secure session, ..."
3089 */ 3118 */
3090 KASSERT(wgs->wgs_time_established != 0); 3119 KASSERT(wgs->wgs_time_established != 0);
3091 if ((time_uptime - wgs->wgs_time_established) > wg_reject_after_time) { 3120 if ((time_uptime - wgs->wgs_time_established) > wg_reject_after_time) {
3092 WG_DLOG("The session hits REJECT_AFTER_TIME\n"); 3121 WG_DLOG("The session hits REJECT_AFTER_TIME\n");
3093 return true; 3122 return true;
3094 } else if (wg_session_get_send_counter(wgs) > 3123 } else if (wg_session_get_send_counter(wgs) >
3095 wg_reject_after_messages) { 3124 wg_reject_after_messages) {
3096 WG_DLOG("The session hits REJECT_AFTER_MESSAGES\n"); 3125 WG_DLOG("The session hits REJECT_AFTER_MESSAGES\n");
3097 return true; 3126 return true;
3098 } 3127 }
3099 3128
3100 return false; 3129 return false;
3101} 3130}
3102 3131
3103static void 3132static void
3104wg_peer_softint(void *arg) 3133wg_peer_softint(void *arg)
3105{ 3134{
3106 struct wg_peer *wgp = arg; 3135 struct wg_peer *wgp = arg;
3107 struct wg_session *wgs; 3136 struct wg_session *wgs;
3108 struct mbuf *m; 3137 struct mbuf *m;
3109 struct psref psref; 3138 struct psref psref;
3110 3139
3111 wgs = wg_get_stable_session(wgp, &psref); 3140 wgs = wg_get_stable_session(wgp, &psref);
3112 if (wgs->wgs_state != WGS_STATE_ESTABLISHED) { 3141 if (wgs->wgs_state != WGS_STATE_ESTABLISHED) {
3113 /* XXX how to treat? */ 3142 /* XXX how to treat? */
3114 WG_TRACE("skipped"); 3143 WG_TRACE("skipped");
3115 goto out; 3144 goto out;
3116 } 3145 }
3117 if (wg_session_hit_limits(wgs)) { 3146 if (wg_session_hit_limits(wgs)) {
3118 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3147 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
3119 goto out; 3148 goto out;
3120 } 3149 }
3121 WG_TRACE("running"); 3150 WG_TRACE("running");
3122 3151
3123 while ((m = pcq_get(wgp->wgp_q)) != NULL) { 3152 while ((m = pcq_get(wgp->wgp_q)) != NULL) {
3124 wg_send_data_msg(wgp, wgs, m); 3153 wg_send_data_msg(wgp, wgs, m);
3125 } 3154 }
3126out: 3155out:
3127 wg_put_session(wgs, &psref); 3156 wg_put_session(wgs, &psref);
3128} 3157}
3129 3158
3130static void 3159static void
3131wg_rekey_timer(void *arg) 3160wg_rekey_timer(void *arg)
3132{ 3161{
3133 struct wg_peer *wgp = arg; 3162 struct wg_peer *wgp = arg;
3134 3163
3135 mutex_enter(wgp->wgp_lock); 3164 mutex_enter(wgp->wgp_lock);
3136 if (__predict_true(wgp->wgp_state != WGP_STATE_DESTROYING)) { 3165 if (__predict_true(wgp->wgp_state != WGP_STATE_DESTROYING)) {
3137 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3166 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
3138 } 3167 }
3139 mutex_exit(wgp->wgp_lock); 3168 mutex_exit(wgp->wgp_lock);
3140} 3169}
3141 3170
3142static void 3171static void
3143wg_purge_pending_packets(struct wg_peer *wgp) 3172wg_purge_pending_packets(struct wg_peer *wgp)
3144{ 3173{
3145 struct mbuf *m; 3174 struct mbuf *m;
3146 3175
3147 while ((m = pcq_get(wgp->wgp_q)) != NULL) { 3176 while ((m = pcq_get(wgp->wgp_q)) != NULL) {
3148 m_freem(m); 3177 m_freem(m);
3149 } 3178 }
3150} 3179}
3151 3180
3152static void 3181static void
3153wg_handshake_timeout_timer(void *arg) 3182wg_handshake_timeout_timer(void *arg)
3154{ 3183{
3155 struct wg_peer *wgp = arg; 3184 struct wg_peer *wgp = arg;
3156 struct wg_session *wgs; 3185 struct wg_session *wgs;
3157 struct psref psref; 3186 struct psref psref;
3158 3187
3159 WG_TRACE("enter"); 3188 WG_TRACE("enter");
3160 3189
3161 mutex_enter(wgp->wgp_lock); 3190 mutex_enter(wgp->wgp_lock);
3162 if (__predict_false(wgp->wgp_state == WGP_STATE_DESTROYING)) { 3191 if (__predict_false(wgp->wgp_state == WGP_STATE_DESTROYING)) {
3163 mutex_exit(wgp->wgp_lock); 3192 mutex_exit(wgp->wgp_lock);
3164 return; 3193 return;
3165 } 3194 }
3166 mutex_exit(wgp->wgp_lock); 3195 mutex_exit(wgp->wgp_lock);
3167 3196
3168 KASSERT(wgp->wgp_handshake_start_time != 0); 3197 KASSERT(wgp->wgp_handshake_start_time != 0);
3169 wgs = wg_get_unstable_session(wgp, &psref); 3198 wgs = wg_get_unstable_session(wgp, &psref);
3170 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE); 3199 KASSERT(wgs->wgs_state == WGS_STATE_INIT_ACTIVE);
3171 3200
3172 /* [W] 6.4 Handshake Initiation Retransmission */ 3201 /* [W] 6.4 Handshake Initiation Retransmission */
3173 if ((time_uptime - wgp->wgp_handshake_start_time) > 3202 if ((time_uptime - wgp->wgp_handshake_start_time) >
3174 wg_rekey_attempt_time) { 3203 wg_rekey_attempt_time) {
3175 /* Give up handshaking */ 3204 /* Give up handshaking */
3176 wgs->wgs_state = WGS_STATE_UNKNOWN; 3205 wgs->wgs_state = WGS_STATE_UNKNOWN;
3177 wg_clear_states(wgs); 3206 wg_clear_states(wgs);
3178 wgp->wgp_state = WGP_STATE_GIVEUP; 3207 wgp->wgp_state = WGP_STATE_GIVEUP;
3179 wgp->wgp_handshake_start_time = 0; 3208 wgp->wgp_handshake_start_time = 0;
3180 wg_put_session(wgs, &psref); 3209 wg_put_session(wgs, &psref);
3181 WG_TRACE("give up"); 3210 WG_TRACE("give up");
3182 /* 3211 /*
3183 * If a new data packet comes, handshaking will be retried 3212 * If a new data packet comes, handshaking will be retried
3184 * and a new session would be established at that time, 3213 * and a new session would be established at that time,
3185 * however we don't want to send pending packets then. 3214 * however we don't want to send pending packets then.
3186 */ 3215 */
3187 wg_purge_pending_packets(wgp); 3216 wg_purge_pending_packets(wgp);
3188 return; 3217 return;
3189 } 3218 }
3190 3219
3191 /* No response for an initiation message sent, retry handshaking */ 3220 /* No response for an initiation message sent, retry handshaking */
3192 wgs->wgs_state = WGS_STATE_UNKNOWN; 3221 wgs->wgs_state = WGS_STATE_UNKNOWN;
3193 wg_clear_states(wgs); 3222 wg_clear_states(wgs);
3194 wg_put_session(wgs, &psref); 3223 wg_put_session(wgs, &psref);
3195 3224
3196 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3225 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
3197} 3226}
3198 3227
3199static struct wg_peer * 3228static struct wg_peer *
3200wg_alloc_peer(struct wg_softc *wg) 3229wg_alloc_peer(struct wg_softc *wg)
3201{ 3230{
3202 struct wg_peer *wgp; 3231 struct wg_peer *wgp;
3203 3232
3204 wgp = kmem_zalloc(sizeof(*wgp), KM_SLEEP); 3233 wgp = kmem_zalloc(sizeof(*wgp), KM_SLEEP);
3205 3234
3206 wgp->wgp_sc = wg; 3235 wgp->wgp_sc = wg;
3207 wgp->wgp_state = WGP_STATE_INIT; 3236 wgp->wgp_state = WGP_STATE_INIT;
3208 wgp->wgp_q = pcq_create(1024, KM_SLEEP); 3237 wgp->wgp_q = pcq_create(1024, KM_SLEEP);
3209 wgp->wgp_si = softint_establish(SOFTINT_NET, wg_peer_softint, wgp); 3238 wgp->wgp_si = softint_establish(SOFTINT_NET, wg_peer_softint, wgp);
3210 callout_init(&wgp->wgp_rekey_timer, CALLOUT_MPSAFE); 3239 callout_init(&wgp->wgp_rekey_timer, CALLOUT_MPSAFE);
3211 callout_setfunc(&wgp->wgp_rekey_timer, wg_rekey_timer, wgp); 3240 callout_setfunc(&wgp->wgp_rekey_timer, wg_rekey_timer, wgp);
3212 callout_init(&wgp->wgp_handshake_timeout_timer, CALLOUT_MPSAFE); 3241 callout_init(&wgp->wgp_handshake_timeout_timer, CALLOUT_MPSAFE);
3213 callout_setfunc(&wgp->wgp_handshake_timeout_timer, 3242 callout_setfunc(&wgp->wgp_handshake_timeout_timer,
3214 wg_handshake_timeout_timer, wgp); 3243 wg_handshake_timeout_timer, wgp);
3215 callout_init(&wgp->wgp_session_dtor_timer, CALLOUT_MPSAFE); 3244 callout_init(&wgp->wgp_session_dtor_timer, CALLOUT_MPSAFE);
3216 callout_setfunc(&wgp->wgp_session_dtor_timer, 3245 callout_setfunc(&wgp->wgp_session_dtor_timer,
3217 wg_session_dtor_timer, wgp); 3246 wg_session_dtor_timer, wgp);
3218 PSLIST_ENTRY_INIT(wgp, wgp_peerlist_entry); 3247 PSLIST_ENTRY_INIT(wgp, wgp_peerlist_entry);
3219 wgp->wgp_endpoint_changing = false; 3248 wgp->wgp_endpoint_changing = false;
3220 wgp->wgp_endpoint_available = false; 3249 wgp->wgp_endpoint_available = false;
3221 wgp->wgp_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3250 wgp->wgp_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
3222 wgp->wgp_psz = pserialize_create(); 3251 wgp->wgp_psz = pserialize_create();
3223 psref_target_init(&wgp->wgp_psref, wg_psref_class); 3252 psref_target_init(&wgp->wgp_psref, wg_psref_class);
3224 3253
3225 wgp->wgp_endpoint = kmem_zalloc(sizeof(*wgp->wgp_endpoint), KM_SLEEP); 3254 wgp->wgp_endpoint = kmem_zalloc(sizeof(*wgp->wgp_endpoint), KM_SLEEP);
3226 wgp->wgp_endpoint0 = kmem_zalloc(sizeof(*wgp->wgp_endpoint0), KM_SLEEP); 3255 wgp->wgp_endpoint0 = kmem_zalloc(sizeof(*wgp->wgp_endpoint0), KM_SLEEP);
3227 psref_target_init(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3256 psref_target_init(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class);
3228 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3257 psref_target_init(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class);
3229 3258
3230 struct wg_session *wgs; 3259 struct wg_session *wgs;
3231 wgp->wgp_session_stable = 3260 wgp->wgp_session_stable =
3232 kmem_zalloc(sizeof(*wgp->wgp_session_stable), KM_SLEEP); 3261 kmem_zalloc(sizeof(*wgp->wgp_session_stable), KM_SLEEP);
3233 wgp->wgp_session_unstable = 3262 wgp->wgp_session_unstable =
3234 kmem_zalloc(sizeof(*wgp->wgp_session_unstable), KM_SLEEP); 3263 kmem_zalloc(sizeof(*wgp->wgp_session_unstable), KM_SLEEP);
3235 wgs = wgp->wgp_session_stable; 3264 wgs = wgp->wgp_session_stable;
3236 wgs->wgs_peer = wgp; 3265 wgs->wgs_peer = wgp;
3237 wgs->wgs_state = WGS_STATE_UNKNOWN; 3266 wgs->wgs_state = WGS_STATE_UNKNOWN;
3238 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3267 psref_target_init(&wgs->wgs_psref, wg_psref_class);
3239 wgs->wgs_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3268 wgs->wgs_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
3240#ifndef __HAVE_ATOMIC64_LOADSTORE 3269#ifndef __HAVE_ATOMIC64_LOADSTORE
3241 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3270 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET);
3242#endif 3271#endif
3243 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3272 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP);
3244 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_NONE); 3273 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_NONE);
3245 3274
3246 wgs = wgp->wgp_session_unstable; 3275 wgs = wgp->wgp_session_unstable;
3247 wgs->wgs_peer = wgp; 3276 wgs->wgs_peer = wgp;
3248 wgs->wgs_state = WGS_STATE_UNKNOWN; 3277 wgs->wgs_state = WGS_STATE_UNKNOWN;
3249 psref_target_init(&wgs->wgs_psref, wg_psref_class); 3278 psref_target_init(&wgs->wgs_psref, wg_psref_class);
3250 wgs->wgs_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3279 wgs->wgs_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
3251#ifndef __HAVE_ATOMIC64_LOADSTORE 3280#ifndef __HAVE_ATOMIC64_LOADSTORE
3252 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET); 3281 mutex_init(&wgs->wgs_send_counter_lock, MUTEX_DEFAULT, IPL_SOFTNET);
3253#endif 3282#endif
3254 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP); 3283 wgs->wgs_recvwin = kmem_zalloc(sizeof(*wgs->wgs_recvwin), KM_SLEEP);
3255 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_NONE); 3284 mutex_init(&wgs->wgs_recvwin->lock, MUTEX_DEFAULT, IPL_NONE);
3256 3285
3257 return wgp; 3286 return wgp;
3258} 3287}
3259 3288
3260static void 3289static void
3261wg_destroy_peer(struct wg_peer *wgp) 3290wg_destroy_peer(struct wg_peer *wgp)
3262{ 3291{
3263 struct wg_session *wgs; 3292 struct wg_session *wgs;
3264 struct wg_softc *wg = wgp->wgp_sc; 3293 struct wg_softc *wg = wgp->wgp_sc;
 3294 uint32_t index;
 3295 void *garbage;
3265 3296
 3297 /* Prevent new packets from this peer on any source address. */
3266 rw_enter(wg->wg_rwlock, RW_WRITER); 3298 rw_enter(wg->wg_rwlock, RW_WRITER);
3267 for (int i = 0; i < wgp->wgp_n_allowedips; i++) { 3299 for (int i = 0; i < wgp->wgp_n_allowedips; i++) {
3268 struct wg_allowedip *wga = &wgp->wgp_allowedips[i]; 3300 struct wg_allowedip *wga = &wgp->wgp_allowedips[i];
3269 struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family); 3301 struct radix_node_head *rnh = wg_rnh(wg, wga->wga_family);
3270 struct radix_node *rn; 3302 struct radix_node *rn;
3271 3303
3272 KASSERT(rnh != NULL); 3304 KASSERT(rnh != NULL);
3273 rn = rnh->rnh_deladdr(&wga->wga_sa_addr, 3305 rn = rnh->rnh_deladdr(&wga->wga_sa_addr,
3274 &wga->wga_sa_mask, rnh); 3306 &wga->wga_sa_mask, rnh);
3275 if (rn == NULL) { 3307 if (rn == NULL) {
3276 char addrstr[128]; 3308 char addrstr[128];
3277 sockaddr_format(&wga->wga_sa_addr, addrstr, 3309 sockaddr_format(&wga->wga_sa_addr, addrstr,
3278 sizeof(addrstr)); 3310 sizeof(addrstr));
3279 WGLOG(LOG_WARNING, "Couldn't delete %s", addrstr); 3311 WGLOG(LOG_WARNING, "Couldn't delete %s", addrstr);
3280 } 3312 }
3281 } 3313 }
3282 rw_exit(wg->wg_rwlock); 3314 rw_exit(wg->wg_rwlock);
3283 3315
 3316 /* Halt all packet processing and timeouts. */
3284 softint_disestablish(wgp->wgp_si); 3317 softint_disestablish(wgp->wgp_si);
3285 callout_halt(&wgp->wgp_rekey_timer, NULL); 3318 callout_halt(&wgp->wgp_rekey_timer, NULL);
3286 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL); 3319 callout_halt(&wgp->wgp_handshake_timeout_timer, NULL);
3287 callout_halt(&wgp->wgp_session_dtor_timer, NULL); 3320 callout_halt(&wgp->wgp_session_dtor_timer, NULL);
3288 3321
 3322 /* Remove the sessions by index. */
 3323 if ((index = wgp->wgp_session_stable->wgs_sender_index) != 0) {
 3324 thmap_del(wg->wg_sessions_byindex, &index, sizeof index);
 3325 wgp->wgp_session_stable->wgs_sender_index = 0;
 3326 }
 3327 if ((index = wgp->wgp_session_unstable->wgs_sender_index) != 0) {
 3328 thmap_del(wg->wg_sessions_byindex, &index, sizeof index);
 3329 wgp->wgp_session_unstable->wgs_sender_index = 0;
 3330 }
 3331
 3332 /* Wait for all thmap_gets to complete, and GC. */
 3333 garbage = thmap_stage_gc(wg->wg_sessions_byindex);
 3334 mutex_enter(wgp->wgp_lock);
 3335 pserialize_perform(wgp->wgp_psz);
 3336 mutex_exit(wgp->wgp_lock);
 3337 thmap_gc(wg->wg_sessions_byindex, garbage);
 3338
3289 wgs = wgp->wgp_session_unstable; 3339 wgs = wgp->wgp_session_unstable;
3290 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 3340 psref_target_destroy(&wgs->wgs_psref, wg_psref_class);
3291 mutex_obj_free(wgs->wgs_lock); 3341 mutex_obj_free(wgs->wgs_lock);
3292 mutex_destroy(&wgs->wgs_recvwin->lock); 3342 mutex_destroy(&wgs->wgs_recvwin->lock);
3293 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3343 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin));
3294#ifndef __HAVE_ATOMIC64_LOADSTORE 3344#ifndef __HAVE_ATOMIC64_LOADSTORE
3295 mutex_destroy(&wgs->wgs_send_counter_lock); 3345 mutex_destroy(&wgs->wgs_send_counter_lock);
3296#endif 3346#endif
3297 kmem_free(wgs, sizeof(*wgs)); 3347 kmem_free(wgs, sizeof(*wgs));
 3348
3298 wgs = wgp->wgp_session_stable; 3349 wgs = wgp->wgp_session_stable;
3299 psref_target_destroy(&wgs->wgs_psref, wg_psref_class); 3350 psref_target_destroy(&wgs->wgs_psref, wg_psref_class);
3300 mutex_obj_free(wgs->wgs_lock); 3351 mutex_obj_free(wgs->wgs_lock);
3301 mutex_destroy(&wgs->wgs_recvwin->lock); 3352 mutex_destroy(&wgs->wgs_recvwin->lock);
3302 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin)); 3353 kmem_free(wgs->wgs_recvwin, sizeof(*wgs->wgs_recvwin));
3303#ifndef __HAVE_ATOMIC64_LOADSTORE 3354#ifndef __HAVE_ATOMIC64_LOADSTORE
3304 mutex_destroy(&wgs->wgs_send_counter_lock); 3355 mutex_destroy(&wgs->wgs_send_counter_lock);
3305#endif 3356#endif
3306 kmem_free(wgs, sizeof(*wgs)); 3357 kmem_free(wgs, sizeof(*wgs));
3307 3358
3308 psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class); 3359 psref_target_destroy(&wgp->wgp_endpoint->wgsa_psref, wg_psref_class);
3309 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class); 3360 psref_target_destroy(&wgp->wgp_endpoint0->wgsa_psref, wg_psref_class);
3310 kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint)); 3361 kmem_free(wgp->wgp_endpoint, sizeof(*wgp->wgp_endpoint));
3311 kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0)); 3362 kmem_free(wgp->wgp_endpoint0, sizeof(*wgp->wgp_endpoint0));
3312 3363
3313 pserialize_destroy(wgp->wgp_psz); 3364 pserialize_destroy(wgp->wgp_psz);
3314 pcq_destroy(wgp->wgp_q); 3365 pcq_destroy(wgp->wgp_q);
3315 mutex_obj_free(wgp->wgp_lock); 3366 mutex_obj_free(wgp->wgp_lock);
3316 3367
3317 kmem_free(wgp, sizeof(*wgp)); 3368 kmem_free(wgp, sizeof(*wgp));
3318} 3369}
3319 3370
3320static void 3371static void
3321wg_destroy_all_peers(struct wg_softc *wg) 3372wg_destroy_all_peers(struct wg_softc *wg)
3322{ 3373{
3323 struct wg_peer *wgp; 3374 struct wg_peer *wgp, *wgp0 __diagused;
 3375 void *garbage_byname, *garbage_bypubkey;
3324 3376
3325restart: 3377restart:
 3378 garbage_byname = garbage_bypubkey = NULL;
3326 mutex_enter(wg->wg_lock); 3379 mutex_enter(wg->wg_lock);
3327 WG_PEER_WRITER_FOREACH(wgp, wg) { 3380 WG_PEER_WRITER_FOREACH(wgp, wg) {
 3381 if (wgp->wgp_name[0]) {
 3382 wgp0 = thmap_del(wg->wg_peers_byname, wgp->wgp_name,
 3383 strlen(wgp->wgp_name));
 3384 KASSERT(wgp0 == wgp);
 3385 garbage_byname = thmap_stage_gc(wg->wg_peers_byname);
 3386 }
 3387 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey,
 3388 sizeof(wgp->wgp_pubkey));
 3389 KASSERT(wgp0 == wgp);
 3390 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey);
3328 WG_PEER_WRITER_REMOVE(wgp); 3391 WG_PEER_WRITER_REMOVE(wgp);
3329 wg->wg_npeers--; 3392 wg->wg_npeers--;
3330 mutex_enter(wgp->wgp_lock); 3393 mutex_enter(wgp->wgp_lock);
3331 wgp->wgp_state = WGP_STATE_DESTROYING; 3394 wgp->wgp_state = WGP_STATE_DESTROYING;
3332 pserialize_perform(wgp->wgp_psz); 3395 pserialize_perform(wgp->wgp_psz);
3333 mutex_exit(wgp->wgp_lock); 3396 mutex_exit(wgp->wgp_lock);
3334 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3397 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry);
3335 break; 3398 break;
3336 } 3399 }
3337 mutex_exit(wg->wg_lock); 3400 mutex_exit(wg->wg_lock);
3338 3401
3339 if (wgp == NULL) 3402 if (wgp == NULL)
3340 return; 3403 return;
3341 3404
3342 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3405 psref_target_destroy(&wgp->wgp_psref, wg_psref_class);
3343 3406
3344 wg_destroy_peer(wgp); 3407 wg_destroy_peer(wgp);
 3408 thmap_gc(wg->wg_peers_byname, garbage_byname);
 3409 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey);
3345 3410
3346 goto restart; 3411 goto restart;
3347} 3412}
3348 3413
3349static int 3414static int
3350wg_destroy_peer_name(struct wg_softc *wg, const char *name) 3415wg_destroy_peer_name(struct wg_softc *wg, const char *name)
3351{ 3416{
3352 struct wg_peer *wgp; 3417 struct wg_peer *wgp, *wgp0 __diagused;
 3418 void *garbage_byname, *garbage_bypubkey;
3353 3419
3354 mutex_enter(wg->wg_lock); 3420 mutex_enter(wg->wg_lock);
3355 WG_PEER_WRITER_FOREACH(wgp, wg) { 3421 wgp = thmap_del(wg->wg_peers_byname, name, strlen(name));
3356 if (strcmp(wgp->wgp_name, name) == 0) 
3357 break; 
3358 } 
3359 if (wgp != NULL) { 3422 if (wgp != NULL) {
 3423 wgp0 = thmap_del(wg->wg_peers_bypubkey, wgp->wgp_pubkey,
 3424 sizeof(wgp->wgp_pubkey));
 3425 KASSERT(wgp0 == wgp);
 3426 garbage_byname = thmap_stage_gc(wg->wg_peers_byname);
 3427 garbage_bypubkey = thmap_stage_gc(wg->wg_peers_bypubkey);
3360 WG_PEER_WRITER_REMOVE(wgp); 3428 WG_PEER_WRITER_REMOVE(wgp);
3361 wg->wg_npeers--; 3429 wg->wg_npeers--;
3362 mutex_enter(wgp->wgp_lock); 3430 mutex_enter(wgp->wgp_lock);
3363 wgp->wgp_state = WGP_STATE_DESTROYING; 3431 wgp->wgp_state = WGP_STATE_DESTROYING;
3364 pserialize_perform(wgp->wgp_psz); 3432 pserialize_perform(wgp->wgp_psz);
3365 mutex_exit(wgp->wgp_lock); 3433 mutex_exit(wgp->wgp_lock);
3366 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry); 3434 PSLIST_ENTRY_DESTROY(wgp, wgp_peerlist_entry);
3367 } 3435 }
3368 mutex_exit(wg->wg_lock); 3436 mutex_exit(wg->wg_lock);
3369 3437
3370 if (wgp == NULL) 3438 if (wgp == NULL)
3371 return ENOENT; 3439 return ENOENT;
3372 3440
3373 psref_target_destroy(&wgp->wgp_psref, wg_psref_class); 3441 psref_target_destroy(&wgp->wgp_psref, wg_psref_class);
3374 3442
3375 wg_destroy_peer(wgp); 3443 wg_destroy_peer(wgp);
 3444 thmap_gc(wg->wg_peers_byname, garbage_byname);
 3445 thmap_gc(wg->wg_peers_bypubkey, garbage_bypubkey);
3376 3446
3377 return 0; 3447 return 0;
3378} 3448}
3379 3449
3380static int 3450static int
3381wg_if_attach(struct wg_softc *wg) 3451wg_if_attach(struct wg_softc *wg)
3382{ 3452{
3383 int error; 3453 int error;
3384 3454
3385 wg->wg_if.if_addrlen = 0; 3455 wg->wg_if.if_addrlen = 0;
3386 wg->wg_if.if_mtu = WG_MTU; 3456 wg->wg_if.if_mtu = WG_MTU;
3387 wg->wg_if.if_flags = IFF_MULTICAST; 3457 wg->wg_if.if_flags = IFF_MULTICAST;
3388 wg->wg_if.if_extflags = IFEF_NO_LINK_STATE_CHANGE; 3458 wg->wg_if.if_extflags = IFEF_NO_LINK_STATE_CHANGE;
3389 wg->wg_if.if_extflags |= IFEF_MPSAFE; 3459 wg->wg_if.if_extflags |= IFEF_MPSAFE;
3390 wg->wg_if.if_ioctl = wg_ioctl; 3460 wg->wg_if.if_ioctl = wg_ioctl;
3391 wg->wg_if.if_output = wg_output; 3461 wg->wg_if.if_output = wg_output;
3392 wg->wg_if.if_init = wg_init; 3462 wg->wg_if.if_init = wg_init;
3393 wg->wg_if.if_stop = wg_stop; 3463 wg->wg_if.if_stop = wg_stop;
3394 wg->wg_if.if_type = IFT_OTHER; 3464 wg->wg_if.if_type = IFT_OTHER;
3395 wg->wg_if.if_dlt = DLT_NULL; 3465 wg->wg_if.if_dlt = DLT_NULL;
3396 wg->wg_if.if_softc = wg; 3466 wg->wg_if.if_softc = wg;
3397 IFQ_SET_READY(&wg->wg_if.if_snd); 3467 IFQ_SET_READY(&wg->wg_if.if_snd);
3398 3468
3399 error = if_initialize(&wg->wg_if); 3469 error = if_initialize(&wg->wg_if);
3400 if (error != 0) 3470 if (error != 0)
3401 return error; 3471 return error;
3402 3472
3403 if_alloc_sadl(&wg->wg_if); 3473 if_alloc_sadl(&wg->wg_if);
3404 if_register(&wg->wg_if); 3474 if_register(&wg->wg_if);
3405 3475
3406 bpf_attach(&wg->wg_if, DLT_NULL, sizeof(uint32_t)); 3476 bpf_attach(&wg->wg_if, DLT_NULL, sizeof(uint32_t));
3407 3477
3408 return 0; 3478 return 0;
3409} 3479}
3410 3480
3411static int 3481static int
3412wg_clone_create(struct if_clone *ifc, int unit) 3482wg_clone_create(struct if_clone *ifc, int unit)
3413{ 3483{
3414 struct wg_softc *wg; 3484 struct wg_softc *wg;
3415 int error; 3485 int error;
3416 3486
3417 wg = kmem_zalloc(sizeof(struct wg_softc), KM_SLEEP); 3487 wg = kmem_zalloc(sizeof(struct wg_softc), KM_SLEEP);
3418 3488
3419 if_initname(&wg->wg_if, ifc->ifc_name, unit); 3489 if_initname(&wg->wg_if, ifc->ifc_name, unit);
3420 3490
3421 error = wg_worker_init(wg); 3491 error = wg_worker_init(wg);
3422 if (error != 0) { 3492 if (error != 0) {
3423 kmem_free(wg, sizeof(struct wg_softc)); 3493 kmem_free(wg, sizeof(struct wg_softc));
3424 return error; 3494 return error;
3425 } 3495 }
3426 3496
3427 rn_inithead((void **)&wg->wg_rtable_ipv4, 3497 rn_inithead((void **)&wg->wg_rtable_ipv4,
3428 offsetof(struct sockaddr_in, sin_addr) * NBBY); 3498 offsetof(struct sockaddr_in, sin_addr) * NBBY);
3429#ifdef INET6 3499#ifdef INET6
3430 rn_inithead((void **)&wg->wg_rtable_ipv6, 3500 rn_inithead((void **)&wg->wg_rtable_ipv6,
3431 offsetof(struct sockaddr_in6, sin6_addr) * NBBY); 3501 offsetof(struct sockaddr_in6, sin6_addr) * NBBY);
3432#endif 3502#endif
3433 3503
3434 PSLIST_INIT(&wg->wg_peers); 3504 PSLIST_INIT(&wg->wg_peers);
 3505 wg->wg_peers_bypubkey = thmap_create(0, NULL, THMAP_NOCOPY);
 3506 wg->wg_peers_byname = thmap_create(0, NULL, THMAP_NOCOPY);
 3507 wg->wg_sessions_byindex = thmap_create(0, NULL, THMAP_NOCOPY);
3435 wg->wg_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 3508 wg->wg_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
3436 wg->wg_rwlock = rw_obj_alloc(); 3509 wg->wg_rwlock = rw_obj_alloc();
3437 wg->wg_ops = &wg_ops_rumpkernel; 3510 wg->wg_ops = &wg_ops_rumpkernel;
3438 3511
3439 error = wg_if_attach(wg); 3512 error = wg_if_attach(wg);
3440 if (error != 0) { 3513 if (error != 0) {
3441 wg_worker_destroy(wg); 3514 wg_worker_destroy(wg);
3442 if (wg->wg_rtable_ipv4 != NULL) 3515 if (wg->wg_rtable_ipv4 != NULL)
3443 free(wg->wg_rtable_ipv4, M_RTABLE); 3516 free(wg->wg_rtable_ipv4, M_RTABLE);
3444 if (wg->wg_rtable_ipv6 != NULL) 3517 if (wg->wg_rtable_ipv6 != NULL)
3445 free(wg->wg_rtable_ipv6, M_RTABLE); 3518 free(wg->wg_rtable_ipv6, M_RTABLE);
3446 PSLIST_DESTROY(&wg->wg_peers); 3519 PSLIST_DESTROY(&wg->wg_peers);
3447 mutex_obj_free(wg->wg_lock); 3520 mutex_obj_free(wg->wg_lock);
 3521 thmap_destroy(wg->wg_sessions_byindex);
 3522 thmap_destroy(wg->wg_peers_byname);
 3523 thmap_destroy(wg->wg_peers_bypubkey);
3448 kmem_free(wg, sizeof(struct wg_softc)); 3524 kmem_free(wg, sizeof(struct wg_softc));
3449 return error; 3525 return error;
3450 } 3526 }
3451 3527
3452 mutex_enter(&wg_softcs.lock); 3528 mutex_enter(&wg_softcs.lock);
3453 LIST_INSERT_HEAD(&wg_softcs.list, wg, wg_list); 3529 LIST_INSERT_HEAD(&wg_softcs.list, wg, wg_list);
3454 mutex_exit(&wg_softcs.lock); 3530 mutex_exit(&wg_softcs.lock);
3455 3531
3456 return 0; 3532 return 0;
3457} 3533}
3458 3534
3459static int 3535static int
3460wg_clone_destroy(struct ifnet *ifp) 3536wg_clone_destroy(struct ifnet *ifp)
3461{ 3537{
3462 struct wg_softc *wg = container_of(ifp, struct wg_softc, wg_if); 3538 struct wg_softc *wg = container_of(ifp, struct wg_softc, wg_if);
3463 3539
3464 mutex_enter(&wg_softcs.lock); 3540 mutex_enter(&wg_softcs.lock);
3465 LIST_REMOVE(wg, wg_list); 3541 LIST_REMOVE(wg, wg_list);
3466 mutex_exit(&wg_softcs.lock); 3542 mutex_exit(&wg_softcs.lock);
3467 3543
3468#ifdef WG_RUMPKERNEL 3544#ifdef WG_RUMPKERNEL
3469 if (wg_user_mode(wg)) { 3545 if (wg_user_mode(wg)) {
3470 rumpuser_wg_destroy(wg->wg_user); 3546 rumpuser_wg_destroy(wg->wg_user);
3471 wg->wg_user = NULL; 3547 wg->wg_user = NULL;
3472 } 3548 }
3473#endif 3549#endif
3474 3550
3475 bpf_detach(ifp); 3551 bpf_detach(ifp);
3476 if_detach(ifp); 3552 if_detach(ifp);
3477 wg_worker_destroy(wg); 3553 wg_worker_destroy(wg);
3478 wg_destroy_all_peers(wg); 3554 wg_destroy_all_peers(wg);
3479 if (wg->wg_rtable_ipv4 != NULL) 3555 if (wg->wg_rtable_ipv4 != NULL)
3480 free(wg->wg_rtable_ipv4, M_RTABLE); 3556 free(wg->wg_rtable_ipv4, M_RTABLE);
3481 if (wg->wg_rtable_ipv6 != NULL) 3557 if (wg->wg_rtable_ipv6 != NULL)
3482 free(wg->wg_rtable_ipv6, M_RTABLE); 3558 free(wg->wg_rtable_ipv6, M_RTABLE);
3483 3559
3484 PSLIST_DESTROY(&wg->wg_peers); 3560 PSLIST_DESTROY(&wg->wg_peers);
 3561 thmap_destroy(wg->wg_sessions_byindex);
 3562 thmap_destroy(wg->wg_peers_byname);
 3563 thmap_destroy(wg->wg_peers_bypubkey);
3485 mutex_obj_free(wg->wg_lock); 3564 mutex_obj_free(wg->wg_lock);
3486 rw_obj_free(wg->wg_rwlock); 3565 rw_obj_free(wg->wg_rwlock);
3487 3566
3488 kmem_free(wg, sizeof(struct wg_softc)); 3567 kmem_free(wg, sizeof(struct wg_softc));
3489 3568
3490 return 0; 3569 return 0;
3491} 3570}
3492 3571
3493static struct wg_peer * 3572static struct wg_peer *
3494wg_pick_peer_by_sa(struct wg_softc *wg, const struct sockaddr *sa, 3573wg_pick_peer_by_sa(struct wg_softc *wg, const struct sockaddr *sa,
3495 struct psref *psref) 3574 struct psref *psref)
3496{ 3575{
3497 struct radix_node_head *rnh; 3576 struct radix_node_head *rnh;
3498 struct radix_node *rn; 3577 struct radix_node *rn;
3499 struct wg_peer *wgp = NULL; 3578 struct wg_peer *wgp = NULL;
3500 struct wg_allowedip *wga; 3579 struct wg_allowedip *wga;
3501 3580
3502#ifdef WG_DEBUG_LOG 3581#ifdef WG_DEBUG_LOG
3503 char addrstr[128]; 3582 char addrstr[128];
3504 sockaddr_format(sa, addrstr, sizeof(addrstr)); 3583 sockaddr_format(sa, addrstr, sizeof(addrstr));
3505 WG_DLOG("sa=%s\n", addrstr); 3584 WG_DLOG("sa=%s\n", addrstr);
3506#endif 3585#endif
3507 3586
3508 rw_enter(wg->wg_rwlock, RW_READER); 3587 rw_enter(wg->wg_rwlock, RW_READER);
3509 3588
3510 rnh = wg_rnh(wg, sa->sa_family); 3589 rnh = wg_rnh(wg, sa->sa_family);
3511 if (rnh == NULL) 3590 if (rnh == NULL)
3512 goto out; 3591 goto out;
3513 3592
3514 rn = rnh->rnh_matchaddr(sa, rnh); 3593 rn = rnh->rnh_matchaddr(sa, rnh);
3515 if (rn == NULL || (rn->rn_flags & RNF_ROOT) != 0) 3594 if (rn == NULL || (rn->rn_flags & RNF_ROOT) != 0)
3516 goto out; 3595 goto out;
3517 3596
3518 WG_TRACE("success"); 3597 WG_TRACE("success");
3519 3598
3520 wga = container_of(rn, struct wg_allowedip, wga_nodes[0]); 3599 wga = container_of(rn, struct wg_allowedip, wga_nodes[0]);
3521 wgp = wga->wga_peer; 3600 wgp = wga->wga_peer;
3522 wg_get_peer(wgp, psref); 3601 wg_get_peer(wgp, psref);
3523 3602
3524out: 3603out:
3525 rw_exit(wg->wg_rwlock); 3604 rw_exit(wg->wg_rwlock);
3526 return wgp; 3605 return wgp;
3527} 3606}
3528 3607
3529static void 3608static void
3530wg_fill_msg_data(struct wg_softc *wg, struct wg_peer *wgp, 3609wg_fill_msg_data(struct wg_softc *wg, struct wg_peer *wgp,
3531 struct wg_session *wgs, struct wg_msg_data *wgmd) 3610 struct wg_session *wgs, struct wg_msg_data *wgmd)
3532{ 3611{
3533 3612
3534 memset(wgmd, 0, sizeof(*wgmd)); 3613 memset(wgmd, 0, sizeof(*wgmd));
3535 wgmd->wgmd_type = WG_MSG_TYPE_DATA; 3614 wgmd->wgmd_type = WG_MSG_TYPE_DATA;
3536 wgmd->wgmd_receiver = wgs->wgs_receiver_index; 3615 wgmd->wgmd_receiver = wgs->wgs_receiver_index;
3537 /* [W] 5.4.6: msg.counter := Nm^send */ 3616 /* [W] 5.4.6: msg.counter := Nm^send */
3538 /* [W] 5.4.6: Nm^send := Nm^send + 1 */ 3617 /* [W] 5.4.6: Nm^send := Nm^send + 1 */
3539 wgmd->wgmd_counter = wg_session_inc_send_counter(wgs); 3618 wgmd->wgmd_counter = wg_session_inc_send_counter(wgs);
3540 WG_DLOG("counter=%"PRIu64"\n", wgmd->wgmd_counter); 3619 WG_DLOG("counter=%"PRIu64"\n", wgmd->wgmd_counter);
3541} 3620}
3542 3621
3543static int 3622static int
3544wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst, 3623wg_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
3545 const struct rtentry *rt) 3624 const struct rtentry *rt)
3546{ 3625{
3547 struct wg_softc *wg = ifp->if_softc; 3626 struct wg_softc *wg = ifp->if_softc;
3548 int error = 0; 3627 int error = 0;
3549 int bound; 3628 int bound;
3550 struct psref psref; 3629 struct psref psref;
3551 3630
3552 /* TODO make the nest limit configurable via sysctl */ 3631 /* TODO make the nest limit configurable via sysctl */
3553 error = if_tunnel_check_nesting(ifp, m, 1); 3632 error = if_tunnel_check_nesting(ifp, m, 1);
3554 if (error != 0) { 3633 if (error != 0) {
3555 m_freem(m); 3634 m_freem(m);
3556 WGLOG(LOG_ERR, "tunneling loop detected and packet dropped\n"); 3635 WGLOG(LOG_ERR, "tunneling loop detected and packet dropped\n");
3557 return error; 3636 return error;
3558 } 3637 }
3559 3638
3560 bound = curlwp_bind(); 3639 bound = curlwp_bind();
3561 3640
3562 IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family); 3641 IFQ_CLASSIFY(&ifp->if_snd, m, dst->sa_family);
3563 3642
3564 bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT); 3643 bpf_mtap_af(ifp, dst->sa_family, m, BPF_D_OUT);
3565 3644
3566 m->m_flags &= ~(M_BCAST|M_MCAST); 3645 m->m_flags &= ~(M_BCAST|M_MCAST);
3567 3646
3568 struct wg_peer *wgp = wg_pick_peer_by_sa(wg, dst, &psref); 3647 struct wg_peer *wgp = wg_pick_peer_by_sa(wg, dst, &psref);
3569 if (wgp == NULL) { 3648 if (wgp == NULL) {
3570 WG_TRACE("peer not found"); 3649 WG_TRACE("peer not found");
3571 error = EHOSTUNREACH; 3650 error = EHOSTUNREACH;
3572 goto error; 3651 goto error;
3573 } 3652 }
3574 3653
3575 /* Clear checksum-offload flags. */ 3654 /* Clear checksum-offload flags. */
3576 m->m_pkthdr.csum_flags = 0; 3655 m->m_pkthdr.csum_flags = 0;
3577 m->m_pkthdr.csum_data = 0; 3656 m->m_pkthdr.csum_data = 0;
3578 3657
3579 if (!pcq_put(wgp->wgp_q, m)) { 3658 if (!pcq_put(wgp->wgp_q, m)) {
3580 error = ENOBUFS; 3659 error = ENOBUFS;
3581 goto error; 3660 goto error;
3582 } 3661 }
3583 3662
3584 struct psref psref_wgs; 3663 struct psref psref_wgs;
3585 struct wg_session *wgs; 3664 struct wg_session *wgs;
3586 wgs = wg_get_stable_session(wgp, &psref_wgs); 3665 wgs = wg_get_stable_session(wgp, &psref_wgs);
3587 if (wgs->wgs_state == WGS_STATE_ESTABLISHED && 3666 if (wgs->wgs_state == WGS_STATE_ESTABLISHED &&
3588 !wg_session_hit_limits(wgs)) { 3667 !wg_session_hit_limits(wgs)) {
3589 kpreempt_disable(); 3668 kpreempt_disable();
3590 softint_schedule(wgp->wgp_si); 3669 softint_schedule(wgp->wgp_si);
3591 kpreempt_enable(); 3670 kpreempt_enable();
3592 WG_TRACE("softint scheduled"); 3671 WG_TRACE("softint scheduled");
3593 } else { 3672 } else {
3594 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3673 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
3595 WG_TRACE("softint NOT scheduled"); 3674 WG_TRACE("softint NOT scheduled");
3596 } 3675 }
3597 wg_put_session(wgs, &psref_wgs); 3676 wg_put_session(wgs, &psref_wgs);
3598 wg_put_peer(wgp, &psref); 3677 wg_put_peer(wgp, &psref);
3599 3678
3600 return 0; 3679 return 0;
3601 3680
3602error: 3681error:
3603 if (wgp != NULL) 3682 if (wgp != NULL)
3604 wg_put_peer(wgp, &psref); 3683 wg_put_peer(wgp, &psref);
3605 if (m != NULL) 3684 if (m != NULL)
3606 m_freem(m); 3685 m_freem(m);
3607 curlwp_bindx(bound); 3686 curlwp_bindx(bound);
3608 return error; 3687 return error;
3609} 3688}
3610 3689
3611static int 3690static int
3612wg_send_udp(struct wg_peer *wgp, struct mbuf *m) 3691wg_send_udp(struct wg_peer *wgp, struct mbuf *m)
3613{ 3692{
3614 struct psref psref; 3693 struct psref psref;
3615 struct wg_sockaddr *wgsa; 3694 struct wg_sockaddr *wgsa;
3616 int error; 3695 int error;
3617 struct socket *so = wg_get_so_by_peer(wgp); 3696 struct socket *so = wg_get_so_by_peer(wgp);
3618 3697
3619 solock(so); 3698 solock(so);
3620 wgsa = wg_get_endpoint_sa(wgp, &psref); 3699 wgsa = wg_get_endpoint_sa(wgp, &psref);
3621 if (wgsatosa(wgsa)->sa_family == AF_INET) { 3700 if (wgsatosa(wgsa)->sa_family == AF_INET) {
3622 error = udp_send(so, m, wgsatosa(wgsa), NULL, curlwp); 3701 error = udp_send(so, m, wgsatosa(wgsa), NULL, curlwp);
3623 } else { 3702 } else {
3624#ifdef INET6 3703#ifdef INET6
3625 error = udp6_output(sotoin6pcb(so), m, wgsatosin6(wgsa), 3704 error = udp6_output(sotoin6pcb(so), m, wgsatosin6(wgsa),
3626 NULL, curlwp); 3705 NULL, curlwp);
3627#else 3706#else
3628 error = EPROTONOSUPPORT; 3707 error = EPROTONOSUPPORT;
3629#endif 3708#endif
3630 } 3709 }
3631 wg_put_sa(wgp, wgsa, &psref); 3710 wg_put_sa(wgp, wgsa, &psref);
3632 sounlock(so); 3711 sounlock(so);
3633 3712
3634 return error; 3713 return error;
3635} 3714}
3636 3715
3637/* Inspired by pppoe_get_mbuf */ 3716/* Inspired by pppoe_get_mbuf */
3638static struct mbuf * 3717static struct mbuf *
3639wg_get_mbuf(size_t leading_len, size_t len) 3718wg_get_mbuf(size_t leading_len, size_t len)
3640{ 3719{
3641 struct mbuf *m; 3720 struct mbuf *m;
3642 3721
3643 KASSERT(leading_len <= MCLBYTES); 3722 KASSERT(leading_len <= MCLBYTES);
3644 KASSERT(len <= MCLBYTES - leading_len); 3723 KASSERT(len <= MCLBYTES - leading_len);
3645 3724
3646 m = m_gethdr(M_DONTWAIT, MT_DATA); 3725 m = m_gethdr(M_DONTWAIT, MT_DATA);
3647 if (m == NULL) 3726 if (m == NULL)
3648 return NULL; 3727 return NULL;
3649 if (len + leading_len > MHLEN) { 3728 if (len + leading_len > MHLEN) {
3650 m_clget(m, M_DONTWAIT); 3729 m_clget(m, M_DONTWAIT);
3651 if ((m->m_flags & M_EXT) == 0) { 3730 if ((m->m_flags & M_EXT) == 0) {
3652 m_free(m); 3731 m_free(m);
3653 return NULL; 3732 return NULL;
3654 } 3733 }
3655 } 3734 }
3656 m->m_data += leading_len; 3735 m->m_data += leading_len;
3657 m->m_pkthdr.len = m->m_len = len; 3736 m->m_pkthdr.len = m->m_len = len;
3658 3737
3659 return m; 3738 return m;
3660} 3739}
3661 3740
3662static int 3741static int
3663wg_send_data_msg(struct wg_peer *wgp, struct wg_session *wgs, 3742wg_send_data_msg(struct wg_peer *wgp, struct wg_session *wgs,
3664 struct mbuf *m) 3743 struct mbuf *m)
3665{ 3744{
3666 struct wg_softc *wg = wgp->wgp_sc; 3745 struct wg_softc *wg = wgp->wgp_sc;
3667 int error; 3746 int error;
3668 size_t inner_len, padded_len, encrypted_len; 3747 size_t inner_len, padded_len, encrypted_len;
3669 char *padded_buf = NULL; 3748 char *padded_buf = NULL;
3670 size_t mlen; 3749 size_t mlen;
3671 struct wg_msg_data *wgmd; 3750 struct wg_msg_data *wgmd;
3672 bool free_padded_buf = false; 3751 bool free_padded_buf = false;
3673 struct mbuf *n; 3752 struct mbuf *n;
3674 size_t leading_len = max_linkhdr + sizeof(struct ip6_hdr) + 3753 size_t leading_len = max_linkhdr + sizeof(struct ip6_hdr) +
3675 sizeof(struct udphdr); 3754 sizeof(struct udphdr);
3676 3755
3677 mlen = m_length(m); 3756 mlen = m_length(m);
3678 inner_len = mlen; 3757 inner_len = mlen;
3679 padded_len = roundup(mlen, 16); 3758 padded_len = roundup(mlen, 16);
3680 encrypted_len = padded_len + WG_AUTHTAG_LEN; 3759 encrypted_len = padded_len + WG_AUTHTAG_LEN;
3681 WG_DLOG("inner=%lu, padded=%lu, encrypted_len=%lu\n", 3760 WG_DLOG("inner=%lu, padded=%lu, encrypted_len=%lu\n",
3682 inner_len, padded_len, encrypted_len); 3761 inner_len, padded_len, encrypted_len);
3683 if (mlen != 0) { 3762 if (mlen != 0) {
3684 bool success; 3763 bool success;
3685 success = m_ensure_contig(&m, padded_len); 3764 success = m_ensure_contig(&m, padded_len);
3686 if (success) { 3765 if (success) {
3687 padded_buf = mtod(m, char *); 3766 padded_buf = mtod(m, char *);
3688 } else { 3767 } else {
3689 padded_buf = kmem_intr_alloc(padded_len, KM_NOSLEEP); 3768 padded_buf = kmem_intr_alloc(padded_len, KM_NOSLEEP);
3690 if (padded_buf == NULL) { 3769 if (padded_buf == NULL) {
3691 error = ENOBUFS; 3770 error = ENOBUFS;
3692 goto end; 3771 goto end;
3693 } 3772 }
3694 free_padded_buf = true; 3773 free_padded_buf = true;
3695 m_copydata(m, 0, mlen, padded_buf); 3774 m_copydata(m, 0, mlen, padded_buf);
3696 } 3775 }
3697 memset(padded_buf + mlen, 0, padded_len - inner_len); 3776 memset(padded_buf + mlen, 0, padded_len - inner_len);
3698 } 3777 }
3699 3778
3700 n = wg_get_mbuf(leading_len, sizeof(*wgmd) + encrypted_len); 3779 n = wg_get_mbuf(leading_len, sizeof(*wgmd) + encrypted_len);
3701 if (n == NULL) { 3780 if (n == NULL) {
3702 error = ENOBUFS; 3781 error = ENOBUFS;
3703 goto end; 3782 goto end;
3704 } 3783 }
3705 KASSERT(n->m_len >= sizeof(*wgmd)); 3784 KASSERT(n->m_len >= sizeof(*wgmd));
3706 wgmd = mtod(n, struct wg_msg_data *); 3785 wgmd = mtod(n, struct wg_msg_data *);
3707 wg_fill_msg_data(wg, wgp, wgs, wgmd); 3786 wg_fill_msg_data(wg, wgp, wgs, wgmd);
3708 /* [W] 5.4.6: AEAD(Tm^send, Nm^send, P, e) */ 3787 /* [W] 5.4.6: AEAD(Tm^send, Nm^send, P, e) */
3709 wg_algo_aead_enc((char *)wgmd + sizeof(*wgmd), encrypted_len, 3788 wg_algo_aead_enc((char *)wgmd + sizeof(*wgmd), encrypted_len,
3710 wgs->wgs_tkey_send, wgmd->wgmd_counter, padded_buf, padded_len, 3789 wgs->wgs_tkey_send, wgmd->wgmd_counter, padded_buf, padded_len,
3711 NULL, 0); 3790 NULL, 0);
3712 3791
3713 error = wg->wg_ops->send_data_msg(wgp, n); 3792 error = wg->wg_ops->send_data_msg(wgp, n);
3714 if (error == 0) { 3793 if (error == 0) {
3715 struct ifnet *ifp = &wg->wg_if; 3794 struct ifnet *ifp = &wg->wg_if;
3716 if_statadd(ifp, if_obytes, mlen); 3795 if_statadd(ifp, if_obytes, mlen);
3717 if_statinc(ifp, if_opackets); 3796 if_statinc(ifp, if_opackets);
3718 if (wgs->wgs_is_initiator && 3797 if (wgs->wgs_is_initiator &&
3719 wgs->wgs_time_last_data_sent == 0) { 3798 wgs->wgs_time_last_data_sent == 0) {
3720 /* 3799 /*
3721 * [W] 6.2 Transport Message Limits 3800 * [W] 6.2 Transport Message Limits
3722 * "if a peer is the initiator of a current secure 3801 * "if a peer is the initiator of a current secure
3723 * session, WireGuard will send a handshake initiation 3802 * session, WireGuard will send a handshake initiation
3724 * message to begin a new secure session if, after 3803 * message to begin a new secure session if, after
3725 * transmitting a transport data message, the current 3804 * transmitting a transport data message, the current
3726 * secure session is REKEY-AFTER-TIME seconds old," 3805 * secure session is REKEY-AFTER-TIME seconds old,"
3727 */ 3806 */
3728 wg_schedule_rekey_timer(wgp); 3807 wg_schedule_rekey_timer(wgp);
3729 } 3808 }
3730 wgs->wgs_time_last_data_sent = time_uptime; 3809 wgs->wgs_time_last_data_sent = time_uptime;
3731 if (wg_session_get_send_counter(wgs) >= 3810 if (wg_session_get_send_counter(wgs) >=
3732 wg_rekey_after_messages) { 3811 wg_rekey_after_messages) {
3733 /* 3812 /*
3734 * [W] 6.2 Transport Message Limits 3813 * [W] 6.2 Transport Message Limits
3735 * "WireGuard will try to create a new session, by 3814 * "WireGuard will try to create a new session, by
3736 * sending a handshake initiation message (section 3815 * sending a handshake initiation message (section
3737 * 5.4.2), after it has sent REKEY-AFTER-MESSAGES 3816 * 5.4.2), after it has sent REKEY-AFTER-MESSAGES
3738 * transport data messages..." 3817 * transport data messages..."
3739 */ 3818 */
3740 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE); 3819 wg_schedule_peer_task(wgp, WGP_TASK_SEND_INIT_MESSAGE);
3741 } 3820 }
3742 } 3821 }
3743end: 3822end:
3744 m_freem(m); 3823 m_freem(m);
3745 if (free_padded_buf) 3824 if (free_padded_buf)
3746 kmem_intr_free(padded_buf, padded_len); 3825 kmem_intr_free(padded_buf, padded_len);
3747 return error; 3826 return error;
3748} 3827}
3749 3828
3750static void 3829static void
3751wg_input(struct ifnet *ifp, struct mbuf *m, const int af) 3830wg_input(struct ifnet *ifp, struct mbuf *m, const int af)
3752{ 3831{
3753 pktqueue_t *pktq; 3832 pktqueue_t *pktq;
3754 size_t pktlen; 3833 size_t pktlen;
3755 3834
3756 KASSERT(af == AF_INET || af == AF_INET6); 3835 KASSERT(af == AF_INET || af == AF_INET6);
3757 3836
3758 WG_TRACE(""); 3837 WG_TRACE("");
3759 3838
3760 m_set_rcvif(m, ifp); 3839 m_set_rcvif(m, ifp);
3761 pktlen = m->m_pkthdr.len; 3840 pktlen = m->m_pkthdr.len;
3762 3841
3763 bpf_mtap_af(ifp, af, m, BPF_D_IN); 3842 bpf_mtap_af(ifp, af, m, BPF_D_IN);
3764 3843
3765 switch (af) { 3844 switch (af) {
3766 case AF_INET: 3845 case AF_INET:
3767 pktq = ip_pktq; 3846 pktq = ip_pktq;
3768 break; 3847 break;
3769#ifdef INET6 3848#ifdef INET6
3770 case AF_INET6: 3849 case AF_INET6:
3771 pktq = ip6_pktq; 3850 pktq = ip6_pktq;
3772 break; 3851 break;
3773#endif 3852#endif
3774 default: 3853 default:
3775 panic("invalid af=%d", af); 3854 panic("invalid af=%d", af);
3776 } 3855 }
3777 3856
3778 const u_int h = curcpu()->ci_index; 3857 const u_int h = curcpu()->ci_index;
3779 if (__predict_true(pktq_enqueue(pktq, m, h))) { 3858 if (__predict_true(pktq_enqueue(pktq, m, h))) {
3780 if_statadd(ifp, if_ibytes, pktlen); 3859 if_statadd(ifp, if_ibytes, pktlen);
3781 if_statinc(ifp, if_ipackets); 3860 if_statinc(ifp, if_ipackets);
3782 } else { 3861 } else {
3783 m_freem(m); 3862 m_freem(m);
3784 } 3863 }
3785} 3864}
3786 3865
3787static void 3866static void
3788wg_calc_pubkey(uint8_t pubkey[WG_STATIC_KEY_LEN], 3867wg_calc_pubkey(uint8_t pubkey[WG_STATIC_KEY_LEN],
3789 const uint8_t privkey[WG_STATIC_KEY_LEN]) 3868 const uint8_t privkey[WG_STATIC_KEY_LEN])
3790{ 3869{
3791 3870
3792 crypto_scalarmult_base(pubkey, privkey); 3871 crypto_scalarmult_base(pubkey, privkey);
3793} 3872}
3794 3873
3795static int 3874static int
3796wg_rtable_add_route(struct wg_softc *wg, struct wg_allowedip *wga) 3875wg_rtable_add_route(struct wg_softc *wg, struct wg_allowedip *wga)
3797{ 3876{
3798 struct radix_node_head *rnh; 3877 struct radix_node_head *rnh;
3799 struct radix_node *rn; 3878 struct radix_node *rn;
3800 int error = 0; 3879 int error = 0;
3801 3880
3802 rw_enter(wg->wg_rwlock, RW_WRITER); 3881 rw_enter(wg->wg_rwlock, RW_WRITER);
3803 rnh = wg_rnh(wg, wga->wga_family); 3882 rnh = wg_rnh(wg, wga->wga_family);
3804 KASSERT(rnh != NULL); 3883 KASSERT(rnh != NULL);
3805 rn = rnh->rnh_addaddr(&wga->wga_sa_addr, &wga->wga_sa_mask, rnh, 3884 rn = rnh->rnh_addaddr(&wga->wga_sa_addr, &wga->wga_sa_mask, rnh,
3806 wga->wga_nodes); 3885 wga->wga_nodes);
3807 rw_exit(wg->wg_rwlock); 3886 rw_exit(wg->wg_rwlock);
3808 3887
3809 if (rn == NULL) 3888 if (rn == NULL)
3810 error = EEXIST; 3889 error = EEXIST;
3811 3890
3812 return error; 3891 return error;
3813} 3892}
3814 3893
3815static int 3894static int
3816wg_handle_prop_peer(struct wg_softc *wg, prop_dictionary_t peer, 3895wg_handle_prop_peer(struct wg_softc *wg, prop_dictionary_t peer,
3817 struct wg_peer **wgpp) 3896 struct wg_peer **wgpp)
3818{ 3897{
3819 int error = 0; 3898 int error = 0;
3820 const void *pubkey; 3899 const void *pubkey;
3821 size_t pubkey_len; 3900 size_t pubkey_len;
3822 const void *psk; 3901 const void *psk;
3823 size_t psk_len; 3902 size_t psk_len;
3824 const char *name = NULL; 3903 const char *name = NULL;
3825 3904
3826 if (prop_dictionary_get_string(peer, "name", &name)) { 3905 if (prop_dictionary_get_string(peer, "name", &name)) {
3827 if (strlen(name) > WG_PEER_NAME_MAXLEN) { 3906 if (strlen(name) > WG_PEER_NAME_MAXLEN) {
3828 error = EINVAL; 3907 error = EINVAL;
3829 goto out; 3908 goto out;
3830 } 3909 }
3831 } 3910 }
3832 3911
3833 if (!prop_dictionary_get_data(peer, "public_key", 3912 if (!prop_dictionary_get_data(peer, "public_key",
3834 &pubkey, &pubkey_len)) { 3913 &pubkey, &pubkey_len)) {
3835 error = EINVAL; 3914 error = EINVAL;
3836 goto out; 3915 goto out;
3837 } 3916 }
3838#ifdef WG_DEBUG_DUMP 3917#ifdef WG_DEBUG_DUMP
3839 log(LOG_DEBUG, "pubkey=%p, pubkey_len=%lu\n", pubkey, pubkey_len); 3918 log(LOG_DEBUG, "pubkey=%p, pubkey_len=%lu\n", pubkey, pubkey_len);
3840 for (int _i = 0; _i < pubkey_len; _i++) 3919 for (int _i = 0; _i < pubkey_len; _i++)
3841 log(LOG_DEBUG, "%c", ((const char *)pubkey)[_i]); 3920 log(LOG_DEBUG, "%c", ((const char *)pubkey)[_i]);
3842 log(LOG_DEBUG, "\n"); 3921 log(LOG_DEBUG, "\n");
3843#endif 3922#endif
3844 3923
3845 struct wg_peer *wgp = wg_alloc_peer(wg); 3924 struct wg_peer *wgp = wg_alloc_peer(wg);
3846 memcpy(wgp->wgp_pubkey, pubkey, sizeof(wgp->wgp_pubkey)); 3925 memcpy(wgp->wgp_pubkey, pubkey, sizeof(wgp->wgp_pubkey));
3847 if (name != NULL) 3926 if (name != NULL)
3848 strncpy(wgp->wgp_name, name, sizeof(wgp->wgp_name)); 3927 strncpy(wgp->wgp_name, name, sizeof(wgp->wgp_name));
3849 3928
3850 if (prop_dictionary_get_data(peer, "preshared_key", &psk, &psk_len)) { 3929 if (prop_dictionary_get_data(peer, "preshared_key", &psk, &psk_len)) {
3851 if (psk_len != sizeof(wgp->wgp_psk)) { 3930 if (psk_len != sizeof(wgp->wgp_psk)) {
3852 error = EINVAL; 3931 error = EINVAL;
3853 goto out; 3932 goto out;
3854 } 3933 }
3855 memcpy(wgp->wgp_psk, psk, sizeof(wgp->wgp_psk)); 3934 memcpy(wgp->wgp_psk, psk, sizeof(wgp->wgp_psk));
3856 } 3935 }
3857 3936
3858 struct sockaddr_storage sockaddr; 3937 struct sockaddr_storage sockaddr;
3859 const void *addr; 3938 const void *addr;
3860 size_t addr_len; 3939 size_t addr_len;
3861 3940
3862 if (!prop_dictionary_get_data(peer, "endpoint", &addr, &addr_len)) 3941 if (!prop_dictionary_get_data(peer, "endpoint", &addr, &addr_len))
3863 goto skip_endpoint; 3942 goto skip_endpoint;
3864 memcpy(&sockaddr, addr, addr_len); 3943 memcpy(&sockaddr, addr, addr_len);
3865 switch (sockaddr.ss_family) { 3944 switch (sockaddr.ss_family) {
3866 case AF_INET: { 3945 case AF_INET: {
3867 struct sockaddr_in sin; 3946 struct sockaddr_in sin;
3868 sockaddr_copy(sintosa(&sin), sizeof(sin), 3947 sockaddr_copy(sintosa(&sin), sizeof(sin),
3869 (const struct sockaddr *)&sockaddr); 3948 (const struct sockaddr *)&sockaddr);
3870 sockaddr_copy(sintosa(&wgp->wgp_sin), 3949 sockaddr_copy(sintosa(&wgp->wgp_sin),
3871 sizeof(wgp->wgp_sin), (const struct sockaddr *)&sockaddr); 3950 sizeof(wgp->wgp_sin), (const struct sockaddr *)&sockaddr);
3872 char addrstr[128]; 3951 char addrstr[128];
3873 sockaddr_format(sintosa(&sin), addrstr, sizeof(addrstr)); 3952 sockaddr_format(sintosa(&sin), addrstr, sizeof(addrstr));
3874 WG_DLOG("addr=%s\n", addrstr); 3953 WG_DLOG("addr=%s\n", addrstr);
3875 break; 3954 break;
3876 } 3955 }
3877#ifdef INET6 3956#ifdef INET6
3878 case AF_INET6: { 3957 case AF_INET6: {
3879 struct sockaddr_in6 sin6; 3958 struct sockaddr_in6 sin6;
3880 char addrstr[128]; 3959 char addrstr[128];
3881 sockaddr_copy(sintosa(&sin6), sizeof(sin6), 3960 sockaddr_copy(sintosa(&sin6), sizeof(sin6),
3882 (const struct sockaddr *)&sockaddr); 3961 (const struct sockaddr *)&sockaddr);
3883 sockaddr_format(sintosa(&sin6), addrstr, sizeof(addrstr)); 3962 sockaddr_format(sintosa(&sin6), addrstr, sizeof(addrstr));
3884 WG_DLOG("addr=%s\n", addrstr); 3963 WG_DLOG("addr=%s\n", addrstr);
3885 sockaddr_copy(sin6tosa(&wgp->wgp_sin6), 3964 sockaddr_copy(sin6tosa(&wgp->wgp_sin6),
3886 sizeof(wgp->wgp_sin6), (const struct sockaddr *)&sockaddr); 3965 sizeof(wgp->wgp_sin6), (const struct sockaddr *)&sockaddr);
3887 break; 3966 break;
3888 } 3967 }
3889#endif 3968#endif
3890 default: 3969 default:
3891 break; 3970 break;
3892 } 3971 }
3893 wgp->wgp_endpoint_available = true; 3972 wgp->wgp_endpoint_available = true;
3894 3973
3895 prop_array_t allowedips; 3974 prop_array_t allowedips;
3896skip_endpoint: 3975skip_endpoint:
3897 allowedips = prop_dictionary_get(peer, "allowedips"); 3976 allowedips = prop_dictionary_get(peer, "allowedips");
3898 if (allowedips == NULL) 3977 if (allowedips == NULL)
3899 goto skip; 3978 goto skip;
3900 3979
3901 prop_object_iterator_t _it = prop_array_iterator(allowedips); 3980 prop_object_iterator_t _it = prop_array_iterator(allowedips);
3902 prop_dictionary_t prop_allowedip; 3981 prop_dictionary_t prop_allowedip;
3903 int j = 0; 3982 int j = 0;
3904 while ((prop_allowedip = prop_object_iterator_next(_it)) != NULL) { 3983 while ((prop_allowedip = prop_object_iterator_next(_it)) != NULL) {
3905 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 3984 struct wg_allowedip *wga = &wgp->wgp_allowedips[j];
3906 3985
3907 if (!prop_dictionary_get_int(prop_allowedip, "family", 3986 if (!prop_dictionary_get_int(prop_allowedip, "family",
3908 &wga->wga_family)) 3987 &wga->wga_family))
3909 continue; 3988 continue;
3910 if (!prop_dictionary_get_data(prop_allowedip, "ip", 3989 if (!prop_dictionary_get_data(prop_allowedip, "ip",
3911 &addr, &addr_len)) 3990 &addr, &addr_len))
3912 continue; 3991 continue;
3913 if (!prop_dictionary_get_uint8(prop_allowedip, "cidr", 3992 if (!prop_dictionary_get_uint8(prop_allowedip, "cidr",
3914 &wga->wga_cidr)) 3993 &wga->wga_cidr))
3915 continue; 3994 continue;
3916 3995
3917 switch (wga->wga_family) { 3996 switch (wga->wga_family) {
3918 case AF_INET: { 3997 case AF_INET: {
3919 struct sockaddr_in sin; 3998 struct sockaddr_in sin;
3920 char addrstr[128]; 3999 char addrstr[128];
3921 struct in_addr mask; 4000 struct in_addr mask;
3922 struct sockaddr_in sin_mask; 4001 struct sockaddr_in sin_mask;
3923 4002
3924 if (addr_len != sizeof(struct in_addr)) 4003 if (addr_len != sizeof(struct in_addr))
3925 return EINVAL; 4004 return EINVAL;
3926 memcpy(&wga->wga_addr4, addr, addr_len); 4005 memcpy(&wga->wga_addr4, addr, addr_len);
3927 4006
3928 sockaddr_in_init(&sin, (const struct in_addr *)addr, 4007 sockaddr_in_init(&sin, (const struct in_addr *)addr,
3929 0); 4008 0);
3930 sockaddr_copy(&wga->wga_sa_addr, 4009 sockaddr_copy(&wga->wga_sa_addr,
3931 sizeof(sin), sintosa(&sin)); 4010 sizeof(sin), sintosa(&sin));
3932 4011
3933 sockaddr_format(sintosa(&sin), 4012 sockaddr_format(sintosa(&sin),
3934 addrstr, sizeof(addrstr)); 4013 addrstr, sizeof(addrstr));
3935 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4014 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr);
3936 4015
3937 in_len2mask(&mask, wga->wga_cidr); 4016 in_len2mask(&mask, wga->wga_cidr);
3938 sockaddr_in_init(&sin_mask, &mask, 0); 4017 sockaddr_in_init(&sin_mask, &mask, 0);
3939 sockaddr_copy(&wga->wga_sa_mask, 4018 sockaddr_copy(&wga->wga_sa_mask,
3940 sizeof(sin_mask), sintosa(&sin_mask)); 4019 sizeof(sin_mask), sintosa(&sin_mask));
3941 4020
3942 break; 4021 break;
3943 } 4022 }
3944#ifdef INET6 4023#ifdef INET6
3945 case AF_INET6: { 4024 case AF_INET6: {
3946 struct sockaddr_in6 sin6; 4025 struct sockaddr_in6 sin6;
3947 char addrstr[128]; 4026 char addrstr[128];
3948 struct in6_addr mask; 4027 struct in6_addr mask;
3949 struct sockaddr_in6 sin6_mask; 4028 struct sockaddr_in6 sin6_mask;
3950 4029
3951 if (addr_len != sizeof(struct in6_addr)) 4030 if (addr_len != sizeof(struct in6_addr))
3952 return EINVAL; 4031 return EINVAL;
3953 memcpy(&wga->wga_addr6, addr, addr_len); 4032 memcpy(&wga->wga_addr6, addr, addr_len);
3954 4033
3955 sockaddr_in6_init(&sin6, (const struct in6_addr *)addr, 4034 sockaddr_in6_init(&sin6, (const struct in6_addr *)addr,
3956 0, 0, 0); 4035 0, 0, 0);
3957 sockaddr_copy(&wga->wga_sa_addr, 4036 sockaddr_copy(&wga->wga_sa_addr,
3958 sizeof(sin6), sin6tosa(&sin6)); 4037 sizeof(sin6), sin6tosa(&sin6));
3959 4038
3960 sockaddr_format(sin6tosa(&sin6), 4039 sockaddr_format(sin6tosa(&sin6),
3961 addrstr, sizeof(addrstr)); 4040 addrstr, sizeof(addrstr));
3962 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr); 4041 WG_DLOG("addr=%s/%d\n", addrstr, wga->wga_cidr);
3963 4042
3964 in6_prefixlen2mask(&mask, wga->wga_cidr); 4043 in6_prefixlen2mask(&mask, wga->wga_cidr);
3965 sockaddr_in6_init(&sin6_mask, &mask, 0, 0, 0); 4044 sockaddr_in6_init(&sin6_mask, &mask, 0, 0, 0);
3966 sockaddr_copy(&wga->wga_sa_mask, 4045 sockaddr_copy(&wga->wga_sa_mask,
3967 sizeof(sin6_mask), sin6tosa(&sin6_mask)); 4046 sizeof(sin6_mask), sin6tosa(&sin6_mask));
3968 4047
3969 break; 4048 break;
3970 } 4049 }
3971#endif 4050#endif
3972 default: 4051 default:
3973 error = EINVAL; 4052 error = EINVAL;
3974 goto out; 4053 goto out;
3975 } 4054 }
3976 wga->wga_peer = wgp; 4055 wga->wga_peer = wgp;
3977 4056
3978 error = wg_rtable_add_route(wg, wga); 4057 error = wg_rtable_add_route(wg, wga);
3979 if (error != 0) 4058 if (error != 0)
3980 goto out; 4059 goto out;
3981 4060
3982 j++; 4061 j++;
3983 } 4062 }
3984 wgp->wgp_n_allowedips = j; 4063 wgp->wgp_n_allowedips = j;
3985skip: 4064skip:
3986 *wgpp = wgp; 4065 *wgpp = wgp;
3987out: 4066out:
3988 return error; 4067 return error;
3989} 4068}
3990 4069
3991static int 4070static int
3992wg_alloc_prop_buf(char **_buf, struct ifdrv *ifd) 4071wg_alloc_prop_buf(char **_buf, struct ifdrv *ifd)
3993{ 4072{
3994 int error; 4073 int error;
3995 char *buf; 4074 char *buf;
3996 4075
3997 WG_DLOG("buf=%p, len=%lu\n", ifd->ifd_data, ifd->ifd_len); 4076 WG_DLOG("buf=%p, len=%lu\n", ifd->ifd_data, ifd->ifd_len);
3998 buf = kmem_alloc(ifd->ifd_len + 1, KM_SLEEP); 4077 buf = kmem_alloc(ifd->ifd_len + 1, KM_SLEEP);
3999 error = copyin(ifd->ifd_data, buf, ifd->ifd_len); 4078 error = copyin(ifd->ifd_data, buf, ifd->ifd_len);
4000 if (error != 0) 4079 if (error != 0)
4001 return error; 4080 return error;
4002 buf[ifd->ifd_len] = '\0'; 4081 buf[ifd->ifd_len] = '\0';
4003#ifdef WG_DEBUG_DUMP 4082#ifdef WG_DEBUG_DUMP
4004 for (int i = 0; i < ifd->ifd_len; i++) 4083 for (int i = 0; i < ifd->ifd_len; i++)
4005 log(LOG_DEBUG, "%c", buf[i]); 4084 log(LOG_DEBUG, "%c", buf[i]);
4006 log(LOG_DEBUG, "\n"); 4085 log(LOG_DEBUG, "\n");
4007#endif 4086#endif
4008 *_buf = buf; 4087 *_buf = buf;
4009 return 0; 4088 return 0;
4010} 4089}
4011 4090
4012static int 4091static int
4013wg_ioctl_set_private_key(struct wg_softc *wg, struct ifdrv *ifd) 4092wg_ioctl_set_private_key(struct wg_softc *wg, struct ifdrv *ifd)
4014{ 4093{
4015 int error; 4094 int error;
4016 prop_dictionary_t prop_dict; 4095 prop_dictionary_t prop_dict;
4017 char *buf = NULL; 4096 char *buf = NULL;
4018 const void *privkey; 4097 const void *privkey;
4019 size_t privkey_len; 4098 size_t privkey_len;
4020 4099
4021 error = wg_alloc_prop_buf(&buf, ifd); 4100 error = wg_alloc_prop_buf(&buf, ifd);
4022 if (error != 0) 4101 if (error != 0)
4023 return error; 4102 return error;
4024 error = EINVAL; 4103 error = EINVAL;
4025 prop_dict = prop_dictionary_internalize(buf); 4104 prop_dict = prop_dictionary_internalize(buf);
4026 if (prop_dict == NULL) 4105 if (prop_dict == NULL)
4027 goto out; 4106 goto out;
4028 if (!prop_dictionary_get_data(prop_dict, "private_key", 4107 if (!prop_dictionary_get_data(prop_dict, "private_key",
4029 &privkey, &privkey_len)) 4108 &privkey, &privkey_len))
4030 goto out; 4109 goto out;
4031#ifdef WG_DEBUG_DUMP 4110#ifdef WG_DEBUG_DUMP
4032 log(LOG_DEBUG, "privkey=%p, privkey_len=%lu\n", privkey, privkey_len); 4111 log(LOG_DEBUG, "privkey=%p, privkey_len=%lu\n", privkey, privkey_len);
4033 for (int i = 0; i < privkey_len; i++) 4112 for (int i = 0; i < privkey_len; i++)
4034 log(LOG_DEBUG, "%c", ((const char *)privkey)[i]); 4113 log(LOG_DEBUG, "%c", ((const char *)privkey)[i]);
4035 log(LOG_DEBUG, "\n"); 4114 log(LOG_DEBUG, "\n");
4036#endif 4115#endif
4037 if (privkey_len != WG_STATIC_KEY_LEN) 4116 if (privkey_len != WG_STATIC_KEY_LEN)
4038 goto out; 4117 goto out;
4039 memcpy(wg->wg_privkey, privkey, WG_STATIC_KEY_LEN); 4118 memcpy(wg->wg_privkey, privkey, WG_STATIC_KEY_LEN);
4040 wg_calc_pubkey(wg->wg_pubkey, wg->wg_privkey); 4119 wg_calc_pubkey(wg->wg_pubkey, wg->wg_privkey);
4041 error = 0; 4120 error = 0;
4042 4121
4043out: 4122out:
4044 kmem_free(buf, ifd->ifd_len + 1); 4123 kmem_free(buf, ifd->ifd_len + 1);
4045 return error; 4124 return error;
4046} 4125}
4047 4126
4048static int 4127static int
4049wg_ioctl_set_listen_port(struct wg_softc *wg, struct ifdrv *ifd) 4128wg_ioctl_set_listen_port(struct wg_softc *wg, struct ifdrv *ifd)
4050{ 4129{
4051 int error; 4130 int error;
4052 prop_dictionary_t prop_dict; 4131 prop_dictionary_t prop_dict;
4053 char *buf = NULL; 4132 char *buf = NULL;
4054 uint16_t port; 4133 uint16_t port;
4055 4134
4056 error = wg_alloc_prop_buf(&buf, ifd); 4135 error = wg_alloc_prop_buf(&buf, ifd);
4057 if (error != 0) 4136 if (error != 0)
4058 return error; 4137 return error;
4059 error = EINVAL; 4138 error = EINVAL;
4060 prop_dict = prop_dictionary_internalize(buf); 4139 prop_dict = prop_dictionary_internalize(buf);
4061 if (prop_dict == NULL) 4140 if (prop_dict == NULL)
4062 goto out; 4141 goto out;
4063 if (!prop_dictionary_get_uint16(prop_dict, "listen_port", &port)) 4142 if (!prop_dictionary_get_uint16(prop_dict, "listen_port", &port))
4064 goto out; 4143 goto out;
4065 4144
4066 error = wg->wg_ops->bind_port(wg, (uint16_t)port); 4145 error = wg->wg_ops->bind_port(wg, (uint16_t)port);
4067 4146
4068out: 4147out:
4069 kmem_free(buf, ifd->ifd_len + 1); 4148 kmem_free(buf, ifd->ifd_len + 1);
4070 return error; 4149 return error;
4071} 4150}
4072 4151
4073static int 4152static int
4074wg_ioctl_add_peer(struct wg_softc *wg, struct ifdrv *ifd) 4153wg_ioctl_add_peer(struct wg_softc *wg, struct ifdrv *ifd)
4075{ 4154{
4076 int error; 4155 int error;
4077 prop_dictionary_t prop_dict; 4156 prop_dictionary_t prop_dict;
4078 char *buf = NULL; 4157 char *buf = NULL;
4079 struct wg_peer *wgp = NULL; 4158 struct wg_peer *wgp = NULL, *wgp0 __diagused;
4080 4159
4081 error = wg_alloc_prop_buf(&buf, ifd); 4160 error = wg_alloc_prop_buf(&buf, ifd);
4082 if (error != 0) 4161 if (error != 0)
4083 return error; 4162 return error;
4084 error = EINVAL; 4163 error = EINVAL;
4085 prop_dict = prop_dictionary_internalize(buf); 4164 prop_dict = prop_dictionary_internalize(buf);
4086 if (prop_dict == NULL) 4165 if (prop_dict == NULL)
4087 goto out; 4166 goto out;
4088 4167
4089 error = wg_handle_prop_peer(wg, prop_dict, &wgp); 4168 error = wg_handle_prop_peer(wg, prop_dict, &wgp);
4090 if (error != 0) 4169 if (error != 0)
4091 goto out; 4170 goto out;
4092 4171
4093 mutex_enter(wg->wg_lock); 4172 mutex_enter(wg->wg_lock);
 4173 if (thmap_get(wg->wg_peers_bypubkey, wgp->wgp_pubkey,
 4174 sizeof(wgp->wgp_pubkey)) != NULL ||
 4175 (wgp->wgp_name[0] &&
 4176 thmap_get(wg->wg_peers_byname, wgp->wgp_name,
 4177 strlen(wgp->wgp_name)) != NULL)) {
 4178 mutex_exit(wg->wg_lock);
 4179 wg_destroy_peer(wgp);
 4180 error = EEXIST;
 4181 goto out;
 4182 }
 4183 wgp0 = thmap_put(wg->wg_peers_bypubkey, wgp->wgp_pubkey,
 4184 sizeof(wgp->wgp_pubkey), wgp);
 4185 KASSERT(wgp0 == wgp);
 4186 if (wgp->wgp_name[0]) {
 4187 wgp0 = thmap_put(wg->wg_peers_byname, wgp->wgp_name,
 4188 strlen(wgp->wgp_name), wgp);
 4189 KASSERT(wgp0 == wgp);
 4190 }
4094 WG_PEER_WRITER_INSERT_HEAD(wgp, wg); 4191 WG_PEER_WRITER_INSERT_HEAD(wgp, wg);
4095 wg->wg_npeers++; 4192 wg->wg_npeers++;
4096 mutex_exit(wg->wg_lock); 4193 mutex_exit(wg->wg_lock);
4097 4194
4098out: 4195out:
4099 kmem_free(buf, ifd->ifd_len + 1); 4196 kmem_free(buf, ifd->ifd_len + 1);
4100 return error; 4197 return error;
4101} 4198}
4102 4199
4103static int 4200static int
4104wg_ioctl_delete_peer(struct wg_softc *wg, struct ifdrv *ifd) 4201wg_ioctl_delete_peer(struct wg_softc *wg, struct ifdrv *ifd)
4105{ 4202{
4106 int error; 4203 int error;
4107 prop_dictionary_t prop_dict; 4204 prop_dictionary_t prop_dict;
4108 char *buf = NULL; 4205 char *buf = NULL;
4109 const char *name; 4206 const char *name;
4110 4207
4111 error = wg_alloc_prop_buf(&buf, ifd); 4208 error = wg_alloc_prop_buf(&buf, ifd);
4112 if (error != 0) 4209 if (error != 0)
4113 return error; 4210 return error;
4114 error = EINVAL; 4211 error = EINVAL;
4115 prop_dict = prop_dictionary_internalize(buf); 4212 prop_dict = prop_dictionary_internalize(buf);
4116 if (prop_dict == NULL) 4213 if (prop_dict == NULL)
4117 goto out; 4214 goto out;
4118 4215
4119 if (!prop_dictionary_get_string(prop_dict, "name", &name)) 4216 if (!prop_dictionary_get_string(prop_dict, "name", &name))
4120 goto out; 4217 goto out;
4121 if (strlen(name) > WG_PEER_NAME_MAXLEN) 4218 if (strlen(name) > WG_PEER_NAME_MAXLEN)
4122 goto out; 4219 goto out;
4123 4220
4124 error = wg_destroy_peer_name(wg, name); 4221 error = wg_destroy_peer_name(wg, name);
4125out: 4222out:
4126 kmem_free(buf, ifd->ifd_len + 1); 4223 kmem_free(buf, ifd->ifd_len + 1);
4127 return error; 4224 return error;
4128} 4225}
4129 4226
4130static int 4227static int
4131wg_ioctl_get(struct wg_softc *wg, struct ifdrv *ifd) 4228wg_ioctl_get(struct wg_softc *wg, struct ifdrv *ifd)
4132{ 4229{
4133 int error = ENOMEM; 4230 int error = ENOMEM;
4134 prop_dictionary_t prop_dict; 4231 prop_dictionary_t prop_dict;
4135 prop_array_t peers = NULL; 4232 prop_array_t peers = NULL;
4136 char *buf; 4233 char *buf;
4137 struct wg_peer *wgp; 4234 struct wg_peer *wgp;
4138 int s, i; 4235 int s, i;
4139 4236
4140 prop_dict = prop_dictionary_create(); 4237 prop_dict = prop_dictionary_create();
4141 if (prop_dict == NULL) 4238 if (prop_dict == NULL)
4142 goto error; 4239 goto error;
4143 4240
4144 if (!prop_dictionary_set_data(prop_dict, "private_key", wg->wg_privkey, 4241 if (!prop_dictionary_set_data(prop_dict, "private_key", wg->wg_privkey,
4145 WG_STATIC_KEY_LEN)) 4242 WG_STATIC_KEY_LEN))
4146 goto error; 4243 goto error;
4147 4244
4148 if (wg->wg_listen_port != 0) { 4245 if (wg->wg_listen_port != 0) {
4149 if (!prop_dictionary_set_uint16(prop_dict, "listen_port", 4246 if (!prop_dictionary_set_uint16(prop_dict, "listen_port",
4150 wg->wg_listen_port)) 4247 wg->wg_listen_port))
4151 goto error; 4248 goto error;
4152 } 4249 }
4153 4250
4154 if (wg->wg_npeers == 0) 4251 if (wg->wg_npeers == 0)
4155 goto skip_peers; 4252 goto skip_peers;
4156 4253
4157 peers = prop_array_create(); 4254 peers = prop_array_create();
4158 if (peers == NULL) 4255 if (peers == NULL)
4159 goto error; 4256 goto error;
4160 4257
4161 s = pserialize_read_enter(); 4258 s = pserialize_read_enter();
4162 i = 0; 4259 i = 0;
4163 WG_PEER_READER_FOREACH(wgp, wg) { 4260 WG_PEER_READER_FOREACH(wgp, wg) {
4164 struct psref psref; 4261 struct psref psref;
4165 prop_dictionary_t prop_peer; 4262 prop_dictionary_t prop_peer;
4166 4263
4167 wg_get_peer(wgp, &psref); 4264 wg_get_peer(wgp, &psref);
4168 pserialize_read_exit(s); 4265 pserialize_read_exit(s);
4169 4266
4170 prop_peer = prop_dictionary_create(); 4267 prop_peer = prop_dictionary_create();
4171 if (prop_peer == NULL) 4268 if (prop_peer == NULL)
4172 goto next; 4269 goto next;
4173 4270
4174 if (strlen(wgp->wgp_name) > 0) { 4271 if (strlen(wgp->wgp_name) > 0) {
4175 if (!prop_dictionary_set_string(prop_peer, "name", 4272 if (!prop_dictionary_set_string(prop_peer, "name",
4176 wgp->wgp_name)) 4273 wgp->wgp_name))
4177 goto next; 4274 goto next;
4178 } 4275 }
4179 4276
4180 if (!prop_dictionary_set_data(prop_peer, "public_key", 4277 if (!prop_dictionary_set_data(prop_peer, "public_key",
4181 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey))) 4278 wgp->wgp_pubkey, sizeof(wgp->wgp_pubkey)))
4182 goto next; 4279 goto next;
4183 4280
4184 uint8_t psk_zero[WG_PRESHARED_KEY_LEN] = {0}; 4281 uint8_t psk_zero[WG_PRESHARED_KEY_LEN] = {0};
4185 if (!consttime_memequal(wgp->wgp_psk, psk_zero, 4282 if (!consttime_memequal(wgp->wgp_psk, psk_zero,
4186 sizeof(wgp->wgp_psk))) { 4283 sizeof(wgp->wgp_psk))) {
4187 if (!prop_dictionary_set_data(prop_peer, 4284 if (!prop_dictionary_set_data(prop_peer,
4188 "preshared_key", 4285 "preshared_key",
4189 wgp->wgp_psk, sizeof(wgp->wgp_psk))) 4286 wgp->wgp_psk, sizeof(wgp->wgp_psk)))
4190 goto next; 4287 goto next;
4191 } 4288 }
4192 4289
4193 switch (wgp->wgp_sa.sa_family) { 4290 switch (wgp->wgp_sa.sa_family) {
4194 case AF_INET: 4291 case AF_INET:
4195 if (!prop_dictionary_set_data(prop_peer, "endpoint", 4292 if (!prop_dictionary_set_data(prop_peer, "endpoint",
4196 &wgp->wgp_sin, sizeof(wgp->wgp_sin))) 4293 &wgp->wgp_sin, sizeof(wgp->wgp_sin)))
4197 goto next; 4294 goto next;
4198 break; 4295 break;
4199#ifdef INET6 4296#ifdef INET6
4200 case AF_INET6: 4297 case AF_INET6:
4201 if (!prop_dictionary_set_data(prop_peer, "endpoint", 4298 if (!prop_dictionary_set_data(prop_peer, "endpoint",
4202 &wgp->wgp_sin6, sizeof(wgp->wgp_sin6))) 4299 &wgp->wgp_sin6, sizeof(wgp->wgp_sin6)))
4203 goto next; 4300 goto next;
4204 break; 4301 break;
4205#endif 4302#endif
4206 } 4303 }
4207 4304
4208 const struct timespec *t = &wgp->wgp_last_handshake_time; 4305 const struct timespec *t = &wgp->wgp_last_handshake_time;
4209 4306
4210 if (!prop_dictionary_set_uint64(prop_peer, 4307 if (!prop_dictionary_set_uint64(prop_peer,
4211 "last_handshake_time_sec", t->tv_sec)) 4308 "last_handshake_time_sec", t->tv_sec))
4212 goto next; 4309 goto next;
4213 if (!prop_dictionary_set_uint32(prop_peer, 4310 if (!prop_dictionary_set_uint32(prop_peer,
4214 "last_handshake_time_nsec", t->tv_nsec)) 4311 "last_handshake_time_nsec", t->tv_nsec))
4215 goto next; 4312 goto next;
4216 4313
4217 if (wgp->wgp_n_allowedips == 0) 4314 if (wgp->wgp_n_allowedips == 0)
4218 goto skip_allowedips; 4315 goto skip_allowedips;
4219 4316
4220 prop_array_t allowedips = prop_array_create(); 4317 prop_array_t allowedips = prop_array_create();
4221 if (allowedips == NULL) 4318 if (allowedips == NULL)
4222 goto next; 4319 goto next;
4223 for (int j = 0; j < wgp->wgp_n_allowedips; j++) { 4320 for (int j = 0; j < wgp->wgp_n_allowedips; j++) {
4224 struct wg_allowedip *wga = &wgp->wgp_allowedips[j]; 4321 struct wg_allowedip *wga = &wgp->wgp_allowedips[j];
4225 prop_dictionary_t prop_allowedip; 4322 prop_dictionary_t prop_allowedip;
4226 4323
4227 prop_allowedip = prop_dictionary_create(); 4324 prop_allowedip = prop_dictionary_create();
4228 if (prop_allowedip == NULL) 4325 if (prop_allowedip == NULL)
4229 break; 4326 break;
4230 4327
4231 if (!prop_dictionary_set_int(prop_allowedip, "family", 4328 if (!prop_dictionary_set_int(prop_allowedip, "family",
4232 wga->wga_family)) 4329 wga->wga_family))
4233 goto _next; 4330 goto _next;
4234 if (!prop_dictionary_set_uint8(prop_allowedip, "cidr", 4331 if (!prop_dictionary_set_uint8(prop_allowedip, "cidr",
4235 wga->wga_cidr)) 4332 wga->wga_cidr))
4236 goto _next; 4333 goto _next;
4237 4334
4238 switch (wga->wga_family) { 4335 switch (wga->wga_family) {
4239 case AF_INET: 4336 case AF_INET:
4240 if (!prop_dictionary_set_data(prop_allowedip, 4337 if (!prop_dictionary_set_data(prop_allowedip,
4241 "ip", &wga->wga_addr4, 4338 "ip", &wga->wga_addr4,
4242 sizeof(wga->wga_addr4))) 4339 sizeof(wga->wga_addr4)))
4243 goto _next; 4340 goto _next;
4244 break; 4341 break;
4245#ifdef INET6 4342#ifdef INET6
4246 case AF_INET6: 4343 case AF_INET6:
4247 if (!prop_dictionary_set_data(prop_allowedip, 4344 if (!prop_dictionary_set_data(prop_allowedip,
4248 "ip", &wga->wga_addr6, 4345 "ip", &wga->wga_addr6,
4249 sizeof(wga->wga_addr6))) 4346 sizeof(wga->wga_addr6)))
4250 goto _next; 4347 goto _next;
4251 break; 4348 break;
4252#endif 4349#endif
4253 default: 4350 default:
4254 break; 4351 break;
4255 } 4352 }
4256 prop_array_set(allowedips, j, prop_allowedip); 4353 prop_array_set(allowedips, j, prop_allowedip);
4257 _next: 4354 _next:
4258 prop_object_release(prop_allowedip); 4355 prop_object_release(prop_allowedip);
4259 } 4356 }
4260 prop_dictionary_set(prop_peer, "allowedips", allowedips); 4357 prop_dictionary_set(prop_peer, "allowedips", allowedips);
4261 prop_object_release(allowedips); 4358 prop_object_release(allowedips);
4262 4359
4263 skip_allowedips: 4360 skip_allowedips:
4264 4361
4265 prop_array_set(peers, i, prop_peer); 4362 prop_array_set(peers, i, prop_peer);
4266 next: 4363 next:
4267 if (prop_peer) 4364 if (prop_peer)
4268 prop_object_release(prop_peer); 4365 prop_object_release(prop_peer);
4269 i++; 4366 i++;
4270 4367
4271 s = pserialize_read_enter(); 4368 s = pserialize_read_enter();
4272 wg_put_peer(wgp, &psref); 4369 wg_put_peer(wgp, &psref);
4273 } 4370 }
4274 pserialize_read_exit(s); 4371 pserialize_read_exit(s);
4275 4372
4276 prop_dictionary_set(prop_dict, "peers", peers); 4373 prop_dictionary_set(prop_dict, "peers", peers);
4277 prop_object_release(peers); 4374 prop_object_release(peers);
4278 peers = NULL; 4375 peers = NULL;
4279 4376
4280skip_peers: 4377skip_peers:
4281 buf = prop_dictionary_externalize(prop_dict); 4378 buf = prop_dictionary_externalize(prop_dict);
4282 if (buf == NULL) 4379 if (buf == NULL)
4283 goto error; 4380 goto error;
4284 if (ifd->ifd_len < (strlen(buf) + 1)) { 4381 if (ifd->ifd_len < (strlen(buf) + 1)) {
4285 error = EINVAL; 4382 error = EINVAL;
4286 goto error; 4383 goto error;
4287 } 4384 }
4288 error = copyout(buf, ifd->ifd_data, strlen(buf) + 1); 4385 error = copyout(buf, ifd->ifd_data, strlen(buf) + 1);
4289 4386
4290 free(buf, 0); 4387 free(buf, 0);
4291error: 4388error:
4292 if (peers != NULL) 4389 if (peers != NULL)
4293 prop_object_release(peers); 4390 prop_object_release(peers);
4294 if (prop_dict != NULL) 4391 if (prop_dict != NULL)
4295 prop_object_release(prop_dict); 4392 prop_object_release(prop_dict);
4296 4393
4297 return error; 4394 return error;
4298} 4395}
4299 4396
4300static int 4397static int
4301wg_ioctl(struct ifnet *ifp, u_long cmd, void *data) 4398wg_ioctl(struct ifnet *ifp, u_long cmd, void *data)
4302{ 4399{
4303 struct wg_softc *wg = ifp->if_softc; 4400 struct wg_softc *wg = ifp->if_softc;
4304 struct ifreq *ifr = data; 4401 struct ifreq *ifr = data;
4305 struct ifaddr *ifa = data; 4402 struct ifaddr *ifa = data;
4306 struct ifdrv *ifd = data; 4403 struct ifdrv *ifd = data;
4307 int error = 0; 4404 int error = 0;
4308 4405
4309 switch (cmd) { 4406 switch (cmd) {
4310 case SIOCINITIFADDR: 4407 case SIOCINITIFADDR:
4311 if (ifa->ifa_addr->sa_family != AF_LINK && 4408 if (ifa->ifa_addr->sa_family != AF_LINK &&
4312 (ifp->if_flags & (IFF_UP | IFF_RUNNING)) != 4409 (ifp->if_flags & (IFF_UP | IFF_RUNNING)) !=
4313 (IFF_UP | IFF_RUNNING)) { 4410 (IFF_UP | IFF_RUNNING)) {
4314 ifp->if_flags |= IFF_UP; 4411 ifp->if_flags |= IFF_UP;
4315 error = ifp->if_init(ifp); 4412 error = ifp->if_init(ifp);
4316 } 4413 }
4317 return error; 4414 return error;
4318 case SIOCADDMULTI: 4415 case SIOCADDMULTI:
4319 case SIOCDELMULTI: 4416 case SIOCDELMULTI:
4320 switch (ifr->ifr_addr.sa_family) { 4417 switch (ifr->ifr_addr.sa_family) {
4321 case AF_INET: /* IP supports Multicast */ 4418 case AF_INET: /* IP supports Multicast */
4322 break; 4419 break;
4323#ifdef INET6 4420#ifdef INET6
4324 case AF_INET6: /* IP6 supports Multicast */ 4421 case AF_INET6: /* IP6 supports Multicast */
4325 break; 4422 break;
4326#endif 4423#endif
4327 default: /* Other protocols doesn't support Multicast */ 4424 default: /* Other protocols doesn't support Multicast */
4328 error = EAFNOSUPPORT; 4425 error = EAFNOSUPPORT;
4329 break; 4426 break;
4330 } 4427 }
4331 return error; 4428 return error;
4332 case SIOCSDRVSPEC: 4429 case SIOCSDRVSPEC:
4333 switch (ifd->ifd_cmd) { 4430 switch (ifd->ifd_cmd) {
4334 case WG_IOCTL_SET_PRIVATE_KEY: 4431 case WG_IOCTL_SET_PRIVATE_KEY:
4335 error = wg_ioctl_set_private_key(wg, ifd); 4432 error = wg_ioctl_set_private_key(wg, ifd);
4336 break; 4433 break;
4337 case WG_IOCTL_SET_LISTEN_PORT: 4434 case WG_IOCTL_SET_LISTEN_PORT:
4338 error = wg_ioctl_set_listen_port(wg, ifd); 4435 error = wg_ioctl_set_listen_port(wg, ifd);
4339 break; 4436 break;
4340 case WG_IOCTL_ADD_PEER: 4437 case WG_IOCTL_ADD_PEER:
4341 error = wg_ioctl_add_peer(wg, ifd); 4438 error = wg_ioctl_add_peer(wg, ifd);
4342 break; 4439 break;
4343 case WG_IOCTL_DELETE_PEER: 4440 case WG_IOCTL_DELETE_PEER:
4344 error = wg_ioctl_delete_peer(wg, ifd); 4441 error = wg_ioctl_delete_peer(wg, ifd);
4345 break; 4442 break;
4346 default: 4443 default:
4347 error = EINVAL; 4444 error = EINVAL;
4348 break; 4445 break;
4349 } 4446 }
4350 return error; 4447 return error;
4351 case SIOCGDRVSPEC: 4448 case SIOCGDRVSPEC:
4352 return wg_ioctl_get(wg, ifd); 4449 return wg_ioctl_get(wg, ifd);
4353 case SIOCSIFFLAGS: 4450 case SIOCSIFFLAGS:
4354 if ((error = ifioctl_common(ifp, cmd, data)) != 0) 4451 if ((error = ifioctl_common(ifp, cmd, data)) != 0)
4355 break; 4452 break;
4356 switch (ifp->if_flags & (IFF_UP|IFF_RUNNING)) { 4453 switch (ifp->if_flags & (IFF_UP|IFF_RUNNING)) {
4357 case IFF_RUNNING: 4454 case IFF_RUNNING:
4358 /* 4455 /*
4359 * If interface is marked down and it is running, 4456 * If interface is marked down and it is running,
4360 * then stop and disable it. 4457 * then stop and disable it.
4361 */ 4458 */
4362 (*ifp->if_stop)(ifp, 1); 4459 (*ifp->if_stop)(ifp, 1);
4363 break; 4460 break;
4364 case IFF_UP: 4461 case IFF_UP:
4365 /* 4462 /*
4366 * If interface is marked up and it is stopped, then 4463 * If interface is marked up and it is stopped, then
4367 * start it. 4464 * start it.
4368 */ 4465 */
4369 error = (*ifp->if_init)(ifp); 4466 error = (*ifp->if_init)(ifp);
4370 break; 4467 break;
4371 default: 4468 default:
4372 break; 4469 break;
4373 } 4470 }
4374 return error; 4471 return error;
4375#ifdef WG_RUMPKERNEL 4472#ifdef WG_RUMPKERNEL
4376 case SIOCSLINKSTR: 4473 case SIOCSLINKSTR:
4377 error = wg_ioctl_linkstr(wg, ifd); 4474 error = wg_ioctl_linkstr(wg, ifd);
4378 if (error == 0) 4475 if (error == 0)
4379 wg->wg_ops = &wg_ops_rumpuser; 4476 wg->wg_ops = &wg_ops_rumpuser;
4380 return error; 4477 return error;
4381#endif 4478#endif
4382 default: 4479 default:
4383 break; 4480 break;
4384 } 4481 }
4385 4482
4386 error = ifioctl_common(ifp, cmd, data); 4483 error = ifioctl_common(ifp, cmd, data);
4387 4484
4388#ifdef WG_RUMPKERNEL 4485#ifdef WG_RUMPKERNEL
4389 if (!wg_user_mode(wg)) 4486 if (!wg_user_mode(wg))
4390 return error; 4487 return error;
4391 4488
4392 /* Do the same to the corresponding tun device on the host */ 4489 /* Do the same to the corresponding tun device on the host */
4393 /* 4490 /*
4394 * XXX Actually the command has not been handled yet. It 4491 * XXX Actually the command has not been handled yet. It
4395 * will be handled via pr_ioctl form doifioctl later. 4492 * will be handled via pr_ioctl form doifioctl later.
4396 */ 4493 */
4397 switch (cmd) { 4494 switch (cmd) {
4398 case SIOCAIFADDR: 4495 case SIOCAIFADDR:
4399 case SIOCDIFADDR: { 4496 case SIOCDIFADDR: {
4400 struct in_aliasreq _ifra = *(const struct in_aliasreq *)data; 4497 struct in_aliasreq _ifra = *(const struct in_aliasreq *)data;
4401 struct in_aliasreq *ifra = &_ifra; 4498 struct in_aliasreq *ifra = &_ifra;
4402 KASSERT(error == ENOTTY); 4499 KASSERT(error == ENOTTY);
4403 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4500 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user),
4404 IFNAMSIZ); 4501 IFNAMSIZ);
4405 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET); 4502 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET);
4406 if (error == 0) 4503 if (error == 0)
4407 error = ENOTTY; 4504 error = ENOTTY;
4408 break; 4505 break;
4409 } 4506 }
4410#ifdef INET6 4507#ifdef INET6
4411 case SIOCAIFADDR_IN6: 4508 case SIOCAIFADDR_IN6:
4412 case SIOCDIFADDR_IN6: { 4509 case SIOCDIFADDR_IN6: {
4413 struct in6_aliasreq _ifra = *(const struct in6_aliasreq *)data; 4510 struct in6_aliasreq _ifra = *(const struct in6_aliasreq *)data;
4414 struct in6_aliasreq *ifra = &_ifra; 4511 struct in6_aliasreq *ifra = &_ifra;
4415 KASSERT(error == ENOTTY); 4512 KASSERT(error == ENOTTY);
4416 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user), 4513 strncpy(ifra->ifra_name, rumpuser_wg_get_tunname(wg->wg_user),
4417 IFNAMSIZ); 4514 IFNAMSIZ);
4418 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET6); 4515 error = rumpuser_wg_ioctl(wg->wg_user, cmd, ifra, AF_INET6);
4419 if (error == 0) 4516 if (error == 0)
4420 error = ENOTTY; 4517 error = ENOTTY;
4421 break; 4518 break;
4422 } 4519 }
4423#endif 4520#endif
4424 } 4521 }
4425#endif /* WG_RUMPKERNEL */ 4522#endif /* WG_RUMPKERNEL */
4426 4523
4427 return error; 4524 return error;
4428} 4525}
4429 4526
4430static int 4527static int
4431wg_init(struct ifnet *ifp) 4528wg_init(struct ifnet *ifp)
4432{ 4529{
4433 4530
4434 ifp->if_flags |= IFF_RUNNING; 4531 ifp->if_flags |= IFF_RUNNING;
4435 4532
4436 /* TODO flush pending packets. */ 4533 /* TODO flush pending packets. */
4437 return 0; 4534 return 0;
4438} 4535}
4439 4536
4440static void 4537static void
4441wg_stop(struct ifnet *ifp, int disable) 4538wg_stop(struct ifnet *ifp, int disable)
4442{ 4539{
4443 4540
4444 KASSERT((ifp->if_flags & IFF_RUNNING) != 0); 4541 KASSERT((ifp->if_flags & IFF_RUNNING) != 0);
4445 ifp->if_flags &= ~IFF_RUNNING; 4542 ifp->if_flags &= ~IFF_RUNNING;
4446 4543
4447 /* Need to do something? */ 4544 /* Need to do something? */
4448} 4545}
4449 4546
4450#ifdef WG_DEBUG_PARAMS 4547#ifdef WG_DEBUG_PARAMS
4451SYSCTL_SETUP(sysctl_net_wg_setup, "sysctl net.wg setup") 4548SYSCTL_SETUP(sysctl_net_wg_setup, "sysctl net.wg setup")
4452{ 4549{
4453 const struct sysctlnode *node = NULL; 4550 const struct sysctlnode *node = NULL;
4454 4551
4455 sysctl_createv(clog, 0, NULL, &node, 4552 sysctl_createv(clog, 0, NULL, &node,
4456 CTLFLAG_PERMANENT, 4553 CTLFLAG_PERMANENT,
4457 CTLTYPE_NODE, "wg", 4554 CTLTYPE_NODE, "wg",
4458 SYSCTL_DESCR("wg(4)"), 4555 SYSCTL_DESCR("wg(4)"),
4459 NULL, 0, NULL, 0, 4556 NULL, 0, NULL, 0,
4460 CTL_NET, CTL_CREATE, CTL_EOL); 4557 CTL_NET, CTL_CREATE, CTL_EOL);
4461 sysctl_createv(clog, 0, &node, NULL, 4558 sysctl_createv(clog, 0, &node, NULL,
4462 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4559 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
4463 CTLTYPE_QUAD, "rekey_after_messages", 4560 CTLTYPE_QUAD, "rekey_after_messages",
4464 SYSCTL_DESCR("session liftime by messages"), 4561 SYSCTL_DESCR("session liftime by messages"),
4465 NULL, 0, &wg_rekey_after_messages, 0, CTL_CREATE, CTL_EOL); 4562 NULL, 0, &wg_rekey_after_messages, 0, CTL_CREATE, CTL_EOL);
4466 sysctl_createv(clog, 0, &node, NULL, 4563 sysctl_createv(clog, 0, &node, NULL,
4467 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4564 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
4468 CTLTYPE_INT, "rekey_after_time", 4565 CTLTYPE_INT, "rekey_after_time",
4469 SYSCTL_DESCR("session liftime"), 4566 SYSCTL_DESCR("session liftime"),
4470 NULL, 0, &wg_rekey_after_time, 0, CTL_CREATE, CTL_EOL); 4567 NULL, 0, &wg_rekey_after_time, 0, CTL_CREATE, CTL_EOL);
4471 sysctl_createv(clog, 0, &node, NULL, 4568 sysctl_createv(clog, 0, &node, NULL,
4472 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4569 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
4473 CTLTYPE_INT, "rekey_timeout", 4570 CTLTYPE_INT, "rekey_timeout",
4474 SYSCTL_DESCR("session handshake retry time"), 4571 SYSCTL_DESCR("session handshake retry time"),
4475 NULL, 0, &wg_rekey_timeout, 0, CTL_CREATE, CTL_EOL); 4572 NULL, 0, &wg_rekey_timeout, 0, CTL_CREATE, CTL_EOL);
4476 sysctl_createv(clog, 0, &node, NULL, 4573 sysctl_createv(clog, 0, &node, NULL,
4477 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4574 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
4478 CTLTYPE_INT, "rekey_attempt_time", 4575 CTLTYPE_INT, "rekey_attempt_time",
4479 SYSCTL_DESCR("session handshake timeout"), 4576 SYSCTL_DESCR("session handshake timeout"),
4480 NULL, 0, &wg_rekey_attempt_time, 0, CTL_CREATE, CTL_EOL); 4577 NULL, 0, &wg_rekey_attempt_time, 0, CTL_CREATE, CTL_EOL);
4481 sysctl_createv(clog, 0, &node, NULL, 4578 sysctl_createv(clog, 0, &node, NULL,
4482 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4579 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
4483 CTLTYPE_INT, "keepalive_timeout", 4580 CTLTYPE_INT, "keepalive_timeout",
4484 SYSCTL_DESCR("keepalive timeout"), 4581 SYSCTL_DESCR("keepalive timeout"),
4485 NULL, 0, &wg_keepalive_timeout, 0, CTL_CREATE, CTL_EOL); 4582 NULL, 0, &wg_keepalive_timeout, 0, CTL_CREATE, CTL_EOL);
4486 sysctl_createv(clog, 0, &node, NULL, 4583 sysctl_createv(clog, 0, &node, NULL,
4487 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 4584 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
4488 CTLTYPE_BOOL, "force_underload", 4585 CTLTYPE_BOOL, "force_underload",
4489 SYSCTL_DESCR("force to detemine under load"), 4586 SYSCTL_DESCR("force to detemine under load"),
4490 NULL, 0, &wg_force_underload, 0, CTL_CREATE, CTL_EOL); 4587 NULL, 0, &wg_force_underload, 0, CTL_CREATE, CTL_EOL);
4491} 4588}
4492#endif 4589#endif
4493 4590
4494#ifdef WG_RUMPKERNEL 4591#ifdef WG_RUMPKERNEL
4495static bool 4592static bool
4496wg_user_mode(struct wg_softc *wg) 4593wg_user_mode(struct wg_softc *wg)
4497{ 4594{
4498 4595
4499 return wg->wg_user != NULL; 4596 return wg->wg_user != NULL;
4500} 4597}
4501 4598
4502static int 4599static int
4503wg_ioctl_linkstr(struct wg_softc *wg, struct ifdrv *ifd) 4600wg_ioctl_linkstr(struct wg_softc *wg, struct ifdrv *ifd)
4504{ 4601{
4505 struct ifnet *ifp = &wg->wg_if; 4602 struct ifnet *ifp = &wg->wg_if;
4506 int error; 4603 int error;
4507 4604
4508 if (ifp->if_flags & IFF_UP) 4605 if (ifp->if_flags & IFF_UP)
4509 return EBUSY; 4606 return EBUSY;
4510 4607
4511 if (ifd->ifd_cmd == IFLINKSTR_UNSET) { 4608 if (ifd->ifd_cmd == IFLINKSTR_UNSET) {
4512 /* XXX do nothing */ 4609 /* XXX do nothing */
4513 return 0; 4610 return 0;
4514 } else if (ifd->ifd_cmd != 0) { 4611 } else if (ifd->ifd_cmd != 0) {
4515 return EINVAL; 4612 return EINVAL;
4516 } else if (wg->wg_user != NULL) { 4613 } else if (wg->wg_user != NULL) {
4517 return EBUSY; 4614 return EBUSY;
4518 } 4615 }
4519 4616
4520 /* Assume \0 included */ 4617 /* Assume \0 included */
4521 if (ifd->ifd_len > IFNAMSIZ) { 4618 if (ifd->ifd_len > IFNAMSIZ) {
4522 return E2BIG; 4619 return E2BIG;
4523 } else if (ifd->ifd_len < 1) { 4620 } else if (ifd->ifd_len < 1) {
4524 return EINVAL; 4621 return EINVAL;
4525 } 4622 }
4526 4623
4527 char tun_name[IFNAMSIZ]; 4624 char tun_name[IFNAMSIZ];
4528 error = copyinstr(ifd->ifd_data, tun_name, ifd->ifd_len, NULL); 4625 error = copyinstr(ifd->ifd_data, tun_name, ifd->ifd_len, NULL);
4529 if (error != 0) 4626 if (error != 0)
4530 return error; 4627 return error;
4531 4628
4532 if (strncmp(tun_name, "tun", 3) != 0) 4629 if (strncmp(tun_name, "tun", 3) != 0)
4533 return EINVAL; 4630 return EINVAL;
4534 4631
4535 error = rumpuser_wg_create(tun_name, wg, &wg->wg_user); 4632 error = rumpuser_wg_create(tun_name, wg, &wg->wg_user);
4536 4633
4537 return error; 4634 return error;
4538} 4635}
4539 4636
4540static int 4637static int
4541wg_send_user(struct wg_peer *wgp, struct mbuf *m) 4638wg_send_user(struct wg_peer *wgp, struct mbuf *m)
4542{ 4639{
4543 int error; 4640 int error;
4544 struct psref psref; 4641 struct psref psref;
4545 struct wg_sockaddr *wgsa; 4642 struct wg_sockaddr *wgsa;
4546 struct wg_softc *wg = wgp->wgp_sc; 4643 struct wg_softc *wg = wgp->wgp_sc;
4547 struct iovec iov[1]; 4644 struct iovec iov[1];
4548 4645
4549 wgsa = wg_get_endpoint_sa(wgp, &psref); 4646 wgsa = wg_get_endpoint_sa(wgp, &psref);
4550 4647
4551 iov[0].iov_base = mtod(m, void *); 4648 iov[0].iov_base = mtod(m, void *);
4552 iov[0].iov_len = m->m_len; 4649 iov[0].iov_len = m->m_len;
4553 4650
4554 /* Send messages to a peer via an ordinary socket. */ 4651 /* Send messages to a peer via an ordinary socket. */
4555 error = rumpuser_wg_send_peer(wg->wg_user, wgsatosa(wgsa), iov, 1); 4652 error = rumpuser_wg_send_peer(wg->wg_user, wgsatosa(wgsa), iov, 1);
4556 4653
4557 wg_put_sa(wgp, wgsa, &psref); 4654 wg_put_sa(wgp, wgsa, &psref);
4558 4655
4559 return error; 4656 return error;
4560} 4657}
4561 4658
4562static void 4659static void
4563wg_input_user(struct ifnet *ifp, struct mbuf *m, const int af) 4660wg_input_user(struct ifnet *ifp, struct mbuf *m, const int af)
4564{ 4661{
4565 struct wg_softc *wg = ifp->if_softc; 4662 struct wg_softc *wg = ifp->if_softc;
4566 struct iovec iov[2]; 4663 struct iovec iov[2];
4567 struct sockaddr_storage ss; 4664 struct sockaddr_storage ss;
4568 4665
4569 KASSERT(af == AF_INET || af == AF_INET6); 4666 KASSERT(af == AF_INET || af == AF_INET6);
4570 4667
4571 WG_TRACE(""); 4668 WG_TRACE("");
4572 4669
4573 if (af == AF_INET) { 4670 if (af == AF_INET) {
4574 struct sockaddr_in *sin = (struct sockaddr_in *)&ss; 4671 struct sockaddr_in *sin = (struct sockaddr_in *)&ss;
4575 struct ip *ip; 4672 struct ip *ip;
4576 4673
4577 KASSERT(m->m_len >= sizeof(struct ip)); 4674 KASSERT(m->m_len >= sizeof(struct ip));
4578 ip = mtod(m, struct ip *); 4675 ip = mtod(m, struct ip *);
4579 sockaddr_in_init(sin, &ip->ip_dst, 0); 4676 sockaddr_in_init(sin, &ip->ip_dst, 0);
4580 } else { 4677 } else {
4581 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; 4678 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss;
4582 struct ip6_hdr *ip6; 4679 struct ip6_hdr *ip6;
4583 4680
4584 KASSERT(m->m_len >= sizeof(struct ip6_hdr)); 4681 KASSERT(m->m_len >= sizeof(struct ip6_hdr));
4585 ip6 = mtod(m, struct ip6_hdr *); 4682 ip6 = mtod(m, struct ip6_hdr *);
4586 sockaddr_in6_init(sin6, &ip6->ip6_dst, 0, 0, 0); 4683 sockaddr_in6_init(sin6, &ip6->ip6_dst, 0, 0, 0);
4587 } 4684 }
4588 4685
4589 iov[0].iov_base = &ss; 4686 iov[0].iov_base = &ss;
4590 iov[0].iov_len = ss.ss_len; 4687 iov[0].iov_len = ss.ss_len;
4591 iov[1].iov_base = mtod(m, void *); 4688 iov[1].iov_base = mtod(m, void *);
4592 iov[1].iov_len = m->m_len; 4689 iov[1].iov_len = m->m_len;
4593 4690
4594 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 4691 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len);
4595 4692
4596 /* Send decrypted packets to users via a tun. */ 4693 /* Send decrypted packets to users via a tun. */
4597 rumpuser_wg_send_user(wg->wg_user, iov, 2); 4694 rumpuser_wg_send_user(wg->wg_user, iov, 2);
4598} 4695}
4599 4696
4600static int 4697static int
4601wg_bind_port_user(struct wg_softc *wg, const uint16_t port) 4698wg_bind_port_user(struct wg_softc *wg, const uint16_t port)
4602{ 4699{
4603 int error; 4700 int error;
4604 uint16_t old_port = wg->wg_listen_port; 4701 uint16_t old_port = wg->wg_listen_port;
4605 4702
4606 if (port != 0 && old_port == port) 4703 if (port != 0 && old_port == port)
4607 return 0; 4704 return 0;
4608 4705
4609 error = rumpuser_wg_sock_bind(wg->wg_user, port); 4706 error = rumpuser_wg_sock_bind(wg->wg_user, port);
4610 if (error == 0) 4707 if (error == 0)
4611 wg->wg_listen_port = port; 4708 wg->wg_listen_port = port;
4612 return error; 4709 return error;
4613} 4710}
4614 4711
4615/* 4712/*
4616 * Receive user packets. 4713 * Receive user packets.
4617 */ 4714 */
4618void 4715void
4619rumpkern_wg_recv_user(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 4716rumpkern_wg_recv_user(struct wg_softc *wg, struct iovec *iov, size_t iovlen)
4620{ 4717{
4621 struct ifnet *ifp = &wg->wg_if; 4718 struct ifnet *ifp = &wg->wg_if;
4622 struct mbuf *m; 4719 struct mbuf *m;
4623 const struct sockaddr *dst; 4720 const struct sockaddr *dst;
4624 4721
4625 WG_TRACE(""); 4722 WG_TRACE("");
4626 4723
4627 dst = iov[0].iov_base; 4724 dst = iov[0].iov_base;
4628 4725
4629 m = m_gethdr(M_NOWAIT, MT_DATA); 4726 m = m_gethdr(M_NOWAIT, MT_DATA);
4630 if (m == NULL) 4727 if (m == NULL)
4631 return; 4728 return;
4632 m->m_len = m->m_pkthdr.len = 0; 4729 m->m_len = m->m_pkthdr.len = 0;
4633 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 4730 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base);
4634 4731
4635 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 4732 WG_DLOG("iov_len=%lu\n", iov[1].iov_len);
4636 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 4733 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len);
4637 4734
4638 (void)wg_output(ifp, m, dst, NULL); 4735 (void)wg_output(ifp, m, dst, NULL);
4639} 4736}
4640 4737
4641/* 4738/*
4642 * Receive packets from a peer. 4739 * Receive packets from a peer.
4643 */ 4740 */
4644void 4741void
4645rumpkern_wg_recv_peer(struct wg_softc *wg, struct iovec *iov, size_t iovlen) 4742rumpkern_wg_recv_peer(struct wg_softc *wg, struct iovec *iov, size_t iovlen)
4646{ 4743{
4647 struct mbuf *m; 4744 struct mbuf *m;
4648 const struct sockaddr *src; 4745 const struct sockaddr *src;
4649 4746
4650 WG_TRACE(""); 4747 WG_TRACE("");
4651 4748
4652 src = iov[0].iov_base; 4749 src = iov[0].iov_base;
4653 4750
4654 m = m_gethdr(M_NOWAIT, MT_DATA); 4751 m = m_gethdr(M_NOWAIT, MT_DATA);
4655 if (m == NULL) 4752 if (m == NULL)
4656 return; 4753 return;
4657 m->m_len = m->m_pkthdr.len = 0; 4754 m->m_len = m->m_pkthdr.len = 0;
4658 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base); 4755 m_copyback(m, 0, iov[1].iov_len, iov[1].iov_base);
4659 4756
4660 WG_DLOG("iov_len=%lu\n", iov[1].iov_len); 4757 WG_DLOG("iov_len=%lu\n", iov[1].iov_len);
4661 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len); 4758 WG_DUMP_BUF(iov[1].iov_base, iov[1].iov_len);
4662 4759
4663 wg_handle_packet(wg, m, src); 4760 wg_handle_packet(wg, m, src);
4664} 4761}
4665#endif /* WG_RUMPKERNEL */ 4762#endif /* WG_RUMPKERNEL */
4666 4763
4667/* 4764/*
4668 * Module infrastructure 4765 * Module infrastructure
4669 */ 4766 */
4670#include "if_module.h" 4767#include "if_module.h"
4671 4768
4672IF_MODULE(MODULE_CLASS_DRIVER, wg, "") 4769IF_MODULE(MODULE_CLASS_DRIVER, wg, "")