| @@ -1,1041 +1,1041 @@ | | | @@ -1,1041 +1,1041 @@ |
1 | /* $NetBSD: bpf.c,v 1.240 2021/06/09 15:44:15 martin Exp $ */ | | 1 | /* $NetBSD: bpf.c,v 1.241 2021/07/14 06:50:22 yamaguchi Exp $ */ |
2 | | | 2 | |
3 | /* | | 3 | /* |
4 | * Copyright (c) 1990, 1991, 1993 | | 4 | * Copyright (c) 1990, 1991, 1993 |
5 | * The Regents of the University of California. All rights reserved. | | 5 | * The Regents of the University of California. All rights reserved. |
6 | * | | 6 | * |
7 | * This code is derived from the Stanford/CMU enet packet filter, | | 7 | * This code is derived from the Stanford/CMU enet packet filter, |
8 | * (net/enet.c) distributed as part of 4.3BSD, and code contributed | | 8 | * (net/enet.c) distributed as part of 4.3BSD, and code contributed |
9 | * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence | | 9 | * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence |
10 | * Berkeley Laboratory. | | 10 | * Berkeley Laboratory. |
11 | * | | 11 | * |
12 | * Redistribution and use in source and binary forms, with or without | | 12 | * Redistribution and use in source and binary forms, with or without |
13 | * modification, are permitted provided that the following conditions | | 13 | * modification, are permitted provided that the following conditions |
14 | * are met: | | 14 | * are met: |
15 | * 1. Redistributions of source code must retain the above copyright | | 15 | * 1. Redistributions of source code must retain the above copyright |
16 | * notice, this list of conditions and the following disclaimer. | | 16 | * notice, this list of conditions and the following disclaimer. |
17 | * 2. Redistributions in binary form must reproduce the above copyright | | 17 | * 2. Redistributions in binary form must reproduce the above copyright |
18 | * notice, this list of conditions and the following disclaimer in the | | 18 | * notice, this list of conditions and the following disclaimer in the |
19 | * documentation and/or other materials provided with the distribution. | | 19 | * documentation and/or other materials provided with the distribution. |
20 | * 3. Neither the name of the University nor the names of its contributors | | 20 | * 3. Neither the name of the University nor the names of its contributors |
21 | * may be used to endorse or promote products derived from this software | | 21 | * may be used to endorse or promote products derived from this software |
22 | * without specific prior written permission. | | 22 | * without specific prior written permission. |
23 | * | | 23 | * |
24 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND | | 24 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND |
25 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | | 25 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
26 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | | 26 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
27 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE | | 27 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE |
28 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | | 28 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
29 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | | 29 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
30 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | | 30 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
31 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | | 31 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
32 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | | 32 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
33 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | | 33 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
34 | * SUCH DAMAGE. | | 34 | * SUCH DAMAGE. |
35 | * | | 35 | * |
36 | * @(#)bpf.c 8.4 (Berkeley) 1/9/95 | | 36 | * @(#)bpf.c 8.4 (Berkeley) 1/9/95 |
37 | * static char rcsid[] = | | 37 | * static char rcsid[] = |
38 | * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; | | 38 | * "Header: bpf.c,v 1.67 96/09/26 22:00:52 leres Exp "; |
39 | */ | | 39 | */ |
40 | | | 40 | |
41 | #include <sys/cdefs.h> | | 41 | #include <sys/cdefs.h> |
42 | __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.240 2021/06/09 15:44:15 martin Exp $"); | | 42 | __KERNEL_RCSID(0, "$NetBSD: bpf.c,v 1.241 2021/07/14 06:50:22 yamaguchi Exp $"); |
43 | | | 43 | |
44 | #if defined(_KERNEL_OPT) | | 44 | #if defined(_KERNEL_OPT) |
45 | #include "opt_bpf.h" | | 45 | #include "opt_bpf.h" |
46 | #include "sl.h" | | 46 | #include "sl.h" |
47 | #include "opt_net_mpsafe.h" | | 47 | #include "opt_net_mpsafe.h" |
48 | #endif | | 48 | #endif |
49 | | | 49 | |
50 | #include <sys/param.h> | | 50 | #include <sys/param.h> |
51 | #include <sys/systm.h> | | 51 | #include <sys/systm.h> |
52 | #include <sys/mbuf.h> | | 52 | #include <sys/mbuf.h> |
53 | #include <sys/buf.h> | | 53 | #include <sys/buf.h> |
54 | #include <sys/time.h> | | 54 | #include <sys/time.h> |
55 | #include <sys/proc.h> | | 55 | #include <sys/proc.h> |
56 | #include <sys/ioctl.h> | | 56 | #include <sys/ioctl.h> |
57 | #include <sys/conf.h> | | 57 | #include <sys/conf.h> |
58 | #include <sys/vnode.h> | | 58 | #include <sys/vnode.h> |
59 | #include <sys/queue.h> | | 59 | #include <sys/queue.h> |
60 | #include <sys/stat.h> | | 60 | #include <sys/stat.h> |
61 | #include <sys/module.h> | | 61 | #include <sys/module.h> |
62 | #include <sys/atomic.h> | | 62 | #include <sys/atomic.h> |
63 | #include <sys/cpu.h> | | 63 | #include <sys/cpu.h> |
64 | | | 64 | |
65 | #include <sys/file.h> | | 65 | #include <sys/file.h> |
66 | #include <sys/filedesc.h> | | 66 | #include <sys/filedesc.h> |
67 | #include <sys/tty.h> | | 67 | #include <sys/tty.h> |
68 | #include <sys/uio.h> | | 68 | #include <sys/uio.h> |
69 | | | 69 | |
70 | #include <sys/protosw.h> | | 70 | #include <sys/protosw.h> |
71 | #include <sys/socket.h> | | 71 | #include <sys/socket.h> |
72 | #include <sys/errno.h> | | 72 | #include <sys/errno.h> |
73 | #include <sys/kernel.h> | | 73 | #include <sys/kernel.h> |
74 | #include <sys/poll.h> | | 74 | #include <sys/poll.h> |
75 | #include <sys/sysctl.h> | | 75 | #include <sys/sysctl.h> |
76 | #include <sys/kauth.h> | | 76 | #include <sys/kauth.h> |
77 | #include <sys/syslog.h> | | 77 | #include <sys/syslog.h> |
78 | #include <sys/percpu.h> | | 78 | #include <sys/percpu.h> |
79 | #include <sys/pserialize.h> | | 79 | #include <sys/pserialize.h> |
80 | #include <sys/lwp.h> | | 80 | #include <sys/lwp.h> |
81 | #include <sys/xcall.h> | | 81 | #include <sys/xcall.h> |
82 | | | 82 | |
83 | #include <net/if.h> | | 83 | #include <net/if.h> |
84 | #include <net/slip.h> | | 84 | #include <net/slip.h> |
85 | | | 85 | |
86 | #include <net/bpf.h> | | 86 | #include <net/bpf.h> |
87 | #include <net/bpfdesc.h> | | 87 | #include <net/bpfdesc.h> |
88 | #include <net/bpfjit.h> | | 88 | #include <net/bpfjit.h> |
89 | | | 89 | |
90 | #include <net/if_arc.h> | | 90 | #include <net/if_arc.h> |
91 | #include <net/if_ether.h> | | 91 | #include <net/if_ether.h> |
92 | | | 92 | |
93 | #include <netinet/in.h> | | 93 | #include <netinet/in.h> |
94 | #include <netinet/if_inarp.h> | | 94 | #include <netinet/if_inarp.h> |
95 | | | 95 | |
96 | | | 96 | |
97 | #include <compat/sys/sockio.h> | | 97 | #include <compat/sys/sockio.h> |
98 | | | 98 | |
99 | #ifndef BPF_BUFSIZE | | 99 | #ifndef BPF_BUFSIZE |
100 | /* | | 100 | /* |
101 | * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet | | 101 | * 4096 is too small for FDDI frames. 8192 is too small for gigabit Ethernet |
102 | * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). | | 102 | * jumbos (circa 9k), ATM, or Intel gig/10gig ethernet jumbos (16k). |
103 | */ | | 103 | */ |
104 | # define BPF_BUFSIZE 32768 | | 104 | # define BPF_BUFSIZE 32768 |
105 | #endif | | 105 | #endif |
106 | | | 106 | |
107 | #define PRINET 26 /* interruptible */ | | 107 | #define PRINET 26 /* interruptible */ |
108 | | | 108 | |
109 | /* | | 109 | /* |
110 | * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. | | 110 | * The default read buffer size, and limit for BIOCSBLEN, is sysctl'able. |
111 | * XXX the default values should be computed dynamically based | | 111 | * XXX the default values should be computed dynamically based |
112 | * on available memory size and available mbuf clusters. | | 112 | * on available memory size and available mbuf clusters. |
113 | */ | | 113 | */ |
114 | static int bpf_bufsize = BPF_BUFSIZE; | | 114 | static int bpf_bufsize = BPF_BUFSIZE; |
115 | static int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ | | 115 | static int bpf_maxbufsize = BPF_DFLTBUFSIZE; /* XXX set dynamically, see above */ |
116 | static bool bpf_jit = false; | | 116 | static bool bpf_jit = false; |
117 | | | 117 | |
118 | struct bpfjit_ops bpfjit_module_ops = { | | 118 | struct bpfjit_ops bpfjit_module_ops = { |
119 | .bj_generate_code = NULL, | | 119 | .bj_generate_code = NULL, |
120 | .bj_free_code = NULL | | 120 | .bj_free_code = NULL |
121 | }; | | 121 | }; |
122 | | | 122 | |
123 | /* | | 123 | /* |
124 | * Global BPF statistics returned by net.bpf.stats sysctl. | | 124 | * Global BPF statistics returned by net.bpf.stats sysctl. |
125 | */ | | 125 | */ |
126 | static struct percpu *bpf_gstats_percpu; /* struct bpf_stat */ | | 126 | static struct percpu *bpf_gstats_percpu; /* struct bpf_stat */ |
127 | | | 127 | |
128 | #define BPF_STATINC(id) \ | | 128 | #define BPF_STATINC(id) \ |
129 | { \ | | 129 | { \ |
130 | struct bpf_stat *__stats = \ | | 130 | struct bpf_stat *__stats = \ |
131 | percpu_getref(bpf_gstats_percpu); \ | | 131 | percpu_getref(bpf_gstats_percpu); \ |
132 | __stats->bs_##id++; \ | | 132 | __stats->bs_##id++; \ |
133 | percpu_putref(bpf_gstats_percpu); \ | | 133 | percpu_putref(bpf_gstats_percpu); \ |
134 | } | | 134 | } |
135 | | | 135 | |
136 | /* | | 136 | /* |
137 | * Locking notes: | | 137 | * Locking notes: |
138 | * - bpf_mtx (adaptive mutex) protects: | | 138 | * - bpf_mtx (adaptive mutex) protects: |
139 | * - Gobal lists: bpf_iflist and bpf_dlist | | 139 | * - Gobal lists: bpf_iflist and bpf_dlist |
140 | * - struct bpf_if | | 140 | * - struct bpf_if |
141 | * - bpf_close | | 141 | * - bpf_close |
142 | * - bpf_psz (pserialize) | | 142 | * - bpf_psz (pserialize) |
143 | * - struct bpf_d has two mutexes: | | 143 | * - struct bpf_d has two mutexes: |
144 | * - bd_buf_mtx (spin mutex) protects the buffers that can be accessed | | 144 | * - bd_buf_mtx (spin mutex) protects the buffers that can be accessed |
145 | * on packet tapping | | 145 | * on packet tapping |
146 | * - bd_mtx (adaptive mutex) protects member variables other than the buffers | | 146 | * - bd_mtx (adaptive mutex) protects member variables other than the buffers |
147 | * - Locking order: bpf_mtx => bpf_d#bd_mtx => bpf_d#bd_buf_mtx | | 147 | * - Locking order: bpf_mtx => bpf_d#bd_mtx => bpf_d#bd_buf_mtx |
148 | * - struct bpf_d obtained via fp->f_bpf in bpf_read and bpf_write is | | 148 | * - struct bpf_d obtained via fp->f_bpf in bpf_read and bpf_write is |
149 | * never freed because struct bpf_d is only freed in bpf_close and | | 149 | * never freed because struct bpf_d is only freed in bpf_close and |
150 | * bpf_close never be called while executing bpf_read and bpf_write | | 150 | * bpf_close never be called while executing bpf_read and bpf_write |
151 | * - A filter that is assigned to bpf_d can be replaced with another filter | | 151 | * - A filter that is assigned to bpf_d can be replaced with another filter |
152 | * while tapping packets, so it needs to be done atomically | | 152 | * while tapping packets, so it needs to be done atomically |
153 | * - struct bpf_d is iterated on bpf_dlist with psz | | 153 | * - struct bpf_d is iterated on bpf_dlist with psz |
154 | * - struct bpf_if is iterated on bpf_iflist with psz or psref | | 154 | * - struct bpf_if is iterated on bpf_iflist with psz or psref |
155 | */ | | 155 | */ |
156 | /* | | 156 | /* |
157 | * Use a mutex to avoid a race condition between gathering the stats/peers | | 157 | * Use a mutex to avoid a race condition between gathering the stats/peers |
158 | * and opening/closing the device. | | 158 | * and opening/closing the device. |
159 | */ | | 159 | */ |
160 | static kmutex_t bpf_mtx; | | 160 | static kmutex_t bpf_mtx; |
161 | | | 161 | |
162 | static struct psref_class *bpf_psref_class __read_mostly; | | 162 | static struct psref_class *bpf_psref_class __read_mostly; |
163 | static pserialize_t bpf_psz; | | 163 | static pserialize_t bpf_psz; |
164 | | | 164 | |
165 | static inline void | | 165 | static inline void |
166 | bpf_if_acquire(struct bpf_if *bp, struct psref *psref) | | 166 | bpf_if_acquire(struct bpf_if *bp, struct psref *psref) |
167 | { | | 167 | { |
168 | | | 168 | |
169 | psref_acquire(psref, &bp->bif_psref, bpf_psref_class); | | 169 | psref_acquire(psref, &bp->bif_psref, bpf_psref_class); |
170 | } | | 170 | } |
171 | | | 171 | |
172 | static inline void | | 172 | static inline void |
173 | bpf_if_release(struct bpf_if *bp, struct psref *psref) | | 173 | bpf_if_release(struct bpf_if *bp, struct psref *psref) |
174 | { | | 174 | { |
175 | | | 175 | |
176 | psref_release(psref, &bp->bif_psref, bpf_psref_class); | | 176 | psref_release(psref, &bp->bif_psref, bpf_psref_class); |
177 | } | | 177 | } |
178 | | | 178 | |
179 | /* | | 179 | /* |
180 | * bpf_iflist is the list of interfaces; each corresponds to an ifnet | | 180 | * bpf_iflist is the list of interfaces; each corresponds to an ifnet |
181 | * bpf_dtab holds the descriptors, indexed by minor device # | | 181 | * bpf_dtab holds the descriptors, indexed by minor device # |
182 | */ | | 182 | */ |
183 | static struct pslist_head bpf_iflist; | | 183 | static struct pslist_head bpf_iflist; |
184 | static struct pslist_head bpf_dlist; | | 184 | static struct pslist_head bpf_dlist; |
185 | | | 185 | |
186 | /* Macros for bpf_d on bpf_dlist */ | | 186 | /* Macros for bpf_d on bpf_dlist */ |
187 | #define BPF_DLIST_WRITER_INSERT_HEAD(__d) \ | | 187 | #define BPF_DLIST_WRITER_INSERT_HEAD(__d) \ |
188 | PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry) | | 188 | PSLIST_WRITER_INSERT_HEAD(&bpf_dlist, (__d), bd_bpf_dlist_entry) |
189 | #define BPF_DLIST_READER_FOREACH(__d) \ | | 189 | #define BPF_DLIST_READER_FOREACH(__d) \ |
190 | PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ | | 190 | PSLIST_READER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ |
191 | bd_bpf_dlist_entry) | | 191 | bd_bpf_dlist_entry) |
192 | #define BPF_DLIST_WRITER_FOREACH(__d) \ | | 192 | #define BPF_DLIST_WRITER_FOREACH(__d) \ |
193 | PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ | | 193 | PSLIST_WRITER_FOREACH((__d), &bpf_dlist, struct bpf_d, \ |
194 | bd_bpf_dlist_entry) | | 194 | bd_bpf_dlist_entry) |
195 | #define BPF_DLIST_ENTRY_INIT(__d) \ | | 195 | #define BPF_DLIST_ENTRY_INIT(__d) \ |
196 | PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry) | | 196 | PSLIST_ENTRY_INIT((__d), bd_bpf_dlist_entry) |
197 | #define BPF_DLIST_WRITER_REMOVE(__d) \ | | 197 | #define BPF_DLIST_WRITER_REMOVE(__d) \ |
198 | PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry) | | 198 | PSLIST_WRITER_REMOVE((__d), bd_bpf_dlist_entry) |
199 | #define BPF_DLIST_ENTRY_DESTROY(__d) \ | | 199 | #define BPF_DLIST_ENTRY_DESTROY(__d) \ |
200 | PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry) | | 200 | PSLIST_ENTRY_DESTROY((__d), bd_bpf_dlist_entry) |
201 | | | 201 | |
202 | /* Macros for bpf_if on bpf_iflist */ | | 202 | /* Macros for bpf_if on bpf_iflist */ |
203 | #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp) \ | | 203 | #define BPF_IFLIST_WRITER_INSERT_HEAD(__bp) \ |
204 | PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry) | | 204 | PSLIST_WRITER_INSERT_HEAD(&bpf_iflist, (__bp), bif_iflist_entry) |
205 | #define BPF_IFLIST_READER_FOREACH(__bp) \ | | 205 | #define BPF_IFLIST_READER_FOREACH(__bp) \ |
206 | PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ | | 206 | PSLIST_READER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ |
207 | bif_iflist_entry) | | 207 | bif_iflist_entry) |
208 | #define BPF_IFLIST_WRITER_FOREACH(__bp) \ | | 208 | #define BPF_IFLIST_WRITER_FOREACH(__bp) \ |
209 | PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ | | 209 | PSLIST_WRITER_FOREACH((__bp), &bpf_iflist, struct bpf_if, \ |
210 | bif_iflist_entry) | | 210 | bif_iflist_entry) |
211 | #define BPF_IFLIST_WRITER_REMOVE(__bp) \ | | 211 | #define BPF_IFLIST_WRITER_REMOVE(__bp) \ |
212 | PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry) | | 212 | PSLIST_WRITER_REMOVE((__bp), bif_iflist_entry) |
213 | #define BPF_IFLIST_ENTRY_INIT(__bp) \ | | 213 | #define BPF_IFLIST_ENTRY_INIT(__bp) \ |
214 | PSLIST_ENTRY_INIT((__bp), bif_iflist_entry) | | 214 | PSLIST_ENTRY_INIT((__bp), bif_iflist_entry) |
215 | #define BPF_IFLIST_ENTRY_DESTROY(__bp) \ | | 215 | #define BPF_IFLIST_ENTRY_DESTROY(__bp) \ |
216 | PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry) | | 216 | PSLIST_ENTRY_DESTROY((__bp), bif_iflist_entry) |
217 | | | 217 | |
218 | /* Macros for bpf_d on bpf_if#bif_dlist_pslist */ | | 218 | /* Macros for bpf_d on bpf_if#bif_dlist_pslist */ |
219 | #define BPFIF_DLIST_READER_FOREACH(__d, __bp) \ | | 219 | #define BPFIF_DLIST_READER_FOREACH(__d, __bp) \ |
220 | PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \ | | 220 | PSLIST_READER_FOREACH((__d), &(__bp)->bif_dlist_head, struct bpf_d, \ |
221 | bd_bif_dlist_entry) | | 221 | bd_bif_dlist_entry) |
222 | #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d) \ | | 222 | #define BPFIF_DLIST_WRITER_INSERT_HEAD(__bp, __d) \ |
223 | PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d), \ | | 223 | PSLIST_WRITER_INSERT_HEAD(&(__bp)->bif_dlist_head, (__d), \ |
224 | bd_bif_dlist_entry) | | 224 | bd_bif_dlist_entry) |
225 | #define BPFIF_DLIST_WRITER_REMOVE(__d) \ | | 225 | #define BPFIF_DLIST_WRITER_REMOVE(__d) \ |
226 | PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry) | | 226 | PSLIST_WRITER_REMOVE((__d), bd_bif_dlist_entry) |
227 | #define BPFIF_DLIST_ENTRY_INIT(__d) \ | | 227 | #define BPFIF_DLIST_ENTRY_INIT(__d) \ |
228 | PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry) | | 228 | PSLIST_ENTRY_INIT((__d), bd_bif_dlist_entry) |
229 | #define BPFIF_DLIST_READER_EMPTY(__bp) \ | | 229 | #define BPFIF_DLIST_READER_EMPTY(__bp) \ |
230 | (PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ | | 230 | (PSLIST_READER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ |
231 | bd_bif_dlist_entry) == NULL) | | 231 | bd_bif_dlist_entry) == NULL) |
232 | #define BPFIF_DLIST_WRITER_EMPTY(__bp) \ | | 232 | #define BPFIF_DLIST_WRITER_EMPTY(__bp) \ |
233 | (PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ | | 233 | (PSLIST_WRITER_FIRST(&(__bp)->bif_dlist_head, struct bpf_d, \ |
234 | bd_bif_dlist_entry) == NULL) | | 234 | bd_bif_dlist_entry) == NULL) |
235 | #define BPFIF_DLIST_ENTRY_DESTROY(__d) \ | | 235 | #define BPFIF_DLIST_ENTRY_DESTROY(__d) \ |
236 | PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry) | | 236 | PSLIST_ENTRY_DESTROY((__d), bd_bif_dlist_entry) |
237 | | | 237 | |
238 | static int bpf_allocbufs(struct bpf_d *); | | 238 | static int bpf_allocbufs(struct bpf_d *); |
239 | static u_int bpf_xfilter(struct bpf_filter **, void *, u_int, u_int); | | 239 | static u_int bpf_xfilter(struct bpf_filter **, void *, u_int, u_int); |
240 | static void bpf_deliver(struct bpf_if *, | | 240 | static void bpf_deliver(struct bpf_if *, |
241 | void *(*cpfn)(void *, const void *, size_t), | | 241 | void *(*cpfn)(void *, const void *, size_t), |
242 | void *, u_int, u_int, const u_int); | | 242 | void *, u_int, u_int, const u_int); |
243 | static void bpf_freed(struct bpf_d *); | | 243 | static void bpf_freed(struct bpf_d *); |
244 | static void bpf_free_filter(struct bpf_filter *); | | 244 | static void bpf_free_filter(struct bpf_filter *); |
245 | static void bpf_ifname(struct ifnet *, struct ifreq *); | | 245 | static void bpf_ifname(struct ifnet *, struct ifreq *); |
246 | static void *bpf_mcpy(void *, const void *, size_t); | | 246 | static void *bpf_mcpy(void *, const void *, size_t); |
247 | static int bpf_movein(struct uio *, int, uint64_t, | | 247 | static int bpf_movein(struct uio *, int, uint64_t, |
248 | struct mbuf **, struct sockaddr *, | | 248 | struct mbuf **, struct sockaddr *, |
249 | struct bpf_filter **); | | 249 | struct bpf_filter **); |
250 | static void bpf_attachd(struct bpf_d *, struct bpf_if *); | | 250 | static void bpf_attachd(struct bpf_d *, struct bpf_if *); |
251 | static void bpf_detachd(struct bpf_d *); | | 251 | static void bpf_detachd(struct bpf_d *); |
252 | static int bpf_setif(struct bpf_d *, struct ifreq *); | | 252 | static int bpf_setif(struct bpf_d *, struct ifreq *); |
253 | static int bpf_setf(struct bpf_d *, struct bpf_program *, u_long); | | 253 | static int bpf_setf(struct bpf_d *, struct bpf_program *, u_long); |
254 | static void bpf_timed_out(void *); | | 254 | static void bpf_timed_out(void *); |
255 | static inline void | | 255 | static inline void |
256 | bpf_wakeup(struct bpf_d *); | | 256 | bpf_wakeup(struct bpf_d *); |
257 | static int bpf_hdrlen(struct bpf_d *); | | 257 | static int bpf_hdrlen(struct bpf_d *); |
258 | static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, | | 258 | static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, |
259 | void *(*)(void *, const void *, size_t), struct timespec *); | | 259 | void *(*)(void *, const void *, size_t), struct timespec *); |
260 | static void reset_d(struct bpf_d *); | | 260 | static void reset_d(struct bpf_d *); |
261 | static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); | | 261 | static int bpf_getdltlist(struct bpf_d *, struct bpf_dltlist *); |
262 | static int bpf_setdlt(struct bpf_d *, u_int); | | 262 | static int bpf_setdlt(struct bpf_d *, u_int); |
263 | | | 263 | |
264 | static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, | | 264 | static int bpf_read(struct file *, off_t *, struct uio *, kauth_cred_t, |
265 | int); | | 265 | int); |
266 | static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, | | 266 | static int bpf_write(struct file *, off_t *, struct uio *, kauth_cred_t, |
267 | int); | | 267 | int); |
268 | static int bpf_ioctl(struct file *, u_long, void *); | | 268 | static int bpf_ioctl(struct file *, u_long, void *); |
269 | static int bpf_poll(struct file *, int); | | 269 | static int bpf_poll(struct file *, int); |
270 | static int bpf_stat(struct file *, struct stat *); | | 270 | static int bpf_stat(struct file *, struct stat *); |
271 | static int bpf_close(struct file *); | | 271 | static int bpf_close(struct file *); |
272 | static int bpf_kqfilter(struct file *, struct knote *); | | 272 | static int bpf_kqfilter(struct file *, struct knote *); |
273 | | | 273 | |
274 | static const struct fileops bpf_fileops = { | | 274 | static const struct fileops bpf_fileops = { |
275 | .fo_name = "bpf", | | 275 | .fo_name = "bpf", |
276 | .fo_read = bpf_read, | | 276 | .fo_read = bpf_read, |
277 | .fo_write = bpf_write, | | 277 | .fo_write = bpf_write, |
278 | .fo_ioctl = bpf_ioctl, | | 278 | .fo_ioctl = bpf_ioctl, |
279 | .fo_fcntl = fnullop_fcntl, | | 279 | .fo_fcntl = fnullop_fcntl, |
280 | .fo_poll = bpf_poll, | | 280 | .fo_poll = bpf_poll, |
281 | .fo_stat = bpf_stat, | | 281 | .fo_stat = bpf_stat, |
282 | .fo_close = bpf_close, | | 282 | .fo_close = bpf_close, |
283 | .fo_kqfilter = bpf_kqfilter, | | 283 | .fo_kqfilter = bpf_kqfilter, |
284 | .fo_restart = fnullop_restart, | | 284 | .fo_restart = fnullop_restart, |
285 | }; | | 285 | }; |
286 | | | 286 | |
287 | dev_type_open(bpfopen); | | 287 | dev_type_open(bpfopen); |
288 | | | 288 | |
289 | const struct cdevsw bpf_cdevsw = { | | 289 | const struct cdevsw bpf_cdevsw = { |
290 | .d_open = bpfopen, | | 290 | .d_open = bpfopen, |
291 | .d_close = noclose, | | 291 | .d_close = noclose, |
292 | .d_read = noread, | | 292 | .d_read = noread, |
293 | .d_write = nowrite, | | 293 | .d_write = nowrite, |
294 | .d_ioctl = noioctl, | | 294 | .d_ioctl = noioctl, |
295 | .d_stop = nostop, | | 295 | .d_stop = nostop, |
296 | .d_tty = notty, | | 296 | .d_tty = notty, |
297 | .d_poll = nopoll, | | 297 | .d_poll = nopoll, |
298 | .d_mmap = nommap, | | 298 | .d_mmap = nommap, |
299 | .d_kqfilter = nokqfilter, | | 299 | .d_kqfilter = nokqfilter, |
300 | .d_discard = nodiscard, | | 300 | .d_discard = nodiscard, |
301 | .d_flag = D_OTHER | D_MPSAFE | | 301 | .d_flag = D_OTHER | D_MPSAFE |
302 | }; | | 302 | }; |
303 | | | 303 | |
304 | bpfjit_func_t | | 304 | bpfjit_func_t |
305 | bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size) | | 305 | bpf_jit_generate(bpf_ctx_t *bc, void *code, size_t size) |
306 | { | | 306 | { |
307 | struct bpfjit_ops *ops = &bpfjit_module_ops; | | 307 | struct bpfjit_ops *ops = &bpfjit_module_ops; |
308 | bpfjit_func_t (*generate_code)(const bpf_ctx_t *, | | 308 | bpfjit_func_t (*generate_code)(const bpf_ctx_t *, |
309 | const struct bpf_insn *, size_t); | | 309 | const struct bpf_insn *, size_t); |
310 | | | 310 | |
311 | generate_code = atomic_load_acquire(&ops->bj_generate_code); | | 311 | generate_code = atomic_load_acquire(&ops->bj_generate_code); |
312 | if (generate_code != NULL) { | | 312 | if (generate_code != NULL) { |
313 | return generate_code(bc, code, size); | | 313 | return generate_code(bc, code, size); |
314 | } | | 314 | } |
315 | return NULL; | | 315 | return NULL; |
316 | } | | 316 | } |
317 | | | 317 | |
318 | void | | 318 | void |
319 | bpf_jit_freecode(bpfjit_func_t jcode) | | 319 | bpf_jit_freecode(bpfjit_func_t jcode) |
320 | { | | 320 | { |
321 | KASSERT(bpfjit_module_ops.bj_free_code != NULL); | | 321 | KASSERT(bpfjit_module_ops.bj_free_code != NULL); |
322 | bpfjit_module_ops.bj_free_code(jcode); | | 322 | bpfjit_module_ops.bj_free_code(jcode); |
323 | } | | 323 | } |
324 | | | 324 | |
325 | static int | | 325 | static int |
326 | bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, | | 326 | bpf_movein(struct uio *uio, int linktype, uint64_t mtu, struct mbuf **mp, |
327 | struct sockaddr *sockp, struct bpf_filter **wfilter) | | 327 | struct sockaddr *sockp, struct bpf_filter **wfilter) |
328 | { | | 328 | { |
329 | struct mbuf *m, *m0, *n; | | 329 | struct mbuf *m, *m0, *n; |
330 | int error; | | 330 | int error; |
331 | size_t len; | | 331 | size_t len; |
332 | size_t hlen; | | 332 | size_t hlen; |
333 | size_t align; | | 333 | size_t align; |
334 | u_int slen; | | 334 | u_int slen; |
335 | | | 335 | |
336 | /* | | 336 | /* |
337 | * Build a sockaddr based on the data link layer type. | | 337 | * Build a sockaddr based on the data link layer type. |
338 | * We do this at this level because the ethernet header | | 338 | * We do this at this level because the ethernet header |
339 | * is copied directly into the data field of the sockaddr. | | 339 | * is copied directly into the data field of the sockaddr. |
340 | * In the case of SLIP, there is no header and the packet | | 340 | * In the case of SLIP, there is no header and the packet |
341 | * is forwarded as is. | | 341 | * is forwarded as is. |
342 | * Also, we are careful to leave room at the front of the mbuf | | 342 | * Also, we are careful to leave room at the front of the mbuf |
343 | * for the link level header. | | 343 | * for the link level header. |
344 | */ | | 344 | */ |
345 | switch (linktype) { | | 345 | switch (linktype) { |
346 | | | 346 | |
347 | case DLT_SLIP: | | 347 | case DLT_SLIP: |
348 | sockp->sa_family = AF_INET; | | 348 | sockp->sa_family = AF_INET; |
349 | hlen = 0; | | 349 | hlen = 0; |
350 | align = 0; | | 350 | align = 0; |
351 | break; | | 351 | break; |
352 | | | 352 | |
353 | case DLT_PPP: | | 353 | case DLT_PPP: |
354 | sockp->sa_family = AF_UNSPEC; | | 354 | sockp->sa_family = AF_UNSPEC; |
355 | hlen = 0; | | 355 | hlen = 0; |
356 | align = 0; | | 356 | align = 0; |
357 | break; | | 357 | break; |
358 | | | 358 | |
359 | case DLT_EN10MB: | | 359 | case DLT_EN10MB: |
360 | sockp->sa_family = AF_UNSPEC; | | 360 | sockp->sa_family = AF_UNSPEC; |
361 | /* XXX Would MAXLINKHDR be better? */ | | 361 | /* XXX Would MAXLINKHDR be better? */ |
362 | /* 6(dst)+6(src)+2(type) */ | | 362 | /* 6(dst)+6(src)+2(type) */ |
363 | hlen = sizeof(struct ether_header); | | 363 | hlen = sizeof(struct ether_header); |
364 | align = 2; | | 364 | align = 2; |
365 | break; | | 365 | break; |
366 | | | 366 | |
367 | case DLT_ARCNET: | | 367 | case DLT_ARCNET: |
368 | sockp->sa_family = AF_UNSPEC; | | 368 | sockp->sa_family = AF_UNSPEC; |
369 | hlen = ARC_HDRLEN; | | 369 | hlen = ARC_HDRLEN; |
370 | align = 5; | | 370 | align = 5; |
371 | break; | | 371 | break; |
372 | | | 372 | |
373 | case DLT_FDDI: | | 373 | case DLT_FDDI: |
374 | sockp->sa_family = AF_LINK; | | 374 | sockp->sa_family = AF_LINK; |
375 | /* XXX 4(FORMAC)+6(dst)+6(src) */ | | 375 | /* XXX 4(FORMAC)+6(dst)+6(src) */ |
376 | hlen = 16; | | 376 | hlen = 16; |
377 | align = 0; | | 377 | align = 0; |
378 | break; | | 378 | break; |
379 | | | 379 | |
380 | case DLT_ECONET: | | 380 | case DLT_ECONET: |
381 | sockp->sa_family = AF_UNSPEC; | | 381 | sockp->sa_family = AF_UNSPEC; |
382 | hlen = 6; | | 382 | hlen = 6; |
383 | align = 2; | | 383 | align = 2; |
384 | break; | | 384 | break; |
385 | | | 385 | |
386 | case DLT_NULL: | | 386 | case DLT_NULL: |
387 | sockp->sa_family = AF_UNSPEC; | | 387 | sockp->sa_family = AF_UNSPEC; |
388 | hlen = 0; | | 388 | hlen = 0; |
389 | align = 0; | | 389 | align = 0; |
390 | break; | | 390 | break; |
391 | | | 391 | |
392 | default: | | 392 | default: |
393 | return (EIO); | | 393 | return (EIO); |
394 | } | | 394 | } |
395 | | | 395 | |
396 | len = uio->uio_resid; | | 396 | len = uio->uio_resid; |
397 | /* | | 397 | /* |
398 | * If there aren't enough bytes for a link level header or the | | 398 | * If there aren't enough bytes for a link level header or the |
399 | * packet length exceeds the interface mtu, return an error. | | 399 | * packet length exceeds the interface mtu, return an error. |
400 | */ | | 400 | */ |
401 | if (len - hlen > mtu) | | 401 | if (len - hlen > mtu) |
402 | return (EMSGSIZE); | | 402 | return (EMSGSIZE); |
403 | | | 403 | |
404 | m0 = m = m_gethdr(M_WAIT, MT_DATA); | | 404 | m0 = m = m_gethdr(M_WAIT, MT_DATA); |
405 | m_reset_rcvif(m); | | 405 | m_reset_rcvif(m); |
406 | m->m_pkthdr.len = (int)(len - hlen); | | 406 | m->m_pkthdr.len = (int)(len - hlen); |
407 | if (len + align > MHLEN) { | | 407 | if (len + align > MHLEN) { |
408 | m_clget(m, M_WAIT); | | 408 | m_clget(m, M_WAIT); |
409 | if ((m->m_flags & M_EXT) == 0) { | | 409 | if ((m->m_flags & M_EXT) == 0) { |
410 | error = ENOBUFS; | | 410 | error = ENOBUFS; |
411 | goto bad; | | 411 | goto bad; |
412 | } | | 412 | } |
413 | } | | 413 | } |
414 | | | 414 | |
415 | /* Insure the data is properly aligned */ | | 415 | /* Insure the data is properly aligned */ |
416 | if (align > 0) | | 416 | if (align > 0) |
417 | m->m_data += align; | | 417 | m->m_data += align; |
418 | | | 418 | |
419 | for (;;) { | | 419 | for (;;) { |
420 | len = M_TRAILINGSPACE(m); | | 420 | len = M_TRAILINGSPACE(m); |
421 | if (len > uio->uio_resid) | | 421 | if (len > uio->uio_resid) |
422 | len = uio->uio_resid; | | 422 | len = uio->uio_resid; |
423 | error = uiomove(mtod(m, void *), len, uio); | | 423 | error = uiomove(mtod(m, void *), len, uio); |
424 | if (error) | | 424 | if (error) |
425 | goto bad; | | 425 | goto bad; |
426 | m->m_len = len; | | 426 | m->m_len = len; |
427 | | | 427 | |
428 | if (uio->uio_resid == 0) | | 428 | if (uio->uio_resid == 0) |
429 | break; | | 429 | break; |
430 | | | 430 | |
431 | n = m_get(M_WAIT, MT_DATA); | | 431 | n = m_get(M_WAIT, MT_DATA); |
432 | m_clget(n, M_WAIT); /* if fails, there is no problem */ | | 432 | m_clget(n, M_WAIT); /* if fails, there is no problem */ |
433 | m->m_next = n; | | 433 | m->m_next = n; |
434 | m = n; | | 434 | m = n; |
435 | } | | 435 | } |
436 | | | 436 | |
437 | slen = bpf_xfilter(wfilter, mtod(m, u_char *), len, len); | | 437 | slen = bpf_xfilter(wfilter, mtod(m, u_char *), len, len); |
438 | if (slen == 0) { | | 438 | if (slen == 0) { |
439 | error = EPERM; | | 439 | error = EPERM; |
440 | goto bad; | | 440 | goto bad; |
441 | } | | 441 | } |
442 | | | 442 | |
443 | if (hlen != 0) { | | 443 | if (hlen != 0) { |
444 | /* move link level header in the top of mbuf to sa_data */ | | 444 | /* move link level header in the top of mbuf to sa_data */ |
445 | memcpy(sockp->sa_data, mtod(m0, void *), hlen); | | 445 | memcpy(sockp->sa_data, mtod(m0, void *), hlen); |
446 | m0->m_data += hlen; | | 446 | m0->m_data += hlen; |
447 | m0->m_len -= hlen; | | 447 | m0->m_len -= hlen; |
448 | } | | 448 | } |
449 | | | 449 | |
450 | *mp = m0; | | 450 | *mp = m0; |
451 | return (0); | | 451 | return (0); |
452 | | | 452 | |
453 | bad: | | 453 | bad: |
454 | m_freem(m0); | | 454 | m_freem(m0); |
455 | return (error); | | 455 | return (error); |
456 | } | | 456 | } |
457 | | | 457 | |
458 | /* | | 458 | /* |
459 | * Attach file to the bpf interface, i.e. make d listen on bp. | | 459 | * Attach file to the bpf interface, i.e. make d listen on bp. |
460 | */ | | 460 | */ |
461 | static void | | 461 | static void |
462 | bpf_attachd(struct bpf_d *d, struct bpf_if *bp) | | 462 | bpf_attachd(struct bpf_d *d, struct bpf_if *bp) |
463 | { | | 463 | { |
464 | struct bpf_event_tracker *t; | | 464 | struct bpf_event_tracker *t; |
465 | | | 465 | |
466 | KASSERT(mutex_owned(&bpf_mtx)); | | 466 | KASSERT(mutex_owned(&bpf_mtx)); |
467 | KASSERT(mutex_owned(d->bd_mtx)); | | 467 | KASSERT(mutex_owned(d->bd_mtx)); |
468 | /* | | 468 | /* |
469 | * Point d at bp, and add d to the interface's list of listeners. | | 469 | * Point d at bp, and add d to the interface's list of listeners. |
470 | * Finally, point the driver's bpf cookie at the interface so | | 470 | * Finally, point the driver's bpf cookie at the interface so |
471 | * it will divert packets to bpf. | | 471 | * it will divert packets to bpf. |
472 | */ | | 472 | */ |
473 | d->bd_bif = bp; | | 473 | d->bd_bif = bp; |
474 | BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d); | | 474 | BPFIF_DLIST_WRITER_INSERT_HEAD(bp, d); |
475 | | | 475 | |
476 | *bp->bif_driverp = bp; | | 476 | *bp->bif_driverp = bp; |
477 | | | 477 | |
478 | SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) { | | 478 | SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) { |
479 | t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt, | | 479 | t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt, |
480 | BPF_TRACK_EVENT_ATTACH); | | 480 | BPF_TRACK_EVENT_ATTACH); |
481 | } | | 481 | } |
482 | } | | 482 | } |
483 | | | 483 | |
484 | /* | | 484 | /* |
485 | * Detach a file from its interface. | | 485 | * Detach a file from its interface. |
486 | */ | | 486 | */ |
487 | static void | | 487 | static void |
488 | bpf_detachd(struct bpf_d *d) | | 488 | bpf_detachd(struct bpf_d *d) |
489 | { | | 489 | { |
490 | struct bpf_if *bp; | | 490 | struct bpf_if *bp; |
491 | struct bpf_event_tracker *t; | | 491 | struct bpf_event_tracker *t; |
492 | | | 492 | |
493 | KASSERT(mutex_owned(&bpf_mtx)); | | 493 | KASSERT(mutex_owned(&bpf_mtx)); |
494 | KASSERT(mutex_owned(d->bd_mtx)); | | 494 | KASSERT(mutex_owned(d->bd_mtx)); |
495 | | | 495 | |
496 | bp = d->bd_bif; | | 496 | bp = d->bd_bif; |
497 | /* | | 497 | /* |
498 | * Check if this descriptor had requested promiscuous mode. | | 498 | * Check if this descriptor had requested promiscuous mode. |
499 | * If so, turn it off. | | 499 | * If so, turn it off. |
500 | */ | | 500 | */ |
501 | if (d->bd_promisc) { | | 501 | if (d->bd_promisc) { |
502 | int error __diagused; | | 502 | int error __diagused; |
503 | | | 503 | |
504 | d->bd_promisc = 0; | | 504 | d->bd_promisc = 0; |
505 | /* | | 505 | /* |
506 | * Take device out of promiscuous mode. Since we were | | 506 | * Take device out of promiscuous mode. Since we were |
507 | * able to enter promiscuous mode, we should be able | | 507 | * able to enter promiscuous mode, we should be able |
508 | * to turn it off. But we can get an error if | | 508 | * to turn it off. But we can get an error if |
509 | * the interface was configured down, so only panic | | 509 | * the interface was configured down, so only panic |
510 | * if we don't get an unexpected error. | | 510 | * if we don't get an unexpected error. |
511 | */ | | 511 | */ |
512 | KERNEL_LOCK_UNLESS_NET_MPSAFE(); | | 512 | KERNEL_LOCK_UNLESS_NET_MPSAFE(); |
513 | error = ifpromisc(bp->bif_ifp, 0); | | 513 | error = ifpromisc(bp->bif_ifp, 0); |
514 | KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); | | 514 | KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); |
515 | #ifdef DIAGNOSTIC | | 515 | #ifdef DIAGNOSTIC |
516 | if (error) | | 516 | if (error) |
517 | printf("%s: ifpromisc failed: %d", __func__, error); | | 517 | printf("%s: ifpromisc failed: %d", __func__, error); |
518 | #endif | | 518 | #endif |
519 | } | | 519 | } |
520 | | | 520 | |
521 | /* Remove d from the interface's descriptor list. */ | | 521 | /* Remove d from the interface's descriptor list. */ |
522 | BPFIF_DLIST_WRITER_REMOVE(d); | | 522 | BPFIF_DLIST_WRITER_REMOVE(d); |
523 | | | 523 | |
524 | pserialize_perform(bpf_psz); | | 524 | pserialize_perform(bpf_psz); |
525 | | | 525 | |
526 | if (BPFIF_DLIST_WRITER_EMPTY(bp)) { | | 526 | if (BPFIF_DLIST_WRITER_EMPTY(bp)) { |
527 | /* | | 527 | /* |
528 | * Let the driver know that there are no more listeners. | | 528 | * Let the driver know that there are no more listeners. |
529 | */ | | 529 | */ |
530 | *d->bd_bif->bif_driverp = NULL; | | 530 | *d->bd_bif->bif_driverp = NULL; |
531 | } | | 531 | } |
532 | | | 532 | |
533 | d->bd_bif = NULL; | | 533 | d->bd_bif = NULL; |
534 | | | 534 | |
535 | SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) { | | 535 | SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) { |
536 | t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt, | | 536 | t->bet_notify(bp, bp->bif_ifp, bp->bif_dlt, |
537 | BPF_TRACK_EVENT_DETACH); | | 537 | BPF_TRACK_EVENT_DETACH); |
538 | } | | 538 | } |
539 | } | | 539 | } |
540 | | | 540 | |
541 | static void | | 541 | static void |
542 | bpf_init(void) | | 542 | bpf_init(void) |
543 | { | | 543 | { |
544 | | | 544 | |
545 | mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); | | 545 | mutex_init(&bpf_mtx, MUTEX_DEFAULT, IPL_NONE); |
546 | bpf_psz = pserialize_create(); | | 546 | bpf_psz = pserialize_create(); |
547 | bpf_psref_class = psref_class_create("bpf", IPL_SOFTNET); | | 547 | bpf_psref_class = psref_class_create("bpf", IPL_SOFTNET); |
548 | | | 548 | |
549 | PSLIST_INIT(&bpf_iflist); | | 549 | PSLIST_INIT(&bpf_iflist); |
550 | PSLIST_INIT(&bpf_dlist); | | 550 | PSLIST_INIT(&bpf_dlist); |
551 | | | 551 | |
552 | bpf_gstats_percpu = percpu_alloc(sizeof(struct bpf_stat)); | | 552 | bpf_gstats_percpu = percpu_alloc(sizeof(struct bpf_stat)); |
553 | | | 553 | |
554 | return; | | 554 | return; |
555 | } | | 555 | } |
556 | | | 556 | |
557 | /* | | 557 | /* |
558 | * bpfilterattach() is called at boot time. We don't need to do anything | | 558 | * bpfilterattach() is called at boot time. We don't need to do anything |
559 | * here, since any initialization will happen as part of module init code. | | 559 | * here, since any initialization will happen as part of module init code. |
560 | */ | | 560 | */ |
561 | /* ARGSUSED */ | | 561 | /* ARGSUSED */ |
562 | void | | 562 | void |
563 | bpfilterattach(int n) | | 563 | bpfilterattach(int n) |
564 | { | | 564 | { |
565 | | | 565 | |
566 | } | | 566 | } |
567 | | | 567 | |
568 | /* | | 568 | /* |
569 | * Open ethernet device. Clones. | | 569 | * Open ethernet device. Clones. |
570 | */ | | 570 | */ |
571 | /* ARGSUSED */ | | 571 | /* ARGSUSED */ |
572 | int | | 572 | int |
573 | bpfopen(dev_t dev, int flag, int mode, struct lwp *l) | | 573 | bpfopen(dev_t dev, int flag, int mode, struct lwp *l) |
574 | { | | 574 | { |
575 | struct bpf_d *d; | | 575 | struct bpf_d *d; |
576 | struct file *fp; | | 576 | struct file *fp; |
577 | int error, fd; | | 577 | int error, fd; |
578 | | | 578 | |
579 | /* falloc() will fill in the descriptor for us. */ | | 579 | /* falloc() will fill in the descriptor for us. */ |
580 | if ((error = fd_allocfile(&fp, &fd)) != 0) | | 580 | if ((error = fd_allocfile(&fp, &fd)) != 0) |
581 | return error; | | 581 | return error; |
582 | | | 582 | |
583 | d = kmem_zalloc(sizeof(*d), KM_SLEEP); | | 583 | d = kmem_zalloc(sizeof(*d), KM_SLEEP); |
584 | d->bd_bufsize = bpf_bufsize; | | 584 | d->bd_bufsize = bpf_bufsize; |
585 | d->bd_direction = BPF_D_INOUT; | | 585 | d->bd_direction = BPF_D_INOUT; |
586 | d->bd_feedback = 0; | | 586 | d->bd_feedback = 0; |
587 | d->bd_pid = l->l_proc->p_pid; | | 587 | d->bd_pid = l->l_proc->p_pid; |
588 | #ifdef _LP64 | | 588 | #ifdef _LP64 |
589 | if (curproc->p_flag & PK_32) | | 589 | if (curproc->p_flag & PK_32) |
590 | d->bd_compat32 = 1; | | 590 | d->bd_compat32 = 1; |
591 | #endif | | 591 | #endif |
592 | getnanotime(&d->bd_btime); | | 592 | getnanotime(&d->bd_btime); |
593 | d->bd_atime = d->bd_mtime = d->bd_btime; | | 593 | d->bd_atime = d->bd_mtime = d->bd_btime; |
594 | callout_init(&d->bd_callout, CALLOUT_MPSAFE); | | 594 | callout_init(&d->bd_callout, CALLOUT_MPSAFE); |
595 | selinit(&d->bd_sel); | | 595 | selinit(&d->bd_sel); |
596 | d->bd_jitcode = NULL; | | 596 | d->bd_jitcode = NULL; |
597 | d->bd_rfilter = NULL; | | 597 | d->bd_rfilter = NULL; |
598 | d->bd_wfilter = NULL; | | 598 | d->bd_wfilter = NULL; |
599 | d->bd_locked = 0; | | 599 | d->bd_locked = 0; |
600 | BPF_DLIST_ENTRY_INIT(d); | | 600 | BPF_DLIST_ENTRY_INIT(d); |
601 | BPFIF_DLIST_ENTRY_INIT(d); | | 601 | BPFIF_DLIST_ENTRY_INIT(d); |
602 | d->bd_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); | | 602 | d->bd_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET); |
603 | d->bd_buf_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); | | 603 | d->bd_buf_mtx = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET); |
604 | cv_init(&d->bd_cv, "bpf"); | | 604 | cv_init(&d->bd_cv, "bpf"); |
605 | | | 605 | |
606 | mutex_enter(&bpf_mtx); | | 606 | mutex_enter(&bpf_mtx); |
607 | BPF_DLIST_WRITER_INSERT_HEAD(d); | | 607 | BPF_DLIST_WRITER_INSERT_HEAD(d); |
608 | mutex_exit(&bpf_mtx); | | 608 | mutex_exit(&bpf_mtx); |
609 | | | 609 | |
610 | return fd_clone(fp, fd, flag, &bpf_fileops, d); | | 610 | return fd_clone(fp, fd, flag, &bpf_fileops, d); |
611 | } | | 611 | } |
612 | | | 612 | |
613 | /* | | 613 | /* |
614 | * Close the descriptor by detaching it from its interface, | | 614 | * Close the descriptor by detaching it from its interface, |
615 | * deallocating its buffers, and marking it free. | | 615 | * deallocating its buffers, and marking it free. |
616 | */ | | 616 | */ |
617 | /* ARGSUSED */ | | 617 | /* ARGSUSED */ |
618 | static int | | 618 | static int |
619 | bpf_close(struct file *fp) | | 619 | bpf_close(struct file *fp) |
620 | { | | 620 | { |
621 | struct bpf_d *d; | | 621 | struct bpf_d *d; |
622 | | | 622 | |
623 | mutex_enter(&bpf_mtx); | | 623 | mutex_enter(&bpf_mtx); |
624 | | | 624 | |
625 | if ((d = fp->f_bpf) == NULL) { | | 625 | if ((d = fp->f_bpf) == NULL) { |
626 | mutex_exit(&bpf_mtx); | | 626 | mutex_exit(&bpf_mtx); |
627 | return 0; | | 627 | return 0; |
628 | } | | 628 | } |
629 | | | 629 | |
630 | /* | | 630 | /* |
631 | * Refresh the PID associated with this bpf file. | | 631 | * Refresh the PID associated with this bpf file. |
632 | */ | | 632 | */ |
633 | d->bd_pid = curproc->p_pid; | | 633 | d->bd_pid = curproc->p_pid; |
634 | | | 634 | |
635 | mutex_enter(d->bd_mtx); | | 635 | mutex_enter(d->bd_mtx); |
636 | if (d->bd_state == BPF_WAITING) | | 636 | if (d->bd_state == BPF_WAITING) |
637 | callout_halt(&d->bd_callout, d->bd_mtx); | | 637 | callout_halt(&d->bd_callout, d->bd_mtx); |
638 | d->bd_state = BPF_IDLE; | | 638 | d->bd_state = BPF_IDLE; |
639 | if (d->bd_bif) | | 639 | if (d->bd_bif) |
640 | bpf_detachd(d); | | 640 | bpf_detachd(d); |
641 | mutex_exit(d->bd_mtx); | | 641 | mutex_exit(d->bd_mtx); |
642 | | | 642 | |
643 | BPF_DLIST_WRITER_REMOVE(d); | | 643 | BPF_DLIST_WRITER_REMOVE(d); |
644 | | | 644 | |
645 | pserialize_perform(bpf_psz); | | 645 | pserialize_perform(bpf_psz); |
646 | mutex_exit(&bpf_mtx); | | 646 | mutex_exit(&bpf_mtx); |
647 | | | 647 | |
648 | BPFIF_DLIST_ENTRY_DESTROY(d); | | 648 | BPFIF_DLIST_ENTRY_DESTROY(d); |
649 | BPF_DLIST_ENTRY_DESTROY(d); | | 649 | BPF_DLIST_ENTRY_DESTROY(d); |
650 | fp->f_bpf = NULL; | | 650 | fp->f_bpf = NULL; |
651 | bpf_freed(d); | | 651 | bpf_freed(d); |
652 | callout_destroy(&d->bd_callout); | | 652 | callout_destroy(&d->bd_callout); |
653 | seldestroy(&d->bd_sel); | | 653 | seldestroy(&d->bd_sel); |
654 | mutex_obj_free(d->bd_mtx); | | 654 | mutex_obj_free(d->bd_mtx); |
655 | mutex_obj_free(d->bd_buf_mtx); | | 655 | mutex_obj_free(d->bd_buf_mtx); |
656 | cv_destroy(&d->bd_cv); | | 656 | cv_destroy(&d->bd_cv); |
657 | | | 657 | |
658 | kmem_free(d, sizeof(*d)); | | 658 | kmem_free(d, sizeof(*d)); |
659 | | | 659 | |
660 | return (0); | | 660 | return (0); |
661 | } | | 661 | } |
662 | | | 662 | |
663 | /* | | 663 | /* |
664 | * Rotate the packet buffers in descriptor d. Move the store buffer | | 664 | * Rotate the packet buffers in descriptor d. Move the store buffer |
665 | * into the hold slot, and the free buffer into the store slot. | | 665 | * into the hold slot, and the free buffer into the store slot. |
666 | * Zero the length of the new store buffer. | | 666 | * Zero the length of the new store buffer. |
667 | */ | | 667 | */ |
668 | #define ROTATE_BUFFERS(d) \ | | 668 | #define ROTATE_BUFFERS(d) \ |
669 | (d)->bd_hbuf = (d)->bd_sbuf; \ | | 669 | (d)->bd_hbuf = (d)->bd_sbuf; \ |
670 | (d)->bd_hlen = (d)->bd_slen; \ | | 670 | (d)->bd_hlen = (d)->bd_slen; \ |
671 | (d)->bd_sbuf = (d)->bd_fbuf; \ | | 671 | (d)->bd_sbuf = (d)->bd_fbuf; \ |
672 | (d)->bd_slen = 0; \ | | 672 | (d)->bd_slen = 0; \ |
673 | (d)->bd_fbuf = NULL; | | 673 | (d)->bd_fbuf = NULL; |
674 | /* | | 674 | /* |
675 | * bpfread - read next chunk of packets from buffers | | 675 | * bpfread - read next chunk of packets from buffers |
676 | */ | | 676 | */ |
677 | static int | | 677 | static int |
678 | bpf_read(struct file *fp, off_t *offp, struct uio *uio, | | 678 | bpf_read(struct file *fp, off_t *offp, struct uio *uio, |
679 | kauth_cred_t cred, int flags) | | 679 | kauth_cred_t cred, int flags) |
680 | { | | 680 | { |
681 | struct bpf_d *d = fp->f_bpf; | | 681 | struct bpf_d *d = fp->f_bpf; |
682 | int timed_out; | | 682 | int timed_out; |
683 | int error; | | 683 | int error; |
684 | | | 684 | |
685 | getnanotime(&d->bd_atime); | | 685 | getnanotime(&d->bd_atime); |
686 | /* | | 686 | /* |
687 | * Restrict application to use a buffer the same size as | | 687 | * Restrict application to use a buffer the same size as |
688 | * the kernel buffers. | | 688 | * the kernel buffers. |
689 | */ | | 689 | */ |
690 | if (uio->uio_resid != d->bd_bufsize) | | 690 | if (uio->uio_resid != d->bd_bufsize) |
691 | return (EINVAL); | | 691 | return (EINVAL); |
692 | | | 692 | |
693 | mutex_enter(d->bd_mtx); | | 693 | mutex_enter(d->bd_mtx); |
694 | if (d->bd_state == BPF_WAITING) | | 694 | if (d->bd_state == BPF_WAITING) |
695 | callout_halt(&d->bd_callout, d->bd_mtx); | | 695 | callout_halt(&d->bd_callout, d->bd_mtx); |
696 | timed_out = (d->bd_state == BPF_TIMED_OUT); | | 696 | timed_out = (d->bd_state == BPF_TIMED_OUT); |
697 | d->bd_state = BPF_IDLE; | | 697 | d->bd_state = BPF_IDLE; |
698 | mutex_exit(d->bd_mtx); | | 698 | mutex_exit(d->bd_mtx); |
699 | /* | | 699 | /* |
700 | * If the hold buffer is empty, then do a timed sleep, which | | 700 | * If the hold buffer is empty, then do a timed sleep, which |
701 | * ends when the timeout expires or when enough packets | | 701 | * ends when the timeout expires or when enough packets |
702 | * have arrived to fill the store buffer. | | 702 | * have arrived to fill the store buffer. |
703 | */ | | 703 | */ |
704 | mutex_enter(d->bd_buf_mtx); | | 704 | mutex_enter(d->bd_buf_mtx); |
705 | while (d->bd_hbuf == NULL) { | | 705 | while (d->bd_hbuf == NULL) { |
706 | if (fp->f_flag & FNONBLOCK) { | | 706 | if (fp->f_flag & FNONBLOCK) { |
707 | if (d->bd_slen == 0) { | | 707 | if (d->bd_slen == 0) { |
708 | error = EWOULDBLOCK; | | 708 | error = EWOULDBLOCK; |
709 | goto out; | | 709 | goto out; |
710 | } | | 710 | } |
711 | ROTATE_BUFFERS(d); | | 711 | ROTATE_BUFFERS(d); |
712 | break; | | 712 | break; |
713 | } | | 713 | } |
714 | | | 714 | |
715 | if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { | | 715 | if ((d->bd_immediate || timed_out) && d->bd_slen != 0) { |
716 | /* | | 716 | /* |
717 | * A packet(s) either arrived since the previous | | 717 | * A packet(s) either arrived since the previous |
718 | * read or arrived while we were asleep. | | 718 | * read or arrived while we were asleep. |
719 | * Rotate the buffers and return what's here. | | 719 | * Rotate the buffers and return what's here. |
720 | */ | | 720 | */ |
721 | ROTATE_BUFFERS(d); | | 721 | ROTATE_BUFFERS(d); |
722 | break; | | 722 | break; |
723 | } | | 723 | } |
724 | | | 724 | |
725 | error = cv_timedwait_sig(&d->bd_cv, d->bd_buf_mtx, d->bd_rtout); | | 725 | error = cv_timedwait_sig(&d->bd_cv, d->bd_buf_mtx, d->bd_rtout); |
726 | | | 726 | |
727 | if (error == EINTR || error == ERESTART) | | 727 | if (error == EINTR || error == ERESTART) |
728 | goto out; | | 728 | goto out; |
729 | | | 729 | |
730 | if (error == EWOULDBLOCK) { | | 730 | if (error == EWOULDBLOCK) { |
731 | /* | | 731 | /* |
732 | * On a timeout, return what's in the buffer, | | 732 | * On a timeout, return what's in the buffer, |
733 | * which may be nothing. If there is something | | 733 | * which may be nothing. If there is something |
734 | * in the store buffer, we can rotate the buffers. | | 734 | * in the store buffer, we can rotate the buffers. |
735 | */ | | 735 | */ |
736 | if (d->bd_hbuf) | | 736 | if (d->bd_hbuf) |
737 | /* | | 737 | /* |
738 | * We filled up the buffer in between | | 738 | * We filled up the buffer in between |
739 | * getting the timeout and arriving | | 739 | * getting the timeout and arriving |
740 | * here, so we don't need to rotate. | | 740 | * here, so we don't need to rotate. |
741 | */ | | 741 | */ |
742 | break; | | 742 | break; |
743 | | | 743 | |
744 | if (d->bd_slen == 0) { | | 744 | if (d->bd_slen == 0) { |
745 | error = 0; | | 745 | error = 0; |
746 | goto out; | | 746 | goto out; |
747 | } | | 747 | } |
748 | ROTATE_BUFFERS(d); | | 748 | ROTATE_BUFFERS(d); |
749 | break; | | 749 | break; |
750 | } | | 750 | } |
751 | if (error != 0) | | 751 | if (error != 0) |
752 | goto out; | | 752 | goto out; |
753 | } | | 753 | } |
754 | /* | | 754 | /* |
755 | * At this point, we know we have something in the hold slot. | | 755 | * At this point, we know we have something in the hold slot. |
756 | */ | | 756 | */ |
757 | mutex_exit(d->bd_buf_mtx); | | 757 | mutex_exit(d->bd_buf_mtx); |
758 | | | 758 | |
759 | /* | | 759 | /* |
760 | * Move data from hold buffer into user space. | | 760 | * Move data from hold buffer into user space. |
761 | * We know the entire buffer is transferred since | | 761 | * We know the entire buffer is transferred since |
762 | * we checked above that the read buffer is bpf_bufsize bytes. | | 762 | * we checked above that the read buffer is bpf_bufsize bytes. |
763 | */ | | 763 | */ |
764 | error = uiomove(d->bd_hbuf, d->bd_hlen, uio); | | 764 | error = uiomove(d->bd_hbuf, d->bd_hlen, uio); |
765 | | | 765 | |
766 | mutex_enter(d->bd_buf_mtx); | | 766 | mutex_enter(d->bd_buf_mtx); |
767 | d->bd_fbuf = d->bd_hbuf; | | 767 | d->bd_fbuf = d->bd_hbuf; |
768 | d->bd_hbuf = NULL; | | 768 | d->bd_hbuf = NULL; |
769 | d->bd_hlen = 0; | | 769 | d->bd_hlen = 0; |
770 | out: | | 770 | out: |
771 | mutex_exit(d->bd_buf_mtx); | | 771 | mutex_exit(d->bd_buf_mtx); |
772 | return (error); | | 772 | return (error); |
773 | } | | 773 | } |
774 | | | 774 | |
775 | | | 775 | |
776 | /* | | 776 | /* |
777 | * If there are processes sleeping on this descriptor, wake them up. | | 777 | * If there are processes sleeping on this descriptor, wake them up. |
778 | */ | | 778 | */ |
779 | static inline void | | 779 | static inline void |
780 | bpf_wakeup(struct bpf_d *d) | | 780 | bpf_wakeup(struct bpf_d *d) |
781 | { | | 781 | { |
782 | | | 782 | |
783 | mutex_enter(d->bd_buf_mtx); | | 783 | mutex_enter(d->bd_buf_mtx); |
784 | cv_broadcast(&d->bd_cv); | | 784 | cv_broadcast(&d->bd_cv); |
785 | mutex_exit(d->bd_buf_mtx); | | 785 | mutex_exit(d->bd_buf_mtx); |
786 | | | 786 | |
787 | if (d->bd_async) | | 787 | if (d->bd_async) |
788 | fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); | | 788 | fownsignal(d->bd_pgid, SIGIO, 0, 0, NULL); |
789 | selnotify(&d->bd_sel, 0, 0); | | 789 | selnotify(&d->bd_sel, 0, 0); |
790 | } | | 790 | } |
791 | | | 791 | |
792 | static void | | 792 | static void |
793 | bpf_timed_out(void *arg) | | 793 | bpf_timed_out(void *arg) |
794 | { | | 794 | { |
795 | struct bpf_d *d = arg; | | 795 | struct bpf_d *d = arg; |
796 | | | 796 | |
797 | mutex_enter(d->bd_mtx); | | 797 | mutex_enter(d->bd_mtx); |
798 | if (d->bd_state == BPF_WAITING) { | | 798 | if (d->bd_state == BPF_WAITING) { |
799 | d->bd_state = BPF_TIMED_OUT; | | 799 | d->bd_state = BPF_TIMED_OUT; |
800 | if (d->bd_slen != 0) | | 800 | if (d->bd_slen != 0) |
801 | bpf_wakeup(d); | | 801 | bpf_wakeup(d); |
802 | } | | 802 | } |
803 | mutex_exit(d->bd_mtx); | | 803 | mutex_exit(d->bd_mtx); |
804 | } | | 804 | } |
805 | | | 805 | |
806 | | | 806 | |
807 | static int | | 807 | static int |
808 | bpf_write(struct file *fp, off_t *offp, struct uio *uio, | | 808 | bpf_write(struct file *fp, off_t *offp, struct uio *uio, |
809 | kauth_cred_t cred, int flags) | | 809 | kauth_cred_t cred, int flags) |
810 | { | | 810 | { |
811 | struct bpf_d *d = fp->f_bpf; | | 811 | struct bpf_d *d = fp->f_bpf; |
812 | struct bpf_if *bp; | | 812 | struct bpf_if *bp; |
813 | struct ifnet *ifp; | | 813 | struct ifnet *ifp; |
814 | struct mbuf *m, *mc; | | 814 | struct mbuf *m, *mc; |
815 | int error; | | 815 | int error; |
816 | static struct sockaddr_storage dst; | | 816 | static struct sockaddr_storage dst; |
817 | struct psref psref; | | 817 | struct psref psref; |
818 | int bound; | | 818 | int bound; |
819 | | | 819 | |
820 | m = NULL; /* XXX gcc */ | | 820 | m = NULL; /* XXX gcc */ |
821 | | | 821 | |
822 | bound = curlwp_bind(); | | 822 | bound = curlwp_bind(); |
823 | mutex_enter(d->bd_mtx); | | 823 | mutex_enter(d->bd_mtx); |
824 | bp = d->bd_bif; | | 824 | bp = d->bd_bif; |
825 | if (bp == NULL) { | | 825 | if (bp == NULL) { |
826 | mutex_exit(d->bd_mtx); | | 826 | mutex_exit(d->bd_mtx); |
827 | error = ENXIO; | | 827 | error = ENXIO; |
828 | goto out_bindx; | | 828 | goto out_bindx; |
829 | } | | 829 | } |
830 | bpf_if_acquire(bp, &psref); | | 830 | bpf_if_acquire(bp, &psref); |
831 | mutex_exit(d->bd_mtx); | | 831 | mutex_exit(d->bd_mtx); |
832 | | | 832 | |
833 | getnanotime(&d->bd_mtime); | | 833 | getnanotime(&d->bd_mtime); |
834 | | | 834 | |
835 | ifp = bp->bif_ifp; | | 835 | ifp = bp->bif_ifp; |
836 | if (if_is_deactivated(ifp)) { | | 836 | if (if_is_deactivated(ifp)) { |
837 | error = ENXIO; | | 837 | error = ENXIO; |
838 | goto out; | | 838 | goto out; |
839 | } | | 839 | } |
840 | | | 840 | |
841 | if (uio->uio_resid == 0) { | | 841 | if (uio->uio_resid == 0) { |
842 | error = 0; | | 842 | error = 0; |
843 | goto out; | | 843 | goto out; |
844 | } | | 844 | } |
845 | | | 845 | |
846 | error = bpf_movein(uio, (int)bp->bif_dlt, ifp->if_mtu, &m, | | 846 | error = bpf_movein(uio, (int)bp->bif_dlt, ifp->if_mtu, &m, |
847 | (struct sockaddr *) &dst, &d->bd_wfilter); | | 847 | (struct sockaddr *) &dst, &d->bd_wfilter); |
848 | if (error) | | 848 | if (error) |
849 | goto out; | | 849 | goto out; |
850 | | | 850 | |
851 | if (m->m_pkthdr.len > ifp->if_mtu) { | | 851 | if (m->m_pkthdr.len > ifp->if_mtu) { |
852 | m_freem(m); | | 852 | m_freem(m); |
853 | error = EMSGSIZE; | | 853 | error = EMSGSIZE; |
854 | goto out; | | 854 | goto out; |
855 | } | | 855 | } |
856 | | | 856 | |
857 | if (d->bd_hdrcmplt) | | 857 | if (d->bd_hdrcmplt) |
858 | dst.ss_family = pseudo_AF_HDRCMPLT; | | 858 | dst.ss_family = pseudo_AF_HDRCMPLT; |
859 | | | 859 | |
860 | if (d->bd_feedback) { | | 860 | if (d->bd_feedback) { |
861 | mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); | | 861 | mc = m_dup(m, 0, M_COPYALL, M_NOWAIT); |
862 | if (mc != NULL) | | 862 | if (mc != NULL) |
863 | m_set_rcvif(mc, ifp); | | 863 | m_set_rcvif(mc, ifp); |
864 | /* Set M_PROMISC for outgoing packets to be discarded. */ | | 864 | /* Set M_PROMISC for outgoing packets to be discarded. */ |
865 | if (1 /*d->bd_direction == BPF_D_INOUT*/) | | 865 | if (1 /*d->bd_direction == BPF_D_INOUT*/) |
866 | m->m_flags |= M_PROMISC; | | 866 | m->m_flags |= M_PROMISC; |
867 | } else | | 867 | } else |
868 | mc = NULL; | | 868 | mc = NULL; |
869 | | | 869 | |
870 | error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL); | | 870 | error = if_output_lock(ifp, ifp, m, (struct sockaddr *) &dst, NULL); |
871 | | | 871 | |
872 | if (mc != NULL) { | | 872 | if (mc != NULL) { |
873 | if (error == 0) { | | 873 | if (error == 0) { |
874 | int s = splsoftnet(); | | 874 | int s = splsoftnet(); |
875 | KERNEL_LOCK_UNLESS_IFP_MPSAFE(ifp); | | 875 | KERNEL_LOCK_UNLESS_IFP_MPSAFE(ifp); |
876 | ifp->_if_input(ifp, mc); | | 876 | ifp->_if_input(ifp, mc); |
877 | KERNEL_UNLOCK_UNLESS_IFP_MPSAFE(ifp); | | 877 | KERNEL_UNLOCK_UNLESS_IFP_MPSAFE(ifp); |
878 | splx(s); | | 878 | splx(s); |
879 | } else | | 879 | } else |
880 | m_freem(mc); | | 880 | m_freem(mc); |
881 | } | | 881 | } |
882 | /* | | 882 | /* |
883 | * The driver frees the mbuf. | | 883 | * The driver frees the mbuf. |
884 | */ | | 884 | */ |
885 | out: | | 885 | out: |
886 | bpf_if_release(bp, &psref); | | 886 | bpf_if_release(bp, &psref); |
887 | out_bindx: | | 887 | out_bindx: |
888 | curlwp_bindx(bound); | | 888 | curlwp_bindx(bound); |
889 | return error; | | 889 | return error; |
890 | } | | 890 | } |
891 | | | 891 | |
892 | /* | | 892 | /* |
893 | * Reset a descriptor by flushing its packet buffer and clearing the | | 893 | * Reset a descriptor by flushing its packet buffer and clearing the |
894 | * receive and drop counts. | | 894 | * receive and drop counts. |
895 | */ | | 895 | */ |
896 | static void | | 896 | static void |
897 | reset_d(struct bpf_d *d) | | 897 | reset_d(struct bpf_d *d) |
898 | { | | 898 | { |
899 | | | 899 | |
900 | KASSERT(mutex_owned(d->bd_mtx)); | | 900 | KASSERT(mutex_owned(d->bd_mtx)); |
901 | | | 901 | |
902 | mutex_enter(d->bd_buf_mtx); | | 902 | mutex_enter(d->bd_buf_mtx); |
903 | if (d->bd_hbuf) { | | 903 | if (d->bd_hbuf) { |
904 | /* Free the hold buffer. */ | | 904 | /* Free the hold buffer. */ |
905 | d->bd_fbuf = d->bd_hbuf; | | 905 | d->bd_fbuf = d->bd_hbuf; |
906 | d->bd_hbuf = NULL; | | 906 | d->bd_hbuf = NULL; |
907 | } | | 907 | } |
908 | d->bd_slen = 0; | | 908 | d->bd_slen = 0; |
909 | d->bd_hlen = 0; | | 909 | d->bd_hlen = 0; |
910 | d->bd_rcount = 0; | | 910 | d->bd_rcount = 0; |
911 | d->bd_dcount = 0; | | 911 | d->bd_dcount = 0; |
912 | d->bd_ccount = 0; | | 912 | d->bd_ccount = 0; |
913 | mutex_exit(d->bd_buf_mtx); | | 913 | mutex_exit(d->bd_buf_mtx); |
914 | } | | 914 | } |
915 | | | 915 | |
916 | /* | | 916 | /* |
917 | * FIONREAD Check for read packet available. | | 917 | * FIONREAD Check for read packet available. |
918 | * BIOCGBLEN Get buffer len [for read()]. | | 918 | * BIOCGBLEN Get buffer len [for read()]. |
919 | * BIOCSETF Set ethernet read filter. | | 919 | * BIOCSETF Set ethernet read filter. |
920 | * BIOCFLUSH Flush read packet buffer. | | 920 | * BIOCFLUSH Flush read packet buffer. |
921 | * BIOCPROMISC Put interface into promiscuous mode. | | 921 | * BIOCPROMISC Put interface into promiscuous mode. |
922 | * BIOCGDLT Get link layer type. | | 922 | * BIOCGDLT Get link layer type. |
923 | * BIOCGETIF Get interface name. | | 923 | * BIOCGETIF Get interface name. |
924 | * BIOCSETIF Set interface. | | 924 | * BIOCSETIF Set interface. |
925 | * BIOCSRTIMEOUT Set read timeout. | | 925 | * BIOCSRTIMEOUT Set read timeout. |
926 | * BIOCGRTIMEOUT Get read timeout. | | 926 | * BIOCGRTIMEOUT Get read timeout. |
927 | * BIOCGSTATS Get packet stats. | | 927 | * BIOCGSTATS Get packet stats. |
928 | * BIOCIMMEDIATE Set immediate mode. | | 928 | * BIOCIMMEDIATE Set immediate mode. |
929 | * BIOCVERSION Get filter language version. | | 929 | * BIOCVERSION Get filter language version. |
930 | * BIOCGHDRCMPLT Get "header already complete" flag. | | 930 | * BIOCGHDRCMPLT Get "header already complete" flag. |
931 | * BIOCSHDRCMPLT Set "header already complete" flag. | | 931 | * BIOCSHDRCMPLT Set "header already complete" flag. |
932 | * BIOCSFEEDBACK Set packet feedback mode. | | 932 | * BIOCSFEEDBACK Set packet feedback mode. |
933 | * BIOCGFEEDBACK Get packet feedback mode. | | 933 | * BIOCGFEEDBACK Get packet feedback mode. |
934 | * BIOCGDIRECTION Get packet direction flag | | 934 | * BIOCGDIRECTION Get packet direction flag |
935 | * BIOCSDIRECTION Set packet direction flag | | 935 | * BIOCSDIRECTION Set packet direction flag |
936 | */ | | 936 | */ |
937 | /* ARGSUSED */ | | 937 | /* ARGSUSED */ |
938 | static int | | 938 | static int |
939 | bpf_ioctl(struct file *fp, u_long cmd, void *addr) | | 939 | bpf_ioctl(struct file *fp, u_long cmd, void *addr) |
940 | { | | 940 | { |
941 | struct bpf_d *d = fp->f_bpf; | | 941 | struct bpf_d *d = fp->f_bpf; |
942 | int error = 0; | | 942 | int error = 0; |
943 | | | 943 | |
944 | /* | | 944 | /* |
945 | * Refresh the PID associated with this bpf file. | | 945 | * Refresh the PID associated with this bpf file. |
946 | */ | | 946 | */ |
947 | d->bd_pid = curproc->p_pid; | | 947 | d->bd_pid = curproc->p_pid; |
948 | #ifdef _LP64 | | 948 | #ifdef _LP64 |
949 | if (curproc->p_flag & PK_32) | | 949 | if (curproc->p_flag & PK_32) |
950 | d->bd_compat32 = 1; | | 950 | d->bd_compat32 = 1; |
951 | else | | 951 | else |
952 | d->bd_compat32 = 0; | | 952 | d->bd_compat32 = 0; |
953 | #endif | | 953 | #endif |
954 | | | 954 | |
955 | mutex_enter(d->bd_mtx); | | 955 | mutex_enter(d->bd_mtx); |
956 | if (d->bd_state == BPF_WAITING) | | 956 | if (d->bd_state == BPF_WAITING) |
957 | callout_halt(&d->bd_callout, d->bd_mtx); | | 957 | callout_halt(&d->bd_callout, d->bd_mtx); |
958 | d->bd_state = BPF_IDLE; | | 958 | d->bd_state = BPF_IDLE; |
959 | mutex_exit(d->bd_mtx); | | 959 | mutex_exit(d->bd_mtx); |
960 | | | 960 | |
961 | if (d->bd_locked) { | | 961 | if (d->bd_locked) { |
962 | switch (cmd) { | | 962 | switch (cmd) { |
963 | case BIOCGBLEN: /* FALLTHROUGH */ | | 963 | case BIOCGBLEN: /* FALLTHROUGH */ |
964 | case BIOCFLUSH: /* FALLTHROUGH */ | | 964 | case BIOCFLUSH: /* FALLTHROUGH */ |
965 | case BIOCGDLT: /* FALLTHROUGH */ | | 965 | case BIOCGDLT: /* FALLTHROUGH */ |
966 | case BIOCGDLTLIST: /* FALLTHROUGH */ | | 966 | case BIOCGDLTLIST: /* FALLTHROUGH */ |
967 | case BIOCGETIF: /* FALLTHROUGH */ | | 967 | case BIOCGETIF: /* FALLTHROUGH */ |
968 | case BIOCGRTIMEOUT: /* FALLTHROUGH */ | | 968 | case BIOCGRTIMEOUT: /* FALLTHROUGH */ |
969 | case BIOCGSTATS: /* FALLTHROUGH */ | | 969 | case BIOCGSTATS: /* FALLTHROUGH */ |
970 | case BIOCVERSION: /* FALLTHROUGH */ | | 970 | case BIOCVERSION: /* FALLTHROUGH */ |
971 | case BIOCGHDRCMPLT: /* FALLTHROUGH */ | | 971 | case BIOCGHDRCMPLT: /* FALLTHROUGH */ |
972 | case FIONREAD: /* FALLTHROUGH */ | | 972 | case FIONREAD: /* FALLTHROUGH */ |
973 | case BIOCLOCK: /* FALLTHROUGH */ | | 973 | case BIOCLOCK: /* FALLTHROUGH */ |
974 | case BIOCSRTIMEOUT: /* FALLTHROUGH */ | | 974 | case BIOCSRTIMEOUT: /* FALLTHROUGH */ |
975 | case BIOCIMMEDIATE: /* FALLTHROUGH */ | | 975 | case BIOCIMMEDIATE: /* FALLTHROUGH */ |
976 | case TIOCGPGRP: | | 976 | case TIOCGPGRP: |
977 | break; | | 977 | break; |
978 | default: | | 978 | default: |
979 | return EPERM; | | 979 | return EPERM; |
980 | } | | 980 | } |
981 | } | | 981 | } |
982 | | | 982 | |
983 | switch (cmd) { | | 983 | switch (cmd) { |
984 | | | 984 | |
985 | default: | | 985 | default: |
986 | error = EINVAL; | | 986 | error = EINVAL; |
987 | break; | | 987 | break; |
988 | | | 988 | |
989 | /* | | 989 | /* |
990 | * Check for read packet available. | | 990 | * Check for read packet available. |
991 | */ | | 991 | */ |
992 | case FIONREAD: | | 992 | case FIONREAD: |
993 | { | | 993 | { |
994 | int n; | | 994 | int n; |
995 | | | 995 | |
996 | mutex_enter(d->bd_buf_mtx); | | 996 | mutex_enter(d->bd_buf_mtx); |
997 | n = d->bd_slen; | | 997 | n = d->bd_slen; |
998 | if (d->bd_hbuf) | | 998 | if (d->bd_hbuf) |
999 | n += d->bd_hlen; | | 999 | n += d->bd_hlen; |
1000 | mutex_exit(d->bd_buf_mtx); | | 1000 | mutex_exit(d->bd_buf_mtx); |
1001 | | | 1001 | |
1002 | *(int *)addr = n; | | 1002 | *(int *)addr = n; |
1003 | break; | | 1003 | break; |
1004 | } | | 1004 | } |
1005 | | | 1005 | |
1006 | /* | | 1006 | /* |
1007 | * Get buffer len [for read()]. | | 1007 | * Get buffer len [for read()]. |
1008 | */ | | 1008 | */ |
1009 | case BIOCGBLEN: | | 1009 | case BIOCGBLEN: |
1010 | *(u_int *)addr = d->bd_bufsize; | | 1010 | *(u_int *)addr = d->bd_bufsize; |
1011 | break; | | 1011 | break; |
1012 | | | 1012 | |
1013 | /* | | 1013 | /* |
1014 | * Set buffer length. | | 1014 | * Set buffer length. |
1015 | */ | | 1015 | */ |
1016 | case BIOCSBLEN: | | 1016 | case BIOCSBLEN: |
1017 | /* | | 1017 | /* |
1018 | * Forbid to change the buffer length if buffers are already | | 1018 | * Forbid to change the buffer length if buffers are already |
1019 | * allocated. | | 1019 | * allocated. |
1020 | */ | | 1020 | */ |
1021 | mutex_enter(d->bd_mtx); | | 1021 | mutex_enter(d->bd_mtx); |
1022 | mutex_enter(d->bd_buf_mtx); | | 1022 | mutex_enter(d->bd_buf_mtx); |
1023 | if (d->bd_bif != NULL || d->bd_sbuf != NULL) | | 1023 | if (d->bd_bif != NULL || d->bd_sbuf != NULL) |
1024 | error = EINVAL; | | 1024 | error = EINVAL; |
1025 | else { | | 1025 | else { |
1026 | u_int size = *(u_int *)addr; | | 1026 | u_int size = *(u_int *)addr; |
1027 | | | 1027 | |
1028 | if (size > bpf_maxbufsize) | | 1028 | if (size > bpf_maxbufsize) |
1029 | *(u_int *)addr = size = bpf_maxbufsize; | | 1029 | *(u_int *)addr = size = bpf_maxbufsize; |
1030 | else if (size < BPF_MINBUFSIZE) | | 1030 | else if (size < BPF_MINBUFSIZE) |
1031 | *(u_int *)addr = size = BPF_MINBUFSIZE; | | 1031 | *(u_int *)addr = size = BPF_MINBUFSIZE; |
1032 | d->bd_bufsize = size; | | 1032 | d->bd_bufsize = size; |
1033 | } | | 1033 | } |
1034 | mutex_exit(d->bd_buf_mtx); | | 1034 | mutex_exit(d->bd_buf_mtx); |
1035 | mutex_exit(d->bd_mtx); | | 1035 | mutex_exit(d->bd_mtx); |
1036 | break; | | 1036 | break; |
1037 | | | 1037 | |
1038 | /* | | 1038 | /* |
1039 | * Set link layer read filter. | | 1039 | * Set link layer read filter. |
1040 | */ | | 1040 | */ |
1041 | case BIOCSETF: /* FALLTHROUGH */ | | 1041 | case BIOCSETF: /* FALLTHROUGH */ |
| @@ -1198,1481 +1198,1480 @@ bpf_ioctl(struct file *fp, u_long cmd, v | | | @@ -1198,1481 +1198,1480 @@ bpf_ioctl(struct file *fp, u_long cmd, v |
1198 | tv->tv_sec = d->bd_rtout / hz; | | 1198 | tv->tv_sec = d->bd_rtout / hz; |
1199 | tv->tv_usec = (d->bd_rtout % hz) * tick; | | 1199 | tv->tv_usec = (d->bd_rtout % hz) * tick; |
1200 | break; | | 1200 | break; |
1201 | } | | 1201 | } |
1202 | /* | | 1202 | /* |
1203 | * Get packet stats. | | 1203 | * Get packet stats. |
1204 | */ | | 1204 | */ |
1205 | case BIOCGSTATS: | | 1205 | case BIOCGSTATS: |
1206 | { | | 1206 | { |
1207 | struct bpf_stat *bs = addr; | | 1207 | struct bpf_stat *bs = addr; |
1208 | | | 1208 | |
1209 | bs->bs_recv = d->bd_rcount; | | 1209 | bs->bs_recv = d->bd_rcount; |
1210 | bs->bs_drop = d->bd_dcount; | | 1210 | bs->bs_drop = d->bd_dcount; |
1211 | bs->bs_capt = d->bd_ccount; | | 1211 | bs->bs_capt = d->bd_ccount; |
1212 | break; | | 1212 | break; |
1213 | } | | 1213 | } |
1214 | | | 1214 | |
1215 | case BIOCGSTATSOLD: | | 1215 | case BIOCGSTATSOLD: |
1216 | { | | 1216 | { |
1217 | struct bpf_stat_old *bs = addr; | | 1217 | struct bpf_stat_old *bs = addr; |
1218 | | | 1218 | |
1219 | bs->bs_recv = d->bd_rcount; | | 1219 | bs->bs_recv = d->bd_rcount; |
1220 | bs->bs_drop = d->bd_dcount; | | 1220 | bs->bs_drop = d->bd_dcount; |
1221 | break; | | 1221 | break; |
1222 | } | | 1222 | } |
1223 | | | 1223 | |
1224 | /* | | 1224 | /* |
1225 | * Set immediate mode. | | 1225 | * Set immediate mode. |
1226 | */ | | 1226 | */ |
1227 | case BIOCIMMEDIATE: | | 1227 | case BIOCIMMEDIATE: |
1228 | d->bd_immediate = *(u_int *)addr; | | 1228 | d->bd_immediate = *(u_int *)addr; |
1229 | break; | | 1229 | break; |
1230 | | | 1230 | |
1231 | case BIOCVERSION: | | 1231 | case BIOCVERSION: |
1232 | { | | 1232 | { |
1233 | struct bpf_version *bv = addr; | | 1233 | struct bpf_version *bv = addr; |
1234 | | | 1234 | |
1235 | bv->bv_major = BPF_MAJOR_VERSION; | | 1235 | bv->bv_major = BPF_MAJOR_VERSION; |
1236 | bv->bv_minor = BPF_MINOR_VERSION; | | 1236 | bv->bv_minor = BPF_MINOR_VERSION; |
1237 | break; | | 1237 | break; |
1238 | } | | 1238 | } |
1239 | | | 1239 | |
1240 | case BIOCGHDRCMPLT: /* get "header already complete" flag */ | | 1240 | case BIOCGHDRCMPLT: /* get "header already complete" flag */ |
1241 | *(u_int *)addr = d->bd_hdrcmplt; | | 1241 | *(u_int *)addr = d->bd_hdrcmplt; |
1242 | break; | | 1242 | break; |
1243 | | | 1243 | |
1244 | case BIOCSHDRCMPLT: /* set "header already complete" flag */ | | 1244 | case BIOCSHDRCMPLT: /* set "header already complete" flag */ |
1245 | d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; | | 1245 | d->bd_hdrcmplt = *(u_int *)addr ? 1 : 0; |
1246 | break; | | 1246 | break; |
1247 | | | 1247 | |
1248 | /* | | 1248 | /* |
1249 | * Get packet direction flag | | 1249 | * Get packet direction flag |
1250 | */ | | 1250 | */ |
1251 | case BIOCGDIRECTION: | | 1251 | case BIOCGDIRECTION: |
1252 | *(u_int *)addr = d->bd_direction; | | 1252 | *(u_int *)addr = d->bd_direction; |
1253 | break; | | 1253 | break; |
1254 | | | 1254 | |
1255 | /* | | 1255 | /* |
1256 | * Set packet direction flag | | 1256 | * Set packet direction flag |
1257 | */ | | 1257 | */ |
1258 | case BIOCSDIRECTION: | | 1258 | case BIOCSDIRECTION: |
1259 | { | | 1259 | { |
1260 | u_int direction; | | 1260 | u_int direction; |
1261 | | | 1261 | |
1262 | direction = *(u_int *)addr; | | 1262 | direction = *(u_int *)addr; |
1263 | switch (direction) { | | 1263 | switch (direction) { |
1264 | case BPF_D_IN: | | 1264 | case BPF_D_IN: |
1265 | case BPF_D_INOUT: | | 1265 | case BPF_D_INOUT: |
1266 | case BPF_D_OUT: | | 1266 | case BPF_D_OUT: |
1267 | d->bd_direction = direction; | | 1267 | d->bd_direction = direction; |
1268 | break; | | 1268 | break; |
1269 | default: | | 1269 | default: |
1270 | error = EINVAL; | | 1270 | error = EINVAL; |
1271 | } | | 1271 | } |
1272 | } | | 1272 | } |
1273 | break; | | 1273 | break; |
1274 | | | 1274 | |
1275 | /* | | 1275 | /* |
1276 | * Set "feed packets from bpf back to input" mode | | 1276 | * Set "feed packets from bpf back to input" mode |
1277 | */ | | 1277 | */ |
1278 | case BIOCSFEEDBACK: | | 1278 | case BIOCSFEEDBACK: |
1279 | d->bd_feedback = *(u_int *)addr; | | 1279 | d->bd_feedback = *(u_int *)addr; |
1280 | break; | | 1280 | break; |
1281 | | | 1281 | |
1282 | /* | | 1282 | /* |
1283 | * Get "feed packets from bpf back to input" mode | | 1283 | * Get "feed packets from bpf back to input" mode |
1284 | */ | | 1284 | */ |
1285 | case BIOCGFEEDBACK: | | 1285 | case BIOCGFEEDBACK: |
1286 | *(u_int *)addr = d->bd_feedback; | | 1286 | *(u_int *)addr = d->bd_feedback; |
1287 | break; | | 1287 | break; |
1288 | | | 1288 | |
1289 | case FIONBIO: /* Non-blocking I/O */ | | 1289 | case FIONBIO: /* Non-blocking I/O */ |
1290 | /* | | 1290 | /* |
1291 | * No need to do anything special as we use IO_NDELAY in | | 1291 | * No need to do anything special as we use IO_NDELAY in |
1292 | * bpfread() as an indication of whether or not to block | | 1292 | * bpfread() as an indication of whether or not to block |
1293 | * the read. | | 1293 | * the read. |
1294 | */ | | 1294 | */ |
1295 | break; | | 1295 | break; |
1296 | | | 1296 | |
1297 | case FIOASYNC: /* Send signal on receive packets */ | | 1297 | case FIOASYNC: /* Send signal on receive packets */ |
1298 | mutex_enter(d->bd_mtx); | | 1298 | mutex_enter(d->bd_mtx); |
1299 | d->bd_async = *(int *)addr; | | 1299 | d->bd_async = *(int *)addr; |
1300 | mutex_exit(d->bd_mtx); | | 1300 | mutex_exit(d->bd_mtx); |
1301 | break; | | 1301 | break; |
1302 | | | 1302 | |
1303 | case TIOCSPGRP: /* Process or group to send signals to */ | | 1303 | case TIOCSPGRP: /* Process or group to send signals to */ |
1304 | case FIOSETOWN: | | 1304 | case FIOSETOWN: |
1305 | error = fsetown(&d->bd_pgid, cmd, addr); | | 1305 | error = fsetown(&d->bd_pgid, cmd, addr); |
1306 | break; | | 1306 | break; |
1307 | | | 1307 | |
1308 | case TIOCGPGRP: | | 1308 | case TIOCGPGRP: |
1309 | case FIOGETOWN: | | 1309 | case FIOGETOWN: |
1310 | error = fgetown(d->bd_pgid, cmd, addr); | | 1310 | error = fgetown(d->bd_pgid, cmd, addr); |
1311 | break; | | 1311 | break; |
1312 | } | | 1312 | } |
1313 | return (error); | | 1313 | return (error); |
1314 | } | | 1314 | } |
1315 | | | 1315 | |
1316 | /* | | 1316 | /* |
1317 | * Set d's packet filter program to fp. If this file already has a filter, | | 1317 | * Set d's packet filter program to fp. If this file already has a filter, |
1318 | * free it and replace it. Returns EINVAL for bogus requests. | | 1318 | * free it and replace it. Returns EINVAL for bogus requests. |
1319 | */ | | 1319 | */ |
1320 | static int | | 1320 | static int |
1321 | bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd) | | 1321 | bpf_setf(struct bpf_d *d, struct bpf_program *fp, u_long cmd) |
1322 | { | | 1322 | { |
1323 | struct bpf_insn *fcode; | | 1323 | struct bpf_insn *fcode; |
1324 | bpfjit_func_t jcode; | | 1324 | bpfjit_func_t jcode; |
1325 | size_t flen, size = 0; | | 1325 | size_t flen, size = 0; |
1326 | struct bpf_filter *oldf, *newf, **storef; | | 1326 | struct bpf_filter *oldf, *newf, **storef; |
1327 | | | 1327 | |
1328 | jcode = NULL; | | 1328 | jcode = NULL; |
1329 | flen = fp->bf_len; | | 1329 | flen = fp->bf_len; |
1330 | | | 1330 | |
1331 | if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { | | 1331 | if ((fp->bf_insns == NULL && flen) || flen > BPF_MAXINSNS) { |
1332 | return EINVAL; | | 1332 | return EINVAL; |
1333 | } | | 1333 | } |
1334 | | | 1334 | |
1335 | if (flen) { | | 1335 | if (flen) { |
1336 | /* | | 1336 | /* |
1337 | * Allocate the buffer, copy the byte-code from | | 1337 | * Allocate the buffer, copy the byte-code from |
1338 | * userspace and validate it. | | 1338 | * userspace and validate it. |
1339 | */ | | 1339 | */ |
1340 | size = flen * sizeof(*fp->bf_insns); | | 1340 | size = flen * sizeof(*fp->bf_insns); |
1341 | fcode = kmem_alloc(size, KM_SLEEP); | | 1341 | fcode = kmem_alloc(size, KM_SLEEP); |
1342 | if (copyin(fp->bf_insns, fcode, size) != 0 || | | 1342 | if (copyin(fp->bf_insns, fcode, size) != 0 || |
1343 | !bpf_validate(fcode, (int)flen)) { | | 1343 | !bpf_validate(fcode, (int)flen)) { |
1344 | kmem_free(fcode, size); | | 1344 | kmem_free(fcode, size); |
1345 | return EINVAL; | | 1345 | return EINVAL; |
1346 | } | | 1346 | } |
1347 | if (bpf_jit) | | 1347 | if (bpf_jit) |
1348 | jcode = bpf_jit_generate(NULL, fcode, flen); | | 1348 | jcode = bpf_jit_generate(NULL, fcode, flen); |
1349 | } else { | | 1349 | } else { |
1350 | fcode = NULL; | | 1350 | fcode = NULL; |
1351 | } | | 1351 | } |
1352 | | | 1352 | |
1353 | newf = kmem_alloc(sizeof(*newf), KM_SLEEP); | | 1353 | newf = kmem_alloc(sizeof(*newf), KM_SLEEP); |
1354 | newf->bf_insn = fcode; | | 1354 | newf->bf_insn = fcode; |
1355 | newf->bf_size = size; | | 1355 | newf->bf_size = size; |
1356 | newf->bf_jitcode = jcode; | | 1356 | newf->bf_jitcode = jcode; |
1357 | if (cmd == BIOCSETF) | | 1357 | if (cmd == BIOCSETF) |
1358 | d->bd_jitcode = jcode; /* XXX just for kvm(3) users */ | | 1358 | d->bd_jitcode = jcode; /* XXX just for kvm(3) users */ |
1359 | | | 1359 | |
1360 | /* Need to hold bpf_mtx for pserialize_perform */ | | 1360 | /* Need to hold bpf_mtx for pserialize_perform */ |
1361 | mutex_enter(&bpf_mtx); | | 1361 | mutex_enter(&bpf_mtx); |
1362 | mutex_enter(d->bd_mtx); | | 1362 | mutex_enter(d->bd_mtx); |
1363 | if (cmd == BIOCSETWF) { | | 1363 | if (cmd == BIOCSETWF) { |
1364 | oldf = d->bd_wfilter; | | 1364 | oldf = d->bd_wfilter; |
1365 | storef = &d->bd_wfilter; | | 1365 | storef = &d->bd_wfilter; |
1366 | } else { | | 1366 | } else { |
1367 | oldf = d->bd_rfilter; | | 1367 | oldf = d->bd_rfilter; |
1368 | storef = &d->bd_rfilter; | | 1368 | storef = &d->bd_rfilter; |
1369 | } | | 1369 | } |
1370 | atomic_store_release(storef, newf); | | 1370 | atomic_store_release(storef, newf); |
1371 | reset_d(d); | | 1371 | reset_d(d); |
1372 | pserialize_perform(bpf_psz); | | 1372 | pserialize_perform(bpf_psz); |
1373 | mutex_exit(d->bd_mtx); | | 1373 | mutex_exit(d->bd_mtx); |
1374 | mutex_exit(&bpf_mtx); | | 1374 | mutex_exit(&bpf_mtx); |
1375 | | | 1375 | |
1376 | if (oldf != NULL) | | 1376 | if (oldf != NULL) |
1377 | bpf_free_filter(oldf); | | 1377 | bpf_free_filter(oldf); |
1378 | | | 1378 | |
1379 | return 0; | | 1379 | return 0; |
1380 | } | | 1380 | } |
1381 | | | 1381 | |
1382 | /* | | 1382 | /* |
1383 | * Detach a file from its current interface (if attached at all) and attach | | 1383 | * Detach a file from its current interface (if attached at all) and attach |
1384 | * to the interface indicated by the name stored in ifr. | | 1384 | * to the interface indicated by the name stored in ifr. |
1385 | * Return an errno or 0. | | 1385 | * Return an errno or 0. |
1386 | */ | | 1386 | */ |
1387 | static int | | 1387 | static int |
1388 | bpf_setif(struct bpf_d *d, struct ifreq *ifr) | | 1388 | bpf_setif(struct bpf_d *d, struct ifreq *ifr) |
1389 | { | | 1389 | { |
1390 | struct bpf_if *bp; | | 1390 | struct bpf_if *bp; |
1391 | char *cp; | | 1391 | char *cp; |
1392 | int unit_seen, i, error; | | 1392 | int unit_seen, i, error; |
1393 | | | 1393 | |
1394 | KASSERT(mutex_owned(&bpf_mtx)); | | 1394 | KASSERT(mutex_owned(&bpf_mtx)); |
1395 | /* | | 1395 | /* |
1396 | * Make sure the provided name has a unit number, and default | | 1396 | * Make sure the provided name has a unit number, and default |
1397 | * it to '0' if not specified. | | 1397 | * it to '0' if not specified. |
1398 | * XXX This is ugly ... do this differently? | | 1398 | * XXX This is ugly ... do this differently? |
1399 | */ | | 1399 | */ |
1400 | unit_seen = 0; | | 1400 | unit_seen = 0; |
1401 | cp = ifr->ifr_name; | | 1401 | cp = ifr->ifr_name; |
1402 | cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ | | 1402 | cp[sizeof(ifr->ifr_name) - 1] = '\0'; /* sanity */ |
1403 | while (*cp++) | | 1403 | while (*cp++) |
1404 | if (*cp >= '0' && *cp <= '9') | | 1404 | if (*cp >= '0' && *cp <= '9') |
1405 | unit_seen = 1; | | 1405 | unit_seen = 1; |
1406 | if (!unit_seen) { | | 1406 | if (!unit_seen) { |
1407 | /* Make sure to leave room for the '\0'. */ | | 1407 | /* Make sure to leave room for the '\0'. */ |
1408 | for (i = 0; i < (IFNAMSIZ - 1); ++i) { | | 1408 | for (i = 0; i < (IFNAMSIZ - 1); ++i) { |
1409 | if ((ifr->ifr_name[i] >= 'a' && | | 1409 | if ((ifr->ifr_name[i] >= 'a' && |
1410 | ifr->ifr_name[i] <= 'z') || | | 1410 | ifr->ifr_name[i] <= 'z') || |
1411 | (ifr->ifr_name[i] >= 'A' && | | 1411 | (ifr->ifr_name[i] >= 'A' && |
1412 | ifr->ifr_name[i] <= 'Z')) | | 1412 | ifr->ifr_name[i] <= 'Z')) |
1413 | continue; | | 1413 | continue; |
1414 | ifr->ifr_name[i] = '0'; | | 1414 | ifr->ifr_name[i] = '0'; |
1415 | } | | 1415 | } |
1416 | } | | 1416 | } |
1417 | | | 1417 | |
1418 | /* | | 1418 | /* |
1419 | * Look through attached interfaces for the named one. | | 1419 | * Look through attached interfaces for the named one. |
1420 | */ | | 1420 | */ |
1421 | BPF_IFLIST_WRITER_FOREACH(bp) { | | 1421 | BPF_IFLIST_WRITER_FOREACH(bp) { |
1422 | struct ifnet *ifp = bp->bif_ifp; | | 1422 | struct ifnet *ifp = bp->bif_ifp; |
1423 | | | 1423 | |
1424 | if (ifp == NULL || | | 1424 | if (ifp == NULL || |
1425 | strcmp(ifp->if_xname, ifr->ifr_name) != 0) | | 1425 | strcmp(ifp->if_xname, ifr->ifr_name) != 0) |
1426 | continue; | | 1426 | continue; |
1427 | /* skip additional entry */ | | 1427 | /* skip additional entry */ |
1428 | if (bp->bif_driverp != &ifp->if_bpf) | | 1428 | if (bp->bif_driverp != &ifp->if_bpf) |
1429 | continue; | | 1429 | continue; |
1430 | /* | | 1430 | /* |
1431 | * We found the requested interface. | | 1431 | * We found the requested interface. |
1432 | * Allocate the packet buffers if we need to. | | 1432 | * Allocate the packet buffers if we need to. |
1433 | * If we're already attached to requested interface, | | 1433 | * If we're already attached to requested interface, |
1434 | * just flush the buffer. | | 1434 | * just flush the buffer. |
1435 | */ | | 1435 | */ |
1436 | /* | | 1436 | /* |
1437 | * bpf_allocbufs is called only here. bpf_mtx ensures that | | 1437 | * bpf_allocbufs is called only here. bpf_mtx ensures that |
1438 | * no race condition happen on d->bd_sbuf. | | 1438 | * no race condition happen on d->bd_sbuf. |
1439 | */ | | 1439 | */ |
1440 | if (d->bd_sbuf == NULL) { | | 1440 | if (d->bd_sbuf == NULL) { |
1441 | error = bpf_allocbufs(d); | | 1441 | error = bpf_allocbufs(d); |
1442 | if (error != 0) | | 1442 | if (error != 0) |
1443 | return (error); | | 1443 | return (error); |
1444 | } | | 1444 | } |
1445 | mutex_enter(d->bd_mtx); | | 1445 | mutex_enter(d->bd_mtx); |
1446 | if (bp != d->bd_bif) { | | 1446 | if (bp != d->bd_bif) { |
1447 | if (d->bd_bif) { | | 1447 | if (d->bd_bif) { |
1448 | /* | | 1448 | /* |
1449 | * Detach if attached to something else. | | 1449 | * Detach if attached to something else. |
1450 | */ | | 1450 | */ |
1451 | bpf_detachd(d); | | 1451 | bpf_detachd(d); |
1452 | BPFIF_DLIST_ENTRY_INIT(d); | | 1452 | BPFIF_DLIST_ENTRY_INIT(d); |
1453 | } | | 1453 | } |
1454 | | | 1454 | |
1455 | bpf_attachd(d, bp); | | 1455 | bpf_attachd(d, bp); |
1456 | } | | 1456 | } |
1457 | reset_d(d); | | 1457 | reset_d(d); |
1458 | mutex_exit(d->bd_mtx); | | 1458 | mutex_exit(d->bd_mtx); |
1459 | return (0); | | 1459 | return (0); |
1460 | } | | 1460 | } |
1461 | /* Not found. */ | | 1461 | /* Not found. */ |
1462 | return (ENXIO); | | 1462 | return (ENXIO); |
1463 | } | | 1463 | } |
1464 | | | 1464 | |
1465 | /* | | 1465 | /* |
1466 | * Copy the interface name to the ifreq. | | 1466 | * Copy the interface name to the ifreq. |
1467 | */ | | 1467 | */ |
1468 | static void | | 1468 | static void |
1469 | bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) | | 1469 | bpf_ifname(struct ifnet *ifp, struct ifreq *ifr) |
1470 | { | | 1470 | { |
1471 | memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); | | 1471 | memcpy(ifr->ifr_name, ifp->if_xname, IFNAMSIZ); |
1472 | } | | 1472 | } |
1473 | | | 1473 | |
1474 | static int | | 1474 | static int |
1475 | bpf_stat(struct file *fp, struct stat *st) | | 1475 | bpf_stat(struct file *fp, struct stat *st) |
1476 | { | | 1476 | { |
1477 | struct bpf_d *d = fp->f_bpf; | | 1477 | struct bpf_d *d = fp->f_bpf; |
1478 | | | 1478 | |
1479 | (void)memset(st, 0, sizeof(*st)); | | 1479 | (void)memset(st, 0, sizeof(*st)); |
1480 | mutex_enter(d->bd_mtx); | | 1480 | mutex_enter(d->bd_mtx); |
1481 | st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); | | 1481 | st->st_dev = makedev(cdevsw_lookup_major(&bpf_cdevsw), d->bd_pid); |
1482 | st->st_atimespec = d->bd_atime; | | 1482 | st->st_atimespec = d->bd_atime; |
1483 | st->st_mtimespec = d->bd_mtime; | | 1483 | st->st_mtimespec = d->bd_mtime; |
1484 | st->st_ctimespec = st->st_birthtimespec = d->bd_btime; | | 1484 | st->st_ctimespec = st->st_birthtimespec = d->bd_btime; |
1485 | st->st_uid = kauth_cred_geteuid(fp->f_cred); | | 1485 | st->st_uid = kauth_cred_geteuid(fp->f_cred); |
1486 | st->st_gid = kauth_cred_getegid(fp->f_cred); | | 1486 | st->st_gid = kauth_cred_getegid(fp->f_cred); |
1487 | st->st_mode = S_IFCHR; | | 1487 | st->st_mode = S_IFCHR; |
1488 | mutex_exit(d->bd_mtx); | | 1488 | mutex_exit(d->bd_mtx); |
1489 | return 0; | | 1489 | return 0; |
1490 | } | | 1490 | } |
1491 | | | 1491 | |
1492 | /* | | 1492 | /* |
1493 | * Support for poll() system call | | 1493 | * Support for poll() system call |
1494 | * | | 1494 | * |
1495 | * Return true iff the specific operation will not block indefinitely - with | | 1495 | * Return true iff the specific operation will not block indefinitely - with |
1496 | * the assumption that it is safe to positively acknowledge a request for the | | 1496 | * the assumption that it is safe to positively acknowledge a request for the |
1497 | * ability to write to the BPF device. | | 1497 | * ability to write to the BPF device. |
1498 | * Otherwise, return false but make a note that a selnotify() must be done. | | 1498 | * Otherwise, return false but make a note that a selnotify() must be done. |
1499 | */ | | 1499 | */ |
1500 | static int | | 1500 | static int |
1501 | bpf_poll(struct file *fp, int events) | | 1501 | bpf_poll(struct file *fp, int events) |
1502 | { | | 1502 | { |
1503 | struct bpf_d *d = fp->f_bpf; | | 1503 | struct bpf_d *d = fp->f_bpf; |
1504 | int revents; | | 1504 | int revents; |
1505 | | | 1505 | |
1506 | /* | | 1506 | /* |
1507 | * Refresh the PID associated with this bpf file. | | 1507 | * Refresh the PID associated with this bpf file. |
1508 | */ | | 1508 | */ |
1509 | mutex_enter(&bpf_mtx); | | 1509 | mutex_enter(&bpf_mtx); |
1510 | d->bd_pid = curproc->p_pid; | | 1510 | d->bd_pid = curproc->p_pid; |
1511 | | | 1511 | |
1512 | revents = events & (POLLOUT | POLLWRNORM); | | 1512 | revents = events & (POLLOUT | POLLWRNORM); |
1513 | if (events & (POLLIN | POLLRDNORM)) { | | 1513 | if (events & (POLLIN | POLLRDNORM)) { |
1514 | /* | | 1514 | /* |
1515 | * An imitation of the FIONREAD ioctl code. | | 1515 | * An imitation of the FIONREAD ioctl code. |
1516 | */ | | 1516 | */ |
1517 | mutex_enter(d->bd_mtx); | | 1517 | mutex_enter(d->bd_mtx); |
1518 | if (d->bd_hlen != 0 || | | 1518 | if (d->bd_hlen != 0 || |
1519 | ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && | | 1519 | ((d->bd_immediate || d->bd_state == BPF_TIMED_OUT) && |
1520 | d->bd_slen != 0)) { | | 1520 | d->bd_slen != 0)) { |
1521 | revents |= events & (POLLIN | POLLRDNORM); | | 1521 | revents |= events & (POLLIN | POLLRDNORM); |
1522 | } else { | | 1522 | } else { |
1523 | selrecord(curlwp, &d->bd_sel); | | 1523 | selrecord(curlwp, &d->bd_sel); |
1524 | /* Start the read timeout if necessary */ | | 1524 | /* Start the read timeout if necessary */ |
1525 | if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { | | 1525 | if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) { |
1526 | callout_reset(&d->bd_callout, d->bd_rtout, | | 1526 | callout_reset(&d->bd_callout, d->bd_rtout, |
1527 | bpf_timed_out, d); | | 1527 | bpf_timed_out, d); |
1528 | d->bd_state = BPF_WAITING; | | 1528 | d->bd_state = BPF_WAITING; |
1529 | } | | 1529 | } |
1530 | } | | 1530 | } |
1531 | mutex_exit(d->bd_mtx); | | 1531 | mutex_exit(d->bd_mtx); |
1532 | } | | 1532 | } |
1533 | | | 1533 | |
1534 | mutex_exit(&bpf_mtx); | | 1534 | mutex_exit(&bpf_mtx); |
1535 | return (revents); | | 1535 | return (revents); |
1536 | } | | 1536 | } |
1537 | | | 1537 | |
1538 | static void | | 1538 | static void |
1539 | filt_bpfrdetach(struct knote *kn) | | 1539 | filt_bpfrdetach(struct knote *kn) |
1540 | { | | 1540 | { |
1541 | struct bpf_d *d = kn->kn_hook; | | 1541 | struct bpf_d *d = kn->kn_hook; |
1542 | | | 1542 | |
1543 | mutex_enter(d->bd_buf_mtx); | | 1543 | mutex_enter(d->bd_buf_mtx); |
1544 | selremove_knote(&d->bd_sel, kn); | | 1544 | selremove_knote(&d->bd_sel, kn); |
1545 | mutex_exit(d->bd_buf_mtx); | | 1545 | mutex_exit(d->bd_buf_mtx); |
1546 | } | | 1546 | } |
1547 | | | 1547 | |
1548 | static int | | 1548 | static int |
1549 | filt_bpfread(struct knote *kn, long hint) | | 1549 | filt_bpfread(struct knote *kn, long hint) |
1550 | { | | 1550 | { |
1551 | struct bpf_d *d = kn->kn_hook; | | 1551 | struct bpf_d *d = kn->kn_hook; |
1552 | int rv; | | 1552 | int rv; |
1553 | | | 1553 | |
1554 | mutex_enter(d->bd_buf_mtx); | | 1554 | mutex_enter(d->bd_buf_mtx); |
1555 | kn->kn_data = d->bd_hlen; | | 1555 | kn->kn_data = d->bd_hlen; |
1556 | if (d->bd_immediate) | | 1556 | if (d->bd_immediate) |
1557 | kn->kn_data += d->bd_slen; | | 1557 | kn->kn_data += d->bd_slen; |
1558 | rv = (kn->kn_data > 0); | | 1558 | rv = (kn->kn_data > 0); |
1559 | mutex_exit(d->bd_buf_mtx); | | 1559 | mutex_exit(d->bd_buf_mtx); |
1560 | return rv; | | 1560 | return rv; |
1561 | } | | 1561 | } |
1562 | | | 1562 | |
1563 | static const struct filterops bpfread_filtops = { | | 1563 | static const struct filterops bpfread_filtops = { |
1564 | .f_isfd = 1, | | 1564 | .f_isfd = 1, |
1565 | .f_attach = NULL, | | 1565 | .f_attach = NULL, |
1566 | .f_detach = filt_bpfrdetach, | | 1566 | .f_detach = filt_bpfrdetach, |
1567 | .f_event = filt_bpfread, | | 1567 | .f_event = filt_bpfread, |
1568 | }; | | 1568 | }; |
1569 | | | 1569 | |
1570 | static int | | 1570 | static int |
1571 | bpf_kqfilter(struct file *fp, struct knote *kn) | | 1571 | bpf_kqfilter(struct file *fp, struct knote *kn) |
1572 | { | | 1572 | { |
1573 | struct bpf_d *d = fp->f_bpf; | | 1573 | struct bpf_d *d = fp->f_bpf; |
1574 | | | 1574 | |
1575 | switch (kn->kn_filter) { | | 1575 | switch (kn->kn_filter) { |
1576 | case EVFILT_READ: | | 1576 | case EVFILT_READ: |
1577 | kn->kn_fop = &bpfread_filtops; | | 1577 | kn->kn_fop = &bpfread_filtops; |
1578 | break; | | 1578 | break; |
1579 | | | 1579 | |
1580 | default: | | 1580 | default: |
1581 | return (EINVAL); | | 1581 | return (EINVAL); |
1582 | } | | 1582 | } |
1583 | | | 1583 | |
1584 | kn->kn_hook = d; | | 1584 | kn->kn_hook = d; |
1585 | | | 1585 | |
1586 | mutex_enter(d->bd_buf_mtx); | | 1586 | mutex_enter(d->bd_buf_mtx); |
1587 | selrecord_knote(&d->bd_sel, kn); | | 1587 | selrecord_knote(&d->bd_sel, kn); |
1588 | mutex_exit(d->bd_buf_mtx); | | 1588 | mutex_exit(d->bd_buf_mtx); |
1589 | | | 1589 | |
1590 | return (0); | | 1590 | return (0); |
1591 | } | | 1591 | } |
1592 | | | 1592 | |
1593 | /* | | 1593 | /* |
1594 | * Copy data from an mbuf chain into a buffer. This code is derived | | 1594 | * Copy data from an mbuf chain into a buffer. This code is derived |
1595 | * from m_copydata in sys/uipc_mbuf.c. | | 1595 | * from m_copydata in sys/uipc_mbuf.c. |
1596 | */ | | 1596 | */ |
1597 | static void * | | 1597 | static void * |
1598 | bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) | | 1598 | bpf_mcpy(void *dst_arg, const void *src_arg, size_t len) |
1599 | { | | 1599 | { |
1600 | const struct mbuf *m; | | 1600 | const struct mbuf *m; |
1601 | u_int count; | | 1601 | u_int count; |
1602 | u_char *dst; | | 1602 | u_char *dst; |
1603 | | | 1603 | |
1604 | m = src_arg; | | 1604 | m = src_arg; |
1605 | dst = dst_arg; | | 1605 | dst = dst_arg; |
1606 | while (len > 0) { | | 1606 | while (len > 0) { |
1607 | if (m == NULL) | | 1607 | if (m == NULL) |
1608 | panic("bpf_mcpy"); | | 1608 | panic("bpf_mcpy"); |
1609 | count = uimin(m->m_len, len); | | 1609 | count = uimin(m->m_len, len); |
1610 | memcpy(dst, mtod(m, const void *), count); | | 1610 | memcpy(dst, mtod(m, const void *), count); |
1611 | m = m->m_next; | | 1611 | m = m->m_next; |
1612 | dst += count; | | 1612 | dst += count; |
1613 | len -= count; | | 1613 | len -= count; |
1614 | } | | 1614 | } |
1615 | return dst_arg; | | 1615 | return dst_arg; |
1616 | } | | 1616 | } |
1617 | | | 1617 | |
1618 | static inline u_int | | 1618 | static inline u_int |
1619 | bpf_xfilter(struct bpf_filter **filter, void *pkt, u_int pktlen, u_int buflen) | | 1619 | bpf_xfilter(struct bpf_filter **filter, void *pkt, u_int pktlen, u_int buflen) |
1620 | { | | 1620 | { |
1621 | struct bpf_filter *filt; | | 1621 | struct bpf_filter *filt; |
1622 | uint32_t mem[BPF_MEMWORDS]; | | 1622 | uint32_t mem[BPF_MEMWORDS]; |
1623 | bpf_args_t args = { | | 1623 | bpf_args_t args = { |
1624 | .pkt = (const uint8_t *)pkt, | | 1624 | .pkt = (const uint8_t *)pkt, |
1625 | .wirelen = pktlen, | | 1625 | .wirelen = pktlen, |
1626 | .buflen = buflen, | | 1626 | .buflen = buflen, |
1627 | .mem = mem, | | 1627 | .mem = mem, |
1628 | .arg = NULL | | 1628 | .arg = NULL |
1629 | }; | | 1629 | }; |
1630 | u_int slen; | | 1630 | u_int slen; |
1631 | | | 1631 | |
1632 | filt = atomic_load_consume(filter); | | 1632 | filt = atomic_load_consume(filter); |
1633 | if (filt == NULL) /* No filter means accept all. */ | | 1633 | if (filt == NULL) /* No filter means accept all. */ |
1634 | return (u_int)-1; | | 1634 | return (u_int)-1; |
1635 | | | 1635 | |
1636 | if (filt->bf_jitcode != NULL) | | 1636 | if (filt->bf_jitcode != NULL) |
1637 | slen = filt->bf_jitcode(NULL, &args); | | 1637 | slen = filt->bf_jitcode(NULL, &args); |
1638 | else | | 1638 | else |
1639 | slen = bpf_filter_ext(NULL, filt->bf_insn, &args); | | 1639 | slen = bpf_filter_ext(NULL, filt->bf_insn, &args); |
1640 | return slen; | | 1640 | return slen; |
1641 | } | | 1641 | } |
1642 | | | 1642 | |
1643 | /* | | 1643 | /* |
1644 | * Dispatch a packet to all the listeners on interface bp. | | 1644 | * Dispatch a packet to all the listeners on interface bp. |
1645 | * | | 1645 | * |
1646 | * pkt pointer to the packet, either a data buffer or an mbuf chain | | 1646 | * pkt pointer to the packet, either a data buffer or an mbuf chain |
1647 | * buflen buffer length, if pkt is a data buffer | | 1647 | * buflen buffer length, if pkt is a data buffer |
1648 | * cpfn a function that can copy pkt into the listener's buffer | | 1648 | * cpfn a function that can copy pkt into the listener's buffer |
1649 | * pktlen length of the packet | | 1649 | * pktlen length of the packet |
1650 | * direction BPF_D_IN or BPF_D_OUT | | 1650 | * direction BPF_D_IN or BPF_D_OUT |
1651 | */ | | 1651 | */ |
1652 | static inline void | | 1652 | static inline void |
1653 | bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), | | 1653 | bpf_deliver(struct bpf_if *bp, void *(*cpfn)(void *, const void *, size_t), |
1654 | void *pkt, u_int pktlen, u_int buflen, const u_int direction) | | 1654 | void *pkt, u_int pktlen, u_int buflen, const u_int direction) |
1655 | { | | 1655 | { |
1656 | bool gottime = false; | | 1656 | bool gottime = false; |
1657 | struct timespec ts; | | 1657 | struct timespec ts; |
1658 | struct bpf_d *d; | | 1658 | struct bpf_d *d; |
1659 | int s; | | 1659 | int s; |
1660 | u_int slen; | | 1660 | u_int slen; |
1661 | | | 1661 | |
1662 | KASSERT(!cpu_intr_p()); | | 1662 | KASSERT(!cpu_intr_p()); |
1663 | | | 1663 | |
1664 | /* | | 1664 | /* |
1665 | * Note that the IPL does not have to be raised at this point. | | 1665 | * Note that the IPL does not have to be raised at this point. |
1666 | * The only problem that could arise here is that if two different | | 1666 | * The only problem that could arise here is that if two different |
1667 | * interfaces shared any data. This is not the case. | | 1667 | * interfaces shared any data. This is not the case. |
1668 | */ | | 1668 | */ |
1669 | s = pserialize_read_enter(); | | 1669 | s = pserialize_read_enter(); |
1670 | BPFIF_DLIST_READER_FOREACH(d, bp) { | | 1670 | BPFIF_DLIST_READER_FOREACH(d, bp) { |
1671 | if (direction == BPF_D_IN) { | | 1671 | if (direction == BPF_D_IN) { |
1672 | if (d->bd_direction == BPF_D_OUT) | | 1672 | if (d->bd_direction == BPF_D_OUT) |
1673 | continue; | | 1673 | continue; |
1674 | } else { /* BPF_D_OUT */ | | 1674 | } else { /* BPF_D_OUT */ |
1675 | if (d->bd_direction == BPF_D_IN) | | 1675 | if (d->bd_direction == BPF_D_IN) |
1676 | continue; | | 1676 | continue; |
1677 | } | | 1677 | } |
1678 | | | 1678 | |
1679 | atomic_inc_ulong(&d->bd_rcount); | | 1679 | atomic_inc_ulong(&d->bd_rcount); |
1680 | BPF_STATINC(recv); | | 1680 | BPF_STATINC(recv); |
1681 | | | 1681 | |
1682 | slen = bpf_xfilter(&d->bd_rfilter, pkt, pktlen, buflen); | | 1682 | slen = bpf_xfilter(&d->bd_rfilter, pkt, pktlen, buflen); |
1683 | if (slen == 0) | | 1683 | if (slen == 0) |
1684 | continue; | | 1684 | continue; |
1685 | | | 1685 | |
1686 | if (!gottime) { | | 1686 | if (!gottime) { |
1687 | gottime = true; | | 1687 | gottime = true; |
1688 | nanotime(&ts); | | 1688 | nanotime(&ts); |
1689 | } | | 1689 | } |
1690 | /* Assume catchpacket doesn't sleep */ | | 1690 | /* Assume catchpacket doesn't sleep */ |
1691 | catchpacket(d, pkt, pktlen, slen, cpfn, &ts); | | 1691 | catchpacket(d, pkt, pktlen, slen, cpfn, &ts); |
1692 | } | | 1692 | } |
1693 | pserialize_read_exit(s); | | 1693 | pserialize_read_exit(s); |
1694 | } | | 1694 | } |
1695 | | | 1695 | |
1696 | /* | | 1696 | /* |
1697 | * Incoming linkage from device drivers, when the head of the packet is in | | 1697 | * Incoming linkage from device drivers, when the head of the packet is in |
1698 | * a buffer, and the tail is in an mbuf chain. | | 1698 | * a buffer, and the tail is in an mbuf chain. |
1699 | */ | | 1699 | */ |
1700 | static void | | 1700 | static void |
1701 | _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m, | | 1701 | _bpf_mtap2(struct bpf_if *bp, void *data, u_int dlen, struct mbuf *m, |
1702 | u_int direction) | | 1702 | u_int direction) |
1703 | { | | 1703 | { |
1704 | u_int pktlen; | | 1704 | u_int pktlen; |
1705 | struct mbuf mb; | | 1705 | struct mbuf mb; |
1706 | | | 1706 | |
1707 | /* Skip outgoing duplicate packets. */ | | 1707 | /* Skip outgoing duplicate packets. */ |
1708 | if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { | | 1708 | if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { |
1709 | m->m_flags &= ~M_PROMISC; | | 1709 | m->m_flags &= ~M_PROMISC; |
1710 | return; | | 1710 | return; |
1711 | } | | 1711 | } |
1712 | | | 1712 | |
1713 | pktlen = m_length(m) + dlen; | | 1713 | pktlen = m_length(m) + dlen; |
1714 | | | 1714 | |
1715 | /* | | 1715 | /* |
1716 | * Craft on-stack mbuf suitable for passing to bpf_filter. | | 1716 | * Craft on-stack mbuf suitable for passing to bpf_filter. |
1717 | * Note that we cut corners here; we only setup what's | | 1717 | * Note that we cut corners here; we only setup what's |
1718 | * absolutely needed--this mbuf should never go anywhere else. | | 1718 | * absolutely needed--this mbuf should never go anywhere else. |
1719 | */ | | 1719 | */ |
1720 | (void)memset(&mb, 0, sizeof(mb)); | | 1720 | (void)memset(&mb, 0, sizeof(mb)); |
1721 | mb.m_type = MT_DATA; | | 1721 | mb.m_type = MT_DATA; |
1722 | mb.m_next = m; | | 1722 | mb.m_next = m; |
1723 | mb.m_data = data; | | 1723 | mb.m_data = data; |
1724 | mb.m_len = dlen; | | 1724 | mb.m_len = dlen; |
1725 | | | 1725 | |
1726 | bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, direction); | | 1726 | bpf_deliver(bp, bpf_mcpy, &mb, pktlen, 0, direction); |
1727 | } | | 1727 | } |
1728 | | | 1728 | |
1729 | /* | | 1729 | /* |
1730 | * Incoming linkage from device drivers, when packet is in an mbuf chain. | | 1730 | * Incoming linkage from device drivers, when packet is in an mbuf chain. |
1731 | */ | | 1731 | */ |
1732 | static void | | 1732 | static void |
1733 | _bpf_mtap(struct bpf_if *bp, struct mbuf *m, u_int direction) | | 1733 | _bpf_mtap(struct bpf_if *bp, struct mbuf *m, u_int direction) |
1734 | { | | 1734 | { |
1735 | void *(*cpfn)(void *, const void *, size_t); | | 1735 | void *(*cpfn)(void *, const void *, size_t); |
1736 | u_int pktlen, buflen; | | 1736 | u_int pktlen, buflen; |
1737 | void *marg; | | 1737 | void *marg; |
1738 | | | 1738 | |
1739 | /* Skip outgoing duplicate packets. */ | | 1739 | /* Skip outgoing duplicate packets. */ |
1740 | if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { | | 1740 | if ((m->m_flags & M_PROMISC) != 0 && m->m_pkthdr.rcvif_index == 0) { |
1741 | m->m_flags &= ~M_PROMISC; | | 1741 | m->m_flags &= ~M_PROMISC; |
1742 | return; | | 1742 | return; |
1743 | } | | 1743 | } |
1744 | | | 1744 | |
1745 | pktlen = m_length(m); | | 1745 | pktlen = m_length(m); |
1746 | | | 1746 | |
1747 | /* Skip zero-sized packets. */ | | 1747 | /* Skip zero-sized packets. */ |
1748 | if (__predict_false(pktlen == 0)) { | | 1748 | if (__predict_false(pktlen == 0)) { |
1749 | return; | | 1749 | return; |
1750 | } | | 1750 | } |
1751 | | | 1751 | |
1752 | if (pktlen == m->m_len) { | | 1752 | if (pktlen == m->m_len) { |
1753 | cpfn = (void *)memcpy; | | 1753 | cpfn = (void *)memcpy; |
1754 | marg = mtod(m, void *); | | 1754 | marg = mtod(m, void *); |
1755 | buflen = pktlen; | | 1755 | buflen = pktlen; |
1756 | KASSERT(buflen != 0); | | 1756 | KASSERT(buflen != 0); |
1757 | } else { | | 1757 | } else { |
1758 | cpfn = bpf_mcpy; | | 1758 | cpfn = bpf_mcpy; |
1759 | marg = m; | | 1759 | marg = m; |
1760 | buflen = 0; | | 1760 | buflen = 0; |
1761 | } | | 1761 | } |
1762 | | | 1762 | |
1763 | bpf_deliver(bp, cpfn, marg, pktlen, buflen, direction); | | 1763 | bpf_deliver(bp, cpfn, marg, pktlen, buflen, direction); |
1764 | } | | 1764 | } |
1765 | | | 1765 | |
1766 | /* | | 1766 | /* |
1767 | * We need to prepend the address family as | | 1767 | * We need to prepend the address family as |
1768 | * a four byte field. Cons up a dummy header | | 1768 | * a four byte field. Cons up a dummy header |
1769 | * to pacify bpf. This is safe because bpf | | 1769 | * to pacify bpf. This is safe because bpf |
1770 | * will only read from the mbuf (i.e., it won't | | 1770 | * will only read from the mbuf (i.e., it won't |
1771 | * try to free it or keep a pointer a to it). | | 1771 | * try to free it or keep a pointer a to it). |
1772 | */ | | 1772 | */ |
1773 | static void | | 1773 | static void |
1774 | _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m, u_int direction) | | 1774 | _bpf_mtap_af(struct bpf_if *bp, uint32_t af, struct mbuf *m, u_int direction) |
1775 | { | | 1775 | { |
1776 | struct mbuf m0; | | 1776 | struct mbuf m0; |
1777 | | | 1777 | |
1778 | m0.m_type = MT_DATA; | | 1778 | m0.m_type = MT_DATA; |
1779 | m0.m_flags = 0; | | 1779 | m0.m_flags = 0; |
1780 | m0.m_next = m; | | 1780 | m0.m_next = m; |
1781 | m0.m_nextpkt = NULL; | | 1781 | m0.m_nextpkt = NULL; |
1782 | m0.m_owner = NULL; | | 1782 | m0.m_owner = NULL; |
1783 | m0.m_len = 4; | | 1783 | m0.m_len = 4; |
1784 | m0.m_data = (char *)⁡ | | 1784 | m0.m_data = (char *)⁡ |
1785 | | | 1785 | |
1786 | _bpf_mtap(bp, &m0, direction); | | 1786 | _bpf_mtap(bp, &m0, direction); |
1787 | } | | 1787 | } |
1788 | | | 1788 | |
1789 | /* | | 1789 | /* |
1790 | * Put the SLIP pseudo-"link header" in place. | | 1790 | * Put the SLIP pseudo-"link header" in place. |
1791 | * Note this M_PREPEND() should never fail, | | 1791 | * Note this M_PREPEND() should never fail, |
1792 | * swince we know we always have enough space | | 1792 | * swince we know we always have enough space |
1793 | * in the input buffer. | | 1793 | * in the input buffer. |
1794 | */ | | 1794 | */ |
1795 | static void | | 1795 | static void |
1796 | _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) | | 1796 | _bpf_mtap_sl_in(struct bpf_if *bp, u_char *chdr, struct mbuf **m) |
1797 | { | | 1797 | { |
1798 | u_char *hp; | | 1798 | u_char *hp; |
1799 | | | 1799 | |
1800 | M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); | | 1800 | M_PREPEND(*m, SLIP_HDRLEN, M_DONTWAIT); |
1801 | if (*m == NULL) | | 1801 | if (*m == NULL) |
1802 | return; | | 1802 | return; |
1803 | | | 1803 | |
1804 | hp = mtod(*m, u_char *); | | 1804 | hp = mtod(*m, u_char *); |
1805 | hp[SLX_DIR] = SLIPDIR_IN; | | 1805 | hp[SLX_DIR] = SLIPDIR_IN; |
1806 | (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); | | 1806 | (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); |
1807 | | | 1807 | |
1808 | _bpf_mtap(bp, *m, BPF_D_IN); | | 1808 | _bpf_mtap(bp, *m, BPF_D_IN); |
1809 | | | 1809 | |
1810 | m_adj(*m, SLIP_HDRLEN); | | 1810 | m_adj(*m, SLIP_HDRLEN); |
1811 | } | | 1811 | } |
1812 | | | 1812 | |
1813 | /* | | 1813 | /* |
1814 | * Put the SLIP pseudo-"link header" in | | 1814 | * Put the SLIP pseudo-"link header" in |
1815 | * place. The compressed header is now | | 1815 | * place. The compressed header is now |
1816 | * at the beginning of the mbuf. | | 1816 | * at the beginning of the mbuf. |
1817 | */ | | 1817 | */ |
1818 | static void | | 1818 | static void |
1819 | _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) | | 1819 | _bpf_mtap_sl_out(struct bpf_if *bp, u_char *chdr, struct mbuf *m) |
1820 | { | | 1820 | { |
1821 | struct mbuf m0; | | 1821 | struct mbuf m0; |
1822 | u_char *hp; | | 1822 | u_char *hp; |
1823 | | | 1823 | |
1824 | m0.m_type = MT_DATA; | | 1824 | m0.m_type = MT_DATA; |
1825 | m0.m_flags = 0; | | 1825 | m0.m_flags = 0; |
1826 | m0.m_next = m; | | 1826 | m0.m_next = m; |
1827 | m0.m_nextpkt = NULL; | | 1827 | m0.m_nextpkt = NULL; |
1828 | m0.m_owner = NULL; | | 1828 | m0.m_owner = NULL; |
1829 | m0.m_data = m0.m_dat; | | 1829 | m0.m_data = m0.m_dat; |
1830 | m0.m_len = SLIP_HDRLEN; | | 1830 | m0.m_len = SLIP_HDRLEN; |
1831 | | | 1831 | |
1832 | hp = mtod(&m0, u_char *); | | 1832 | hp = mtod(&m0, u_char *); |
1833 | | | 1833 | |
1834 | hp[SLX_DIR] = SLIPDIR_OUT; | | 1834 | hp[SLX_DIR] = SLIPDIR_OUT; |
1835 | (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); | | 1835 | (void)memcpy(&hp[SLX_CHDR], chdr, CHDR_LEN); |
1836 | | | 1836 | |
1837 | _bpf_mtap(bp, &m0, BPF_D_OUT); | | 1837 | _bpf_mtap(bp, &m0, BPF_D_OUT); |
1838 | m_freem(m); | | 1838 | m_freem(m); |
1839 | } | | 1839 | } |
1840 | | | 1840 | |
1841 | static struct mbuf * | | 1841 | static struct mbuf * |
1842 | bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m) | | 1842 | bpf_mbuf_enqueue(struct bpf_if *bp, struct mbuf *m) |
1843 | { | | 1843 | { |
1844 | struct mbuf *dup; | | 1844 | struct mbuf *dup; |
1845 | | | 1845 | |
1846 | dup = m_dup(m, 0, M_COPYALL, M_NOWAIT); | | 1846 | dup = m_dup(m, 0, M_COPYALL, M_NOWAIT); |
1847 | if (dup == NULL) | | 1847 | if (dup == NULL) |
1848 | return NULL; | | 1848 | return NULL; |
1849 | | | 1849 | |
1850 | if (bp->bif_mbuf_tail != NULL) { | | 1850 | if (bp->bif_mbuf_tail != NULL) { |
1851 | bp->bif_mbuf_tail->m_nextpkt = dup; | | 1851 | bp->bif_mbuf_tail->m_nextpkt = dup; |
1852 | } else { | | 1852 | } else { |
1853 | bp->bif_mbuf_head = dup; | | 1853 | bp->bif_mbuf_head = dup; |
1854 | } | | 1854 | } |
1855 | bp->bif_mbuf_tail = dup; | | 1855 | bp->bif_mbuf_tail = dup; |
1856 | #ifdef BPF_MTAP_SOFTINT_DEBUG | | 1856 | #ifdef BPF_MTAP_SOFTINT_DEBUG |
1857 | log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n", | | 1857 | log(LOG_DEBUG, "%s: enqueued mbuf=%p to %s\n", |
1858 | __func__, dup, bp->bif_ifp->if_xname); | | 1858 | __func__, dup, bp->bif_ifp->if_xname); |
1859 | #endif | | 1859 | #endif |
1860 | | | 1860 | |
1861 | return dup; | | 1861 | return dup; |
1862 | } | | 1862 | } |
1863 | | | 1863 | |
1864 | static struct mbuf * | | 1864 | static struct mbuf * |
1865 | bpf_mbuf_dequeue(struct bpf_if *bp) | | 1865 | bpf_mbuf_dequeue(struct bpf_if *bp) |
1866 | { | | 1866 | { |
1867 | struct mbuf *m; | | 1867 | struct mbuf *m; |
1868 | int s; | | 1868 | int s; |
1869 | | | 1869 | |
1870 | /* XXX NOMPSAFE: assumed running on one CPU */ | | 1870 | /* XXX NOMPSAFE: assumed running on one CPU */ |
1871 | s = splnet(); | | 1871 | s = splnet(); |
1872 | m = bp->bif_mbuf_head; | | 1872 | m = bp->bif_mbuf_head; |
1873 | if (m != NULL) { | | 1873 | if (m != NULL) { |
1874 | bp->bif_mbuf_head = m->m_nextpkt; | | 1874 | bp->bif_mbuf_head = m->m_nextpkt; |
1875 | m->m_nextpkt = NULL; | | 1875 | m->m_nextpkt = NULL; |
1876 | | | 1876 | |
1877 | if (bp->bif_mbuf_head == NULL) | | 1877 | if (bp->bif_mbuf_head == NULL) |
1878 | bp->bif_mbuf_tail = NULL; | | 1878 | bp->bif_mbuf_tail = NULL; |
1879 | #ifdef BPF_MTAP_SOFTINT_DEBUG | | 1879 | #ifdef BPF_MTAP_SOFTINT_DEBUG |
1880 | log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n", | | 1880 | log(LOG_DEBUG, "%s: dequeued mbuf=%p from %s\n", |
1881 | __func__, m, bp->bif_ifp->if_xname); | | 1881 | __func__, m, bp->bif_ifp->if_xname); |
1882 | #endif | | 1882 | #endif |
1883 | } | | 1883 | } |
1884 | splx(s); | | 1884 | splx(s); |
1885 | | | 1885 | |
1886 | return m; | | 1886 | return m; |
1887 | } | | 1887 | } |
1888 | | | 1888 | |
1889 | static void | | 1889 | static void |
1890 | bpf_mtap_si(void *arg) | | 1890 | bpf_mtap_si(void *arg) |
1891 | { | | 1891 | { |
1892 | struct bpf_if *bp = arg; | | 1892 | struct bpf_if *bp = arg; |
1893 | struct mbuf *m; | | 1893 | struct mbuf *m; |
1894 | | | 1894 | |
1895 | while ((m = bpf_mbuf_dequeue(bp)) != NULL) { | | 1895 | while ((m = bpf_mbuf_dequeue(bp)) != NULL) { |
1896 | #ifdef BPF_MTAP_SOFTINT_DEBUG | | 1896 | #ifdef BPF_MTAP_SOFTINT_DEBUG |
1897 | log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n", | | 1897 | log(LOG_DEBUG, "%s: tapping mbuf=%p on %s\n", |
1898 | __func__, m, bp->bif_ifp->if_xname); | | 1898 | __func__, m, bp->bif_ifp->if_xname); |
1899 | #endif | | 1899 | #endif |
1900 | bpf_ops->bpf_mtap(bp, m, BPF_D_IN); | | 1900 | bpf_ops->bpf_mtap(bp, m, BPF_D_IN); |
1901 | m_freem(m); | | 1901 | m_freem(m); |
1902 | } | | 1902 | } |
1903 | } | | 1903 | } |
1904 | | | 1904 | |
1905 | static void | | 1905 | static void |
1906 | _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m) | | 1906 | _bpf_mtap_softint(struct ifnet *ifp, struct mbuf *m) |
1907 | { | | 1907 | { |
1908 | struct bpf_if *bp = ifp->if_bpf; | | 1908 | struct bpf_if *bp = ifp->if_bpf; |
1909 | struct mbuf *dup; | | 1909 | struct mbuf *dup; |
1910 | | | 1910 | |
1911 | KASSERT(cpu_intr_p()); | | 1911 | KASSERT(cpu_intr_p()); |
1912 | | | 1912 | |
1913 | /* To avoid extra invocations of the softint */ | | 1913 | /* To avoid extra invocations of the softint */ |
1914 | if (BPFIF_DLIST_READER_EMPTY(bp)) | | 1914 | if (BPFIF_DLIST_READER_EMPTY(bp)) |
1915 | return; | | 1915 | return; |
1916 | KASSERT(bp->bif_si != NULL); | | 1916 | KASSERT(bp->bif_si != NULL); |
1917 | | | 1917 | |
1918 | dup = bpf_mbuf_enqueue(bp, m); | | 1918 | dup = bpf_mbuf_enqueue(bp, m); |
1919 | if (dup != NULL) | | 1919 | if (dup != NULL) |
1920 | softint_schedule(bp->bif_si); | | 1920 | softint_schedule(bp->bif_si); |
1921 | } | | 1921 | } |
1922 | | | 1922 | |
1923 | static int | | 1923 | static int |
1924 | bpf_hdrlen(struct bpf_d *d) | | 1924 | bpf_hdrlen(struct bpf_d *d) |
1925 | { | | 1925 | { |
1926 | int hdrlen = d->bd_bif->bif_hdrlen; | | 1926 | int hdrlen = d->bd_bif->bif_hdrlen; |
1927 | /* | | 1927 | /* |
1928 | * Compute the length of the bpf header. This is not necessarily | | 1928 | * Compute the length of the bpf header. This is not necessarily |
1929 | * equal to SIZEOF_BPF_HDR because we want to insert spacing such | | 1929 | * equal to SIZEOF_BPF_HDR because we want to insert spacing such |
1930 | * that the network layer header begins on a longword boundary (for | | 1930 | * that the network layer header begins on a longword boundary (for |
1931 | * performance reasons and to alleviate alignment restrictions). | | 1931 | * performance reasons and to alleviate alignment restrictions). |
1932 | */ | | 1932 | */ |
1933 | #ifdef _LP64 | | 1933 | #ifdef _LP64 |
1934 | if (d->bd_compat32) | | 1934 | if (d->bd_compat32) |
1935 | return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); | | 1935 | return (BPF_WORDALIGN32(hdrlen + SIZEOF_BPF_HDR32) - hdrlen); |
1936 | else | | 1936 | else |
1937 | #endif | | 1937 | #endif |
1938 | return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); | | 1938 | return (BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen); |
1939 | } | | 1939 | } |
1940 | | | 1940 | |
1941 | /* | | 1941 | /* |
1942 | * Move the packet data from interface memory (pkt) into the | | 1942 | * Move the packet data from interface memory (pkt) into the |
1943 | * store buffer. Call the wakeup functions if it's time to wakeup | | 1943 | * store buffer. Call the wakeup functions if it's time to wakeup |
1944 | * a listener (buffer full), "cpfn" is the routine called to do the | | 1944 | * a listener (buffer full), "cpfn" is the routine called to do the |
1945 | * actual data transfer. memcpy is passed in to copy contiguous chunks, | | 1945 | * actual data transfer. memcpy is passed in to copy contiguous chunks, |
1946 | * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, | | 1946 | * while bpf_mcpy is passed in to copy mbuf chains. In the latter case, |
1947 | * pkt is really an mbuf. | | 1947 | * pkt is really an mbuf. |
1948 | */ | | 1948 | */ |
1949 | static void | | 1949 | static void |
1950 | catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, | | 1950 | catchpacket(struct bpf_d *d, u_char *pkt, u_int pktlen, u_int snaplen, |
1951 | void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) | | 1951 | void *(*cpfn)(void *, const void *, size_t), struct timespec *ts) |
1952 | { | | 1952 | { |
1953 | char *h; | | 1953 | char *h; |
1954 | int totlen, curlen, caplen; | | 1954 | int totlen, curlen, caplen; |
1955 | int hdrlen = bpf_hdrlen(d); | | 1955 | int hdrlen = bpf_hdrlen(d); |
1956 | int do_wakeup = 0; | | 1956 | int do_wakeup = 0; |
1957 | | | 1957 | |
1958 | atomic_inc_ulong(&d->bd_ccount); | | 1958 | atomic_inc_ulong(&d->bd_ccount); |
1959 | BPF_STATINC(capt); | | 1959 | BPF_STATINC(capt); |
1960 | /* | | 1960 | /* |
1961 | * Figure out how many bytes to move. If the packet is | | 1961 | * Figure out how many bytes to move. If the packet is |
1962 | * greater or equal to the snapshot length, transfer that | | 1962 | * greater or equal to the snapshot length, transfer that |
1963 | * much. Otherwise, transfer the whole packet (unless | | 1963 | * much. Otherwise, transfer the whole packet (unless |
1964 | * we hit the buffer size limit). | | 1964 | * we hit the buffer size limit). |
1965 | */ | | 1965 | */ |
1966 | totlen = hdrlen + uimin(snaplen, pktlen); | | 1966 | totlen = hdrlen + uimin(snaplen, pktlen); |
1967 | if (totlen > d->bd_bufsize) | | 1967 | if (totlen > d->bd_bufsize) |
1968 | totlen = d->bd_bufsize; | | 1968 | totlen = d->bd_bufsize; |
1969 | /* | | 1969 | /* |
1970 | * If we adjusted totlen to fit the bufsize, it could be that | | 1970 | * If we adjusted totlen to fit the bufsize, it could be that |
1971 | * totlen is smaller than hdrlen because of the link layer header. | | 1971 | * totlen is smaller than hdrlen because of the link layer header. |
1972 | */ | | 1972 | */ |
1973 | caplen = totlen - hdrlen; | | 1973 | caplen = totlen - hdrlen; |
1974 | if (caplen < 0) | | 1974 | if (caplen < 0) |
1975 | caplen = 0; | | 1975 | caplen = 0; |
1976 | | | 1976 | |
1977 | mutex_enter(d->bd_buf_mtx); | | 1977 | mutex_enter(d->bd_buf_mtx); |
1978 | /* | | 1978 | /* |
1979 | * Round up the end of the previous packet to the next longword. | | 1979 | * Round up the end of the previous packet to the next longword. |
1980 | */ | | 1980 | */ |
1981 | #ifdef _LP64 | | 1981 | #ifdef _LP64 |
1982 | if (d->bd_compat32) | | 1982 | if (d->bd_compat32) |
1983 | curlen = BPF_WORDALIGN32(d->bd_slen); | | 1983 | curlen = BPF_WORDALIGN32(d->bd_slen); |
1984 | else | | 1984 | else |
1985 | #endif | | 1985 | #endif |
1986 | curlen = BPF_WORDALIGN(d->bd_slen); | | 1986 | curlen = BPF_WORDALIGN(d->bd_slen); |
1987 | if (curlen + totlen > d->bd_bufsize) { | | 1987 | if (curlen + totlen > d->bd_bufsize) { |
1988 | /* | | 1988 | /* |
1989 | * This packet will overflow the storage buffer. | | 1989 | * This packet will overflow the storage buffer. |
1990 | * Rotate the buffers if we can, then wakeup any | | 1990 | * Rotate the buffers if we can, then wakeup any |
1991 | * pending reads. | | 1991 | * pending reads. |
1992 | */ | | 1992 | */ |
1993 | if (d->bd_fbuf == NULL) { | | 1993 | if (d->bd_fbuf == NULL) { |
1994 | mutex_exit(d->bd_buf_mtx); | | 1994 | mutex_exit(d->bd_buf_mtx); |
1995 | /* | | 1995 | /* |
1996 | * We haven't completed the previous read yet, | | 1996 | * We haven't completed the previous read yet, |
1997 | * so drop the packet. | | 1997 | * so drop the packet. |
1998 | */ | | 1998 | */ |
1999 | atomic_inc_ulong(&d->bd_dcount); | | 1999 | atomic_inc_ulong(&d->bd_dcount); |
2000 | BPF_STATINC(drop); | | 2000 | BPF_STATINC(drop); |
2001 | return; | | 2001 | return; |
2002 | } | | 2002 | } |
2003 | ROTATE_BUFFERS(d); | | 2003 | ROTATE_BUFFERS(d); |
2004 | do_wakeup = 1; | | 2004 | do_wakeup = 1; |
2005 | curlen = 0; | | 2005 | curlen = 0; |
2006 | } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { | | 2006 | } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) { |
2007 | /* | | 2007 | /* |
2008 | * Immediate mode is set, or the read timeout has | | 2008 | * Immediate mode is set, or the read timeout has |
2009 | * already expired during a select call. A packet | | 2009 | * already expired during a select call. A packet |
2010 | * arrived, so the reader should be woken up. | | 2010 | * arrived, so the reader should be woken up. |
2011 | */ | | 2011 | */ |
2012 | do_wakeup = 1; | | 2012 | do_wakeup = 1; |
2013 | } | | 2013 | } |
2014 | | | 2014 | |
2015 | /* | | 2015 | /* |
2016 | * Append the bpf header. | | 2016 | * Append the bpf header. |
2017 | */ | | 2017 | */ |
2018 | h = (char *)d->bd_sbuf + curlen; | | 2018 | h = (char *)d->bd_sbuf + curlen; |
2019 | #ifdef _LP64 | | 2019 | #ifdef _LP64 |
2020 | if (d->bd_compat32) { | | 2020 | if (d->bd_compat32) { |
2021 | struct bpf_hdr32 *hp32; | | 2021 | struct bpf_hdr32 *hp32; |
2022 | | | 2022 | |
2023 | hp32 = (struct bpf_hdr32 *)h; | | 2023 | hp32 = (struct bpf_hdr32 *)h; |
2024 | hp32->bh_tstamp.tv_sec = ts->tv_sec; | | 2024 | hp32->bh_tstamp.tv_sec = ts->tv_sec; |
2025 | hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; | | 2025 | hp32->bh_tstamp.tv_usec = ts->tv_nsec / 1000; |
2026 | hp32->bh_datalen = pktlen; | | 2026 | hp32->bh_datalen = pktlen; |
2027 | hp32->bh_hdrlen = hdrlen; | | 2027 | hp32->bh_hdrlen = hdrlen; |
2028 | hp32->bh_caplen = caplen; | | 2028 | hp32->bh_caplen = caplen; |
2029 | } else | | 2029 | } else |
2030 | #endif | | 2030 | #endif |
2031 | { | | 2031 | { |
2032 | struct bpf_hdr *hp; | | 2032 | struct bpf_hdr *hp; |
2033 | | | 2033 | |
2034 | hp = (struct bpf_hdr *)h; | | 2034 | hp = (struct bpf_hdr *)h; |
2035 | hp->bh_tstamp.tv_sec = ts->tv_sec; | | 2035 | hp->bh_tstamp.tv_sec = ts->tv_sec; |
2036 | hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; | | 2036 | hp->bh_tstamp.tv_usec = ts->tv_nsec / 1000; |
2037 | hp->bh_datalen = pktlen; | | 2037 | hp->bh_datalen = pktlen; |
2038 | hp->bh_hdrlen = hdrlen; | | 2038 | hp->bh_hdrlen = hdrlen; |
2039 | hp->bh_caplen = caplen; | | 2039 | hp->bh_caplen = caplen; |
2040 | } | | 2040 | } |
2041 | | | 2041 | |
2042 | /* | | 2042 | /* |
2043 | * Copy the packet data into the store buffer and update its length. | | 2043 | * Copy the packet data into the store buffer and update its length. |
2044 | */ | | 2044 | */ |
2045 | (*cpfn)(h + hdrlen, pkt, caplen); | | 2045 | (*cpfn)(h + hdrlen, pkt, caplen); |
2046 | d->bd_slen = curlen + totlen; | | 2046 | d->bd_slen = curlen + totlen; |
2047 | mutex_exit(d->bd_buf_mtx); | | 2047 | mutex_exit(d->bd_buf_mtx); |
2048 | | | 2048 | |
2049 | /* | | 2049 | /* |
2050 | * Call bpf_wakeup after bd_slen has been updated so that kevent(2) | | 2050 | * Call bpf_wakeup after bd_slen has been updated so that kevent(2) |
2051 | * will cause filt_bpfread() to be called with it adjusted. | | 2051 | * will cause filt_bpfread() to be called with it adjusted. |
2052 | */ | | 2052 | */ |
2053 | if (do_wakeup) | | 2053 | if (do_wakeup) |
2054 | bpf_wakeup(d); | | 2054 | bpf_wakeup(d); |
2055 | } | | 2055 | } |
2056 | | | 2056 | |
2057 | /* | | 2057 | /* |
2058 | * Initialize all nonzero fields of a descriptor. | | 2058 | * Initialize all nonzero fields of a descriptor. |
2059 | */ | | 2059 | */ |
2060 | static int | | 2060 | static int |
2061 | bpf_allocbufs(struct bpf_d *d) | | 2061 | bpf_allocbufs(struct bpf_d *d) |
2062 | { | | 2062 | { |
2063 | | | 2063 | |
2064 | d->bd_fbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP); | | 2064 | d->bd_fbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP); |
2065 | if (!d->bd_fbuf) | | 2065 | if (!d->bd_fbuf) |
2066 | return (ENOBUFS); | | 2066 | return (ENOBUFS); |
2067 | d->bd_sbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP); | | 2067 | d->bd_sbuf = kmem_zalloc(d->bd_bufsize, KM_NOSLEEP); |
2068 | if (!d->bd_sbuf) { | | 2068 | if (!d->bd_sbuf) { |
2069 | kmem_free(d->bd_fbuf, d->bd_bufsize); | | 2069 | kmem_free(d->bd_fbuf, d->bd_bufsize); |
2070 | return (ENOBUFS); | | 2070 | return (ENOBUFS); |
2071 | } | | 2071 | } |
2072 | d->bd_slen = 0; | | 2072 | d->bd_slen = 0; |
2073 | d->bd_hlen = 0; | | 2073 | d->bd_hlen = 0; |
2074 | return (0); | | 2074 | return (0); |
2075 | } | | 2075 | } |
2076 | | | 2076 | |
2077 | static void | | 2077 | static void |
2078 | bpf_free_filter(struct bpf_filter *filter) | | 2078 | bpf_free_filter(struct bpf_filter *filter) |
2079 | { | | 2079 | { |
2080 | | | 2080 | |
2081 | KASSERT(filter != NULL); | | 2081 | KASSERT(filter != NULL); |
2082 | KASSERT(filter->bf_insn != NULL); | | 2082 | KASSERT(filter->bf_insn != NULL); |
2083 | | | 2083 | |
2084 | kmem_free(filter->bf_insn, filter->bf_size); | | 2084 | kmem_free(filter->bf_insn, filter->bf_size); |
2085 | if (filter->bf_jitcode != NULL) | | 2085 | if (filter->bf_jitcode != NULL) |
2086 | bpf_jit_freecode(filter->bf_jitcode); | | 2086 | bpf_jit_freecode(filter->bf_jitcode); |
2087 | kmem_free(filter, sizeof(*filter)); | | 2087 | kmem_free(filter, sizeof(*filter)); |
2088 | } | | 2088 | } |
2089 | | | 2089 | |
2090 | /* | | 2090 | /* |
2091 | * Free buffers currently in use by a descriptor. | | 2091 | * Free buffers currently in use by a descriptor. |
2092 | * Called on close. | | 2092 | * Called on close. |
2093 | */ | | 2093 | */ |
2094 | static void | | 2094 | static void |
2095 | bpf_freed(struct bpf_d *d) | | 2095 | bpf_freed(struct bpf_d *d) |
2096 | { | | 2096 | { |
2097 | /* | | 2097 | /* |
2098 | * We don't need to lock out interrupts since this descriptor has | | 2098 | * We don't need to lock out interrupts since this descriptor has |
2099 | * been detached from its interface and it yet hasn't been marked | | 2099 | * been detached from its interface and it yet hasn't been marked |
2100 | * free. | | 2100 | * free. |
2101 | */ | | 2101 | */ |
2102 | if (d->bd_sbuf != NULL) { | | 2102 | if (d->bd_sbuf != NULL) { |
2103 | kmem_free(d->bd_sbuf, d->bd_bufsize); | | 2103 | kmem_free(d->bd_sbuf, d->bd_bufsize); |
2104 | if (d->bd_hbuf != NULL) | | 2104 | if (d->bd_hbuf != NULL) |
2105 | kmem_free(d->bd_hbuf, d->bd_bufsize); | | 2105 | kmem_free(d->bd_hbuf, d->bd_bufsize); |
2106 | if (d->bd_fbuf != NULL) | | 2106 | if (d->bd_fbuf != NULL) |
2107 | kmem_free(d->bd_fbuf, d->bd_bufsize); | | 2107 | kmem_free(d->bd_fbuf, d->bd_bufsize); |
2108 | } | | 2108 | } |
2109 | if (d->bd_rfilter != NULL) { | | 2109 | if (d->bd_rfilter != NULL) { |
2110 | bpf_free_filter(d->bd_rfilter); | | 2110 | bpf_free_filter(d->bd_rfilter); |
2111 | d->bd_rfilter = NULL; | | 2111 | d->bd_rfilter = NULL; |
2112 | } | | 2112 | } |
2113 | if (d->bd_wfilter != NULL) { | | 2113 | if (d->bd_wfilter != NULL) { |
2114 | bpf_free_filter(d->bd_wfilter); | | 2114 | bpf_free_filter(d->bd_wfilter); |
2115 | d->bd_wfilter = NULL; | | 2115 | d->bd_wfilter = NULL; |
2116 | } | | 2116 | } |
2117 | d->bd_jitcode = NULL; | | 2117 | d->bd_jitcode = NULL; |
2118 | } | | 2118 | } |
2119 | | | 2119 | |
2120 | /* | | 2120 | /* |
2121 | * Attach an interface to bpf. dlt is the link layer type; | | 2121 | * Attach an interface to bpf. dlt is the link layer type; |
2122 | * hdrlen is the fixed size of the link header for the specified dlt | | 2122 | * hdrlen is the fixed size of the link header for the specified dlt |
2123 | * (variable length headers not yet supported). | | 2123 | * (variable length headers not yet supported). |
2124 | */ | | 2124 | */ |
2125 | static void | | 2125 | static void |
2126 | _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) | | 2126 | _bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen, struct bpf_if **driverp) |
2127 | { | | 2127 | { |
2128 | struct bpf_if *bp; | | 2128 | struct bpf_if *bp; |
2129 | bp = kmem_alloc(sizeof(*bp), KM_NOSLEEP); | | 2129 | bp = kmem_alloc(sizeof(*bp), KM_NOSLEEP); |
2130 | if (bp == NULL) | | 2130 | if (bp == NULL) |
2131 | panic("%s: out of memory", __func__); | | 2131 | panic("%s: out of memory", __func__); |
2132 | | | 2132 | |
2133 | mutex_enter(&bpf_mtx); | | 2133 | mutex_enter(&bpf_mtx); |
2134 | bp->bif_driverp = driverp; | | 2134 | bp->bif_driverp = driverp; |
2135 | bp->bif_ifp = ifp; | | 2135 | bp->bif_ifp = ifp; |
2136 | bp->bif_dlt = dlt; | | 2136 | bp->bif_dlt = dlt; |
2137 | bp->bif_si = NULL; | | 2137 | bp->bif_si = NULL; |
2138 | BPF_IFLIST_ENTRY_INIT(bp); | | 2138 | BPF_IFLIST_ENTRY_INIT(bp); |
2139 | PSLIST_INIT(&bp->bif_dlist_head); | | 2139 | PSLIST_INIT(&bp->bif_dlist_head); |
2140 | psref_target_init(&bp->bif_psref, bpf_psref_class); | | 2140 | psref_target_init(&bp->bif_psref, bpf_psref_class); |
2141 | SLIST_INIT(&bp->bif_trackers); | | 2141 | SLIST_INIT(&bp->bif_trackers); |
2142 | | | 2142 | |
2143 | BPF_IFLIST_WRITER_INSERT_HEAD(bp); | | 2143 | BPF_IFLIST_WRITER_INSERT_HEAD(bp); |
2144 | | | 2144 | |
2145 | *bp->bif_driverp = NULL; | | 2145 | *bp->bif_driverp = NULL; |
2146 | | | 2146 | |
2147 | bp->bif_hdrlen = hdrlen; | | 2147 | bp->bif_hdrlen = hdrlen; |
2148 | mutex_exit(&bpf_mtx); | | 2148 | mutex_exit(&bpf_mtx); |
2149 | #if 0 | | 2149 | #if 0 |
2150 | printf("bpf: %s attached with dlt %x\n", ifp->if_xname, dlt); | | 2150 | printf("bpf: %s attached with dlt %x\n", ifp->if_xname, dlt); |
2151 | #endif | | 2151 | #endif |
2152 | } | | 2152 | } |
2153 | | | 2153 | |
2154 | static void | | 2154 | static void |
2155 | _bpf_mtap_softint_init(struct ifnet *ifp) | | 2155 | _bpf_mtap_softint_init(struct ifnet *ifp) |
2156 | { | | 2156 | { |
2157 | struct bpf_if *bp; | | 2157 | struct bpf_if *bp; |
2158 | | | 2158 | |
2159 | mutex_enter(&bpf_mtx); | | 2159 | mutex_enter(&bpf_mtx); |
2160 | BPF_IFLIST_WRITER_FOREACH(bp) { | | 2160 | BPF_IFLIST_WRITER_FOREACH(bp) { |
2161 | if (bp->bif_ifp != ifp) | | 2161 | if (bp->bif_ifp != ifp) |
2162 | continue; | | 2162 | continue; |
2163 | | | 2163 | |
2164 | bp->bif_mbuf_head = NULL; | | 2164 | bp->bif_mbuf_head = NULL; |
2165 | bp->bif_mbuf_tail = NULL; | | 2165 | bp->bif_mbuf_tail = NULL; |
2166 | bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp); | | 2166 | bp->bif_si = softint_establish(SOFTINT_NET, bpf_mtap_si, bp); |
2167 | if (bp->bif_si == NULL) | | 2167 | if (bp->bif_si == NULL) |
2168 | panic("%s: softint_establish() failed", __func__); | | 2168 | panic("%s: softint_establish() failed", __func__); |
2169 | break; | | 2169 | break; |
2170 | } | | 2170 | } |
2171 | mutex_exit(&bpf_mtx); | | 2171 | mutex_exit(&bpf_mtx); |
2172 | | | 2172 | |
2173 | if (bp == NULL) | | 2173 | if (bp == NULL) |
2174 | panic("%s: no bpf_if found for %s", __func__, ifp->if_xname); | | 2174 | panic("%s: no bpf_if found for %s", __func__, ifp->if_xname); |
2175 | } | | 2175 | } |
2176 | | | 2176 | |
2177 | /* | | 2177 | /* |
2178 | * Remove an interface from bpf. | | 2178 | * Remove an interface from bpf. |
2179 | */ | | 2179 | */ |
2180 | static void | | 2180 | static void |
2181 | _bpfdetach(struct ifnet *ifp) | | 2181 | _bpfdetach(struct ifnet *ifp) |
2182 | { | | 2182 | { |
2183 | struct bpf_if *bp; | | 2183 | struct bpf_if *bp; |
2184 | struct bpf_d *d; | | 2184 | struct bpf_d *d; |
2185 | int s; | | 2185 | int s; |
2186 | | | 2186 | |
2187 | mutex_enter(&bpf_mtx); | | 2187 | mutex_enter(&bpf_mtx); |
2188 | /* Nuke the vnodes for any open instances */ | | 2188 | /* Nuke the vnodes for any open instances */ |
2189 | again_d: | | 2189 | again_d: |
2190 | BPF_DLIST_WRITER_FOREACH(d) { | | 2190 | BPF_DLIST_WRITER_FOREACH(d) { |
2191 | mutex_enter(d->bd_mtx); | | 2191 | mutex_enter(d->bd_mtx); |
2192 | if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { | | 2192 | if (d->bd_bif != NULL && d->bd_bif->bif_ifp == ifp) { |
2193 | /* | | 2193 | /* |
2194 | * Detach the descriptor from an interface now. | | 2194 | * Detach the descriptor from an interface now. |
2195 | * It will be free'ed later by close routine. | | 2195 | * It will be free'ed later by close routine. |
2196 | */ | | 2196 | */ |
2197 | d->bd_promisc = 0; /* we can't touch device. */ | | | |
2198 | bpf_detachd(d); | | 2197 | bpf_detachd(d); |
2199 | mutex_exit(d->bd_mtx); | | 2198 | mutex_exit(d->bd_mtx); |
2200 | goto again_d; | | 2199 | goto again_d; |
2201 | } | | 2200 | } |
2202 | mutex_exit(d->bd_mtx); | | 2201 | mutex_exit(d->bd_mtx); |
2203 | } | | 2202 | } |
2204 | | | 2203 | |
2205 | again: | | 2204 | again: |
2206 | BPF_IFLIST_WRITER_FOREACH(bp) { | | 2205 | BPF_IFLIST_WRITER_FOREACH(bp) { |
2207 | if (bp->bif_ifp == ifp) { | | 2206 | if (bp->bif_ifp == ifp) { |
2208 | BPF_IFLIST_WRITER_REMOVE(bp); | | 2207 | BPF_IFLIST_WRITER_REMOVE(bp); |
2209 | | | 2208 | |
2210 | pserialize_perform(bpf_psz); | | 2209 | pserialize_perform(bpf_psz); |
2211 | psref_target_destroy(&bp->bif_psref, bpf_psref_class); | | 2210 | psref_target_destroy(&bp->bif_psref, bpf_psref_class); |
2212 | | | 2211 | |
2213 | while (!SLIST_EMPTY(&bp->bif_trackers)) { | | 2212 | while (!SLIST_EMPTY(&bp->bif_trackers)) { |
2214 | struct bpf_event_tracker *t = | | 2213 | struct bpf_event_tracker *t = |
2215 | SLIST_FIRST(&bp->bif_trackers); | | 2214 | SLIST_FIRST(&bp->bif_trackers); |
2216 | SLIST_REMOVE_HEAD(&bp->bif_trackers, | | 2215 | SLIST_REMOVE_HEAD(&bp->bif_trackers, |
2217 | bet_entries); | | 2216 | bet_entries); |
2218 | kmem_free(t, sizeof(*t)); | | 2217 | kmem_free(t, sizeof(*t)); |
2219 | } | | 2218 | } |
2220 | | | 2219 | |
2221 | BPF_IFLIST_ENTRY_DESTROY(bp); | | 2220 | BPF_IFLIST_ENTRY_DESTROY(bp); |
2222 | if (bp->bif_si != NULL) { | | 2221 | if (bp->bif_si != NULL) { |
2223 | /* XXX NOMPSAFE: assumed running on one CPU */ | | 2222 | /* XXX NOMPSAFE: assumed running on one CPU */ |
2224 | s = splnet(); | | 2223 | s = splnet(); |
2225 | while (bp->bif_mbuf_head != NULL) { | | 2224 | while (bp->bif_mbuf_head != NULL) { |
2226 | struct mbuf *m = bp->bif_mbuf_head; | | 2225 | struct mbuf *m = bp->bif_mbuf_head; |
2227 | bp->bif_mbuf_head = m->m_nextpkt; | | 2226 | bp->bif_mbuf_head = m->m_nextpkt; |
2228 | m_freem(m); | | 2227 | m_freem(m); |
2229 | } | | 2228 | } |
2230 | splx(s); | | 2229 | splx(s); |
2231 | softint_disestablish(bp->bif_si); | | 2230 | softint_disestablish(bp->bif_si); |
2232 | } | | 2231 | } |
2233 | kmem_free(bp, sizeof(*bp)); | | 2232 | kmem_free(bp, sizeof(*bp)); |
2234 | goto again; | | 2233 | goto again; |
2235 | } | | 2234 | } |
2236 | } | | 2235 | } |
2237 | mutex_exit(&bpf_mtx); | | 2236 | mutex_exit(&bpf_mtx); |
2238 | } | | 2237 | } |
2239 | | | 2238 | |
2240 | /* | | 2239 | /* |
2241 | * Change the data link type of a interface. | | 2240 | * Change the data link type of a interface. |
2242 | */ | | 2241 | */ |
2243 | static void | | 2242 | static void |
2244 | _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) | | 2243 | _bpf_change_type(struct ifnet *ifp, u_int dlt, u_int hdrlen) |
2245 | { | | 2244 | { |
2246 | struct bpf_if *bp; | | 2245 | struct bpf_if *bp; |
2247 | | | 2246 | |
2248 | mutex_enter(&bpf_mtx); | | 2247 | mutex_enter(&bpf_mtx); |
2249 | BPF_IFLIST_WRITER_FOREACH(bp) { | | 2248 | BPF_IFLIST_WRITER_FOREACH(bp) { |
2250 | if (bp->bif_driverp == &ifp->if_bpf) | | 2249 | if (bp->bif_driverp == &ifp->if_bpf) |
2251 | break; | | 2250 | break; |
2252 | } | | 2251 | } |
2253 | if (bp == NULL) | | 2252 | if (bp == NULL) |
2254 | panic("bpf_change_type"); | | 2253 | panic("bpf_change_type"); |
2255 | | | 2254 | |
2256 | bp->bif_dlt = dlt; | | 2255 | bp->bif_dlt = dlt; |
2257 | | | 2256 | |
2258 | bp->bif_hdrlen = hdrlen; | | 2257 | bp->bif_hdrlen = hdrlen; |
2259 | mutex_exit(&bpf_mtx); | | 2258 | mutex_exit(&bpf_mtx); |
2260 | } | | 2259 | } |
2261 | | | 2260 | |
2262 | /* | | 2261 | /* |
2263 | * Get a list of available data link type of the interface. | | 2262 | * Get a list of available data link type of the interface. |
2264 | */ | | 2263 | */ |
2265 | static int | | 2264 | static int |
2266 | bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) | | 2265 | bpf_getdltlist(struct bpf_d *d, struct bpf_dltlist *bfl) |
2267 | { | | 2266 | { |
2268 | int n, error; | | 2267 | int n, error; |
2269 | struct ifnet *ifp; | | 2268 | struct ifnet *ifp; |
2270 | struct bpf_if *bp; | | 2269 | struct bpf_if *bp; |
2271 | int s, bound; | | 2270 | int s, bound; |
2272 | | | 2271 | |
2273 | KASSERT(mutex_owned(d->bd_mtx)); | | 2272 | KASSERT(mutex_owned(d->bd_mtx)); |
2274 | | | 2273 | |
2275 | ifp = d->bd_bif->bif_ifp; | | 2274 | ifp = d->bd_bif->bif_ifp; |
2276 | n = 0; | | 2275 | n = 0; |
2277 | error = 0; | | 2276 | error = 0; |
2278 | | | 2277 | |
2279 | bound = curlwp_bind(); | | 2278 | bound = curlwp_bind(); |
2280 | s = pserialize_read_enter(); | | 2279 | s = pserialize_read_enter(); |
2281 | BPF_IFLIST_READER_FOREACH(bp) { | | 2280 | BPF_IFLIST_READER_FOREACH(bp) { |
2282 | if (bp->bif_ifp != ifp) | | 2281 | if (bp->bif_ifp != ifp) |
2283 | continue; | | 2282 | continue; |
2284 | if (bfl->bfl_list != NULL) { | | 2283 | if (bfl->bfl_list != NULL) { |
2285 | struct psref psref; | | 2284 | struct psref psref; |
2286 | | | 2285 | |
2287 | if (n >= bfl->bfl_len) { | | 2286 | if (n >= bfl->bfl_len) { |
2288 | pserialize_read_exit(s); | | 2287 | pserialize_read_exit(s); |
2289 | return ENOMEM; | | 2288 | return ENOMEM; |
2290 | } | | 2289 | } |
2291 | | | 2290 | |
2292 | bpf_if_acquire(bp, &psref); | | 2291 | bpf_if_acquire(bp, &psref); |
2293 | pserialize_read_exit(s); | | 2292 | pserialize_read_exit(s); |
2294 | | | 2293 | |
2295 | error = copyout(&bp->bif_dlt, | | 2294 | error = copyout(&bp->bif_dlt, |
2296 | bfl->bfl_list + n, sizeof(u_int)); | | 2295 | bfl->bfl_list + n, sizeof(u_int)); |
2297 | | | 2296 | |
2298 | s = pserialize_read_enter(); | | 2297 | s = pserialize_read_enter(); |
2299 | bpf_if_release(bp, &psref); | | 2298 | bpf_if_release(bp, &psref); |
2300 | } | | 2299 | } |
2301 | n++; | | 2300 | n++; |
2302 | } | | 2301 | } |
2303 | pserialize_read_exit(s); | | 2302 | pserialize_read_exit(s); |
2304 | curlwp_bindx(bound); | | 2303 | curlwp_bindx(bound); |
2305 | | | 2304 | |
2306 | bfl->bfl_len = n; | | 2305 | bfl->bfl_len = n; |
2307 | return error; | | 2306 | return error; |
2308 | } | | 2307 | } |
2309 | | | 2308 | |
2310 | /* | | 2309 | /* |
2311 | * Set the data link type of a BPF instance. | | 2310 | * Set the data link type of a BPF instance. |
2312 | */ | | 2311 | */ |
2313 | static int | | 2312 | static int |
2314 | bpf_setdlt(struct bpf_d *d, u_int dlt) | | 2313 | bpf_setdlt(struct bpf_d *d, u_int dlt) |
2315 | { | | 2314 | { |
2316 | int error, opromisc; | | 2315 | int error, opromisc; |
2317 | struct ifnet *ifp; | | 2316 | struct ifnet *ifp; |
2318 | struct bpf_if *bp; | | 2317 | struct bpf_if *bp; |
2319 | | | 2318 | |
2320 | KASSERT(mutex_owned(&bpf_mtx)); | | 2319 | KASSERT(mutex_owned(&bpf_mtx)); |
2321 | KASSERT(mutex_owned(d->bd_mtx)); | | 2320 | KASSERT(mutex_owned(d->bd_mtx)); |
2322 | | | 2321 | |
2323 | if (d->bd_bif->bif_dlt == dlt) | | 2322 | if (d->bd_bif->bif_dlt == dlt) |
2324 | return 0; | | 2323 | return 0; |
2325 | ifp = d->bd_bif->bif_ifp; | | 2324 | ifp = d->bd_bif->bif_ifp; |
2326 | BPF_IFLIST_WRITER_FOREACH(bp) { | | 2325 | BPF_IFLIST_WRITER_FOREACH(bp) { |
2327 | if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) | | 2326 | if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) |
2328 | break; | | 2327 | break; |
2329 | } | | 2328 | } |
2330 | if (bp == NULL) | | 2329 | if (bp == NULL) |
2331 | return EINVAL; | | 2330 | return EINVAL; |
2332 | opromisc = d->bd_promisc; | | 2331 | opromisc = d->bd_promisc; |
2333 | bpf_detachd(d); | | 2332 | bpf_detachd(d); |
2334 | BPFIF_DLIST_ENTRY_INIT(d); | | 2333 | BPFIF_DLIST_ENTRY_INIT(d); |
2335 | bpf_attachd(d, bp); | | 2334 | bpf_attachd(d, bp); |
2336 | reset_d(d); | | 2335 | reset_d(d); |
2337 | if (opromisc) { | | 2336 | if (opromisc) { |
2338 | KERNEL_LOCK_UNLESS_NET_MPSAFE(); | | 2337 | KERNEL_LOCK_UNLESS_NET_MPSAFE(); |
2339 | error = ifpromisc(bp->bif_ifp, 1); | | 2338 | error = ifpromisc(bp->bif_ifp, 1); |
2340 | KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); | | 2339 | KERNEL_UNLOCK_UNLESS_NET_MPSAFE(); |
2341 | if (error) | | 2340 | if (error) |
2342 | printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", | | 2341 | printf("%s: bpf_setdlt: ifpromisc failed (%d)\n", |
2343 | bp->bif_ifp->if_xname, error); | | 2342 | bp->bif_ifp->if_xname, error); |
2344 | else | | 2343 | else |
2345 | d->bd_promisc = 1; | | 2344 | d->bd_promisc = 1; |
2346 | } | | 2345 | } |
2347 | return 0; | | 2346 | return 0; |
2348 | } | | 2347 | } |
2349 | | | 2348 | |
2350 | static int | | 2349 | static int |
2351 | sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) | | 2350 | sysctl_net_bpf_maxbufsize(SYSCTLFN_ARGS) |
2352 | { | | 2351 | { |
2353 | int newsize, error; | | 2352 | int newsize, error; |
2354 | struct sysctlnode node; | | 2353 | struct sysctlnode node; |
2355 | | | 2354 | |
2356 | node = *rnode; | | 2355 | node = *rnode; |
2357 | node.sysctl_data = &newsize; | | 2356 | node.sysctl_data = &newsize; |
2358 | newsize = bpf_maxbufsize; | | 2357 | newsize = bpf_maxbufsize; |
2359 | error = sysctl_lookup(SYSCTLFN_CALL(&node)); | | 2358 | error = sysctl_lookup(SYSCTLFN_CALL(&node)); |
2360 | if (error || newp == NULL) | | 2359 | if (error || newp == NULL) |
2361 | return (error); | | 2360 | return (error); |
2362 | | | 2361 | |
2363 | if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) | | 2362 | if (newsize < BPF_MINBUFSIZE || newsize > BPF_MAXBUFSIZE) |
2364 | return (EINVAL); | | 2363 | return (EINVAL); |
2365 | | | 2364 | |
2366 | bpf_maxbufsize = newsize; | | 2365 | bpf_maxbufsize = newsize; |
2367 | | | 2366 | |
2368 | return (0); | | 2367 | return (0); |
2369 | } | | 2368 | } |
2370 | | | 2369 | |
2371 | #if defined(MODULAR) || defined(BPFJIT) | | 2370 | #if defined(MODULAR) || defined(BPFJIT) |
2372 | static int | | 2371 | static int |
2373 | sysctl_net_bpf_jit(SYSCTLFN_ARGS) | | 2372 | sysctl_net_bpf_jit(SYSCTLFN_ARGS) |
2374 | { | | 2373 | { |
2375 | bool newval; | | 2374 | bool newval; |
2376 | int error; | | 2375 | int error; |
2377 | struct sysctlnode node; | | 2376 | struct sysctlnode node; |
2378 | | | 2377 | |
2379 | node = *rnode; | | 2378 | node = *rnode; |
2380 | node.sysctl_data = &newval; | | 2379 | node.sysctl_data = &newval; |
2381 | newval = bpf_jit; | | 2380 | newval = bpf_jit; |
2382 | error = sysctl_lookup(SYSCTLFN_CALL(&node)); | | 2381 | error = sysctl_lookup(SYSCTLFN_CALL(&node)); |
2383 | if (error != 0 || newp == NULL) | | 2382 | if (error != 0 || newp == NULL) |
2384 | return error; | | 2383 | return error; |
2385 | | | 2384 | |
2386 | bpf_jit = newval; | | 2385 | bpf_jit = newval; |
2387 | if (newval && bpfjit_module_ops.bj_generate_code == NULL) { | | 2386 | if (newval && bpfjit_module_ops.bj_generate_code == NULL) { |
2388 | printf("JIT compilation is postponed " | | 2387 | printf("JIT compilation is postponed " |
2389 | "until after bpfjit module is loaded\n"); | | 2388 | "until after bpfjit module is loaded\n"); |
2390 | } | | 2389 | } |
2391 | | | 2390 | |
2392 | return 0; | | 2391 | return 0; |
2393 | } | | 2392 | } |
2394 | #endif | | 2393 | #endif |
2395 | | | 2394 | |
2396 | static int | | 2395 | static int |
2397 | sysctl_net_bpf_peers(SYSCTLFN_ARGS) | | 2396 | sysctl_net_bpf_peers(SYSCTLFN_ARGS) |
2398 | { | | 2397 | { |
2399 | int error, elem_count; | | 2398 | int error, elem_count; |
2400 | struct bpf_d *dp; | | 2399 | struct bpf_d *dp; |
2401 | struct bpf_d_ext dpe; | | 2400 | struct bpf_d_ext dpe; |
2402 | size_t len, needed, elem_size, out_size; | | 2401 | size_t len, needed, elem_size, out_size; |
2403 | char *sp; | | 2402 | char *sp; |
2404 | | | 2403 | |
2405 | if (namelen == 1 && name[0] == CTL_QUERY) | | 2404 | if (namelen == 1 && name[0] == CTL_QUERY) |
2406 | return (sysctl_query(SYSCTLFN_CALL(rnode))); | | 2405 | return (sysctl_query(SYSCTLFN_CALL(rnode))); |
2407 | | | 2406 | |
2408 | if (namelen != 2) | | 2407 | if (namelen != 2) |
2409 | return (EINVAL); | | 2408 | return (EINVAL); |
2410 | | | 2409 | |
2411 | /* BPF peers is privileged information. */ | | 2410 | /* BPF peers is privileged information. */ |
2412 | error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, | | 2411 | error = kauth_authorize_network(l->l_cred, KAUTH_NETWORK_INTERFACE, |
2413 | KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); | | 2412 | KAUTH_REQ_NETWORK_INTERFACE_GETPRIV, NULL, NULL, NULL); |
2414 | if (error) | | 2413 | if (error) |
2415 | return (EPERM); | | 2414 | return (EPERM); |
2416 | | | 2415 | |
2417 | len = (oldp != NULL) ? *oldlenp : 0; | | 2416 | len = (oldp != NULL) ? *oldlenp : 0; |
2418 | sp = oldp; | | 2417 | sp = oldp; |
2419 | elem_size = name[0]; | | 2418 | elem_size = name[0]; |
2420 | elem_count = name[1]; | | 2419 | elem_count = name[1]; |
2421 | out_size = MIN(sizeof(dpe), elem_size); | | 2420 | out_size = MIN(sizeof(dpe), elem_size); |
2422 | needed = 0; | | 2421 | needed = 0; |
2423 | | | 2422 | |
2424 | if (elem_size < 1 || elem_count < 0) | | 2423 | if (elem_size < 1 || elem_count < 0) |
2425 | return (EINVAL); | | 2424 | return (EINVAL); |
2426 | | | 2425 | |
2427 | mutex_enter(&bpf_mtx); | | 2426 | mutex_enter(&bpf_mtx); |
2428 | BPF_DLIST_WRITER_FOREACH(dp) { | | 2427 | BPF_DLIST_WRITER_FOREACH(dp) { |
2429 | if (len >= elem_size && elem_count > 0) { | | 2428 | if (len >= elem_size && elem_count > 0) { |
2430 | #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field | | 2429 | #define BPF_EXT(field) dpe.bde_ ## field = dp->bd_ ## field |
2431 | BPF_EXT(bufsize); | | 2430 | BPF_EXT(bufsize); |
2432 | BPF_EXT(promisc); | | 2431 | BPF_EXT(promisc); |
2433 | BPF_EXT(state); | | 2432 | BPF_EXT(state); |
2434 | BPF_EXT(immediate); | | 2433 | BPF_EXT(immediate); |
2435 | BPF_EXT(hdrcmplt); | | 2434 | BPF_EXT(hdrcmplt); |
2436 | BPF_EXT(direction); | | 2435 | BPF_EXT(direction); |
2437 | BPF_EXT(pid); | | 2436 | BPF_EXT(pid); |
2438 | BPF_EXT(rcount); | | 2437 | BPF_EXT(rcount); |
2439 | BPF_EXT(dcount); | | 2438 | BPF_EXT(dcount); |
2440 | BPF_EXT(ccount); | | 2439 | BPF_EXT(ccount); |
2441 | #undef BPF_EXT | | 2440 | #undef BPF_EXT |
2442 | mutex_enter(dp->bd_mtx); | | 2441 | mutex_enter(dp->bd_mtx); |
2443 | if (dp->bd_bif) | | 2442 | if (dp->bd_bif) |
2444 | (void)strlcpy(dpe.bde_ifname, | | 2443 | (void)strlcpy(dpe.bde_ifname, |
2445 | dp->bd_bif->bif_ifp->if_xname, | | 2444 | dp->bd_bif->bif_ifp->if_xname, |
2446 | IFNAMSIZ - 1); | | 2445 | IFNAMSIZ - 1); |
2447 | else | | 2446 | else |
2448 | dpe.bde_ifname[0] = '\0'; | | 2447 | dpe.bde_ifname[0] = '\0'; |
2449 | dpe.bde_locked = dp->bd_locked; | | 2448 | dpe.bde_locked = dp->bd_locked; |
2450 | mutex_exit(dp->bd_mtx); | | 2449 | mutex_exit(dp->bd_mtx); |
2451 | | | 2450 | |
2452 | error = copyout(&dpe, sp, out_size); | | 2451 | error = copyout(&dpe, sp, out_size); |
2453 | if (error) | | 2452 | if (error) |
2454 | break; | | 2453 | break; |
2455 | sp += elem_size; | | 2454 | sp += elem_size; |
2456 | len -= elem_size; | | 2455 | len -= elem_size; |
2457 | } | | 2456 | } |
2458 | needed += elem_size; | | 2457 | needed += elem_size; |
2459 | if (elem_count > 0 && elem_count != INT_MAX) | | 2458 | if (elem_count > 0 && elem_count != INT_MAX) |
2460 | elem_count--; | | 2459 | elem_count--; |
2461 | } | | 2460 | } |
2462 | mutex_exit(&bpf_mtx); | | 2461 | mutex_exit(&bpf_mtx); |
2463 | | | 2462 | |
2464 | *oldlenp = needed; | | 2463 | *oldlenp = needed; |
2465 | | | 2464 | |
2466 | return (error); | | 2465 | return (error); |
2467 | } | | 2466 | } |
2468 | | | 2467 | |
2469 | static void | | 2468 | static void |
2470 | bpf_stats(void *p, void *arg, struct cpu_info *ci __unused) | | 2469 | bpf_stats(void *p, void *arg, struct cpu_info *ci __unused) |
2471 | { | | 2470 | { |
2472 | struct bpf_stat *const stats = p; | | 2471 | struct bpf_stat *const stats = p; |
2473 | struct bpf_stat *sum = arg; | | 2472 | struct bpf_stat *sum = arg; |
2474 | | | 2473 | |
2475 | int s = splnet(); | | 2474 | int s = splnet(); |
2476 | | | 2475 | |
2477 | sum->bs_recv += stats->bs_recv; | | 2476 | sum->bs_recv += stats->bs_recv; |
2478 | sum->bs_drop += stats->bs_drop; | | 2477 | sum->bs_drop += stats->bs_drop; |
2479 | sum->bs_capt += stats->bs_capt; | | 2478 | sum->bs_capt += stats->bs_capt; |
2480 | | | 2479 | |
2481 | splx(s); | | 2480 | splx(s); |
2482 | } | | 2481 | } |
2483 | | | 2482 | |
2484 | static int | | 2483 | static int |
2485 | bpf_sysctl_gstats_handler(SYSCTLFN_ARGS) | | 2484 | bpf_sysctl_gstats_handler(SYSCTLFN_ARGS) |
2486 | { | | 2485 | { |
2487 | struct sysctlnode node; | | 2486 | struct sysctlnode node; |
2488 | int error; | | 2487 | int error; |
2489 | struct bpf_stat sum; | | 2488 | struct bpf_stat sum; |
2490 | | | 2489 | |
2491 | memset(&sum, 0, sizeof(sum)); | | 2490 | memset(&sum, 0, sizeof(sum)); |
2492 | node = *rnode; | | 2491 | node = *rnode; |
2493 | | | 2492 | |
2494 | percpu_foreach_xcall(bpf_gstats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET), | | 2493 | percpu_foreach_xcall(bpf_gstats_percpu, XC_HIGHPRI_IPL(IPL_SOFTNET), |
2495 | bpf_stats, &sum); | | 2494 | bpf_stats, &sum); |
2496 | | | 2495 | |
2497 | node.sysctl_data = ∑ | | 2496 | node.sysctl_data = ∑ |
2498 | node.sysctl_size = sizeof(sum); | | 2497 | node.sysctl_size = sizeof(sum); |
2499 | error = sysctl_lookup(SYSCTLFN_CALL(&node)); | | 2498 | error = sysctl_lookup(SYSCTLFN_CALL(&node)); |
2500 | if (error != 0 || newp == NULL) | | 2499 | if (error != 0 || newp == NULL) |
2501 | return error; | | 2500 | return error; |
2502 | | | 2501 | |
2503 | return 0; | | 2502 | return 0; |
2504 | } | | 2503 | } |
2505 | | | 2504 | |
2506 | SYSCTL_SETUP(sysctl_net_bpf_setup, "bpf sysctls") | | 2505 | SYSCTL_SETUP(sysctl_net_bpf_setup, "bpf sysctls") |
2507 | { | | 2506 | { |
2508 | const struct sysctlnode *node; | | 2507 | const struct sysctlnode *node; |
2509 | | | 2508 | |
2510 | node = NULL; | | 2509 | node = NULL; |
2511 | sysctl_createv(clog, 0, NULL, &node, | | 2510 | sysctl_createv(clog, 0, NULL, &node, |
2512 | CTLFLAG_PERMANENT, | | 2511 | CTLFLAG_PERMANENT, |
2513 | CTLTYPE_NODE, "bpf", | | 2512 | CTLTYPE_NODE, "bpf", |
2514 | SYSCTL_DESCR("BPF options"), | | 2513 | SYSCTL_DESCR("BPF options"), |
2515 | NULL, 0, NULL, 0, | | 2514 | NULL, 0, NULL, 0, |
2516 | CTL_NET, CTL_CREATE, CTL_EOL); | | 2515 | CTL_NET, CTL_CREATE, CTL_EOL); |
2517 | if (node != NULL) { | | 2516 | if (node != NULL) { |
2518 | #if defined(MODULAR) || defined(BPFJIT) | | 2517 | #if defined(MODULAR) || defined(BPFJIT) |
2519 | sysctl_createv(clog, 0, NULL, NULL, | | 2518 | sysctl_createv(clog, 0, NULL, NULL, |
2520 | CTLFLAG_PERMANENT|CTLFLAG_READWRITE, | | 2519 | CTLFLAG_PERMANENT|CTLFLAG_READWRITE, |
2521 | CTLTYPE_BOOL, "jit", | | 2520 | CTLTYPE_BOOL, "jit", |
2522 | SYSCTL_DESCR("Toggle Just-In-Time compilation"), | | 2521 | SYSCTL_DESCR("Toggle Just-In-Time compilation"), |
2523 | sysctl_net_bpf_jit, 0, &bpf_jit, 0, | | 2522 | sysctl_net_bpf_jit, 0, &bpf_jit, 0, |
2524 | CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); | | 2523 | CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); |
2525 | #endif | | 2524 | #endif |
2526 | sysctl_createv(clog, 0, NULL, NULL, | | 2525 | sysctl_createv(clog, 0, NULL, NULL, |
2527 | CTLFLAG_PERMANENT|CTLFLAG_READWRITE, | | 2526 | CTLFLAG_PERMANENT|CTLFLAG_READWRITE, |
2528 | CTLTYPE_INT, "maxbufsize", | | 2527 | CTLTYPE_INT, "maxbufsize", |
2529 | SYSCTL_DESCR("Maximum size for data capture buffer"), | | 2528 | SYSCTL_DESCR("Maximum size for data capture buffer"), |
2530 | sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, | | 2529 | sysctl_net_bpf_maxbufsize, 0, &bpf_maxbufsize, 0, |
2531 | CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); | | 2530 | CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); |
2532 | sysctl_createv(clog, 0, NULL, NULL, | | 2531 | sysctl_createv(clog, 0, NULL, NULL, |
2533 | CTLFLAG_PERMANENT, | | 2532 | CTLFLAG_PERMANENT, |
2534 | CTLTYPE_STRUCT, "stats", | | 2533 | CTLTYPE_STRUCT, "stats", |
2535 | SYSCTL_DESCR("BPF stats"), | | 2534 | SYSCTL_DESCR("BPF stats"), |
2536 | bpf_sysctl_gstats_handler, 0, NULL, 0, | | 2535 | bpf_sysctl_gstats_handler, 0, NULL, 0, |
2537 | CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); | | 2536 | CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); |
2538 | sysctl_createv(clog, 0, NULL, NULL, | | 2537 | sysctl_createv(clog, 0, NULL, NULL, |
2539 | CTLFLAG_PERMANENT, | | 2538 | CTLFLAG_PERMANENT, |
2540 | CTLTYPE_STRUCT, "peers", | | 2539 | CTLTYPE_STRUCT, "peers", |
2541 | SYSCTL_DESCR("BPF peers"), | | 2540 | SYSCTL_DESCR("BPF peers"), |
2542 | sysctl_net_bpf_peers, 0, NULL, 0, | | 2541 | sysctl_net_bpf_peers, 0, NULL, 0, |
2543 | CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); | | 2542 | CTL_NET, node->sysctl_num, CTL_CREATE, CTL_EOL); |
2544 | } | | 2543 | } |
2545 | | | 2544 | |
2546 | } | | 2545 | } |
2547 | | | 2546 | |
2548 | static int | | 2547 | static int |
2549 | _bpf_register_track_event(struct bpf_if **driverp, | | 2548 | _bpf_register_track_event(struct bpf_if **driverp, |
2550 | void (*_fun)(struct bpf_if *, struct ifnet *, int, int)) | | 2549 | void (*_fun)(struct bpf_if *, struct ifnet *, int, int)) |
2551 | { | | 2550 | { |
2552 | struct bpf_if *bp; | | 2551 | struct bpf_if *bp; |
2553 | struct bpf_event_tracker *t; | | 2552 | struct bpf_event_tracker *t; |
2554 | int ret = ENOENT; | | 2553 | int ret = ENOENT; |
2555 | | | 2554 | |
2556 | t = kmem_zalloc(sizeof(*t), KM_SLEEP); | | 2555 | t = kmem_zalloc(sizeof(*t), KM_SLEEP); |
2557 | if (!t) | | 2556 | if (!t) |
2558 | return ENOMEM; | | 2557 | return ENOMEM; |
2559 | t->bet_notify = _fun; | | 2558 | t->bet_notify = _fun; |
2560 | | | 2559 | |
2561 | mutex_enter(&bpf_mtx); | | 2560 | mutex_enter(&bpf_mtx); |
2562 | BPF_IFLIST_WRITER_FOREACH(bp) { | | 2561 | BPF_IFLIST_WRITER_FOREACH(bp) { |
2563 | if (bp->bif_driverp != driverp) | | 2562 | if (bp->bif_driverp != driverp) |
2564 | continue; | | 2563 | continue; |
2565 | SLIST_INSERT_HEAD(&bp->bif_trackers, t, bet_entries); | | 2564 | SLIST_INSERT_HEAD(&bp->bif_trackers, t, bet_entries); |
2566 | ret = 0; | | 2565 | ret = 0; |
2567 | break; | | 2566 | break; |
2568 | } | | 2567 | } |
2569 | mutex_exit(&bpf_mtx); | | 2568 | mutex_exit(&bpf_mtx); |
2570 | | | 2569 | |
2571 | return ret; | | 2570 | return ret; |
2572 | } | | 2571 | } |
2573 | | | 2572 | |
2574 | static int | | 2573 | static int |
2575 | _bpf_deregister_track_event(struct bpf_if **driverp, | | 2574 | _bpf_deregister_track_event(struct bpf_if **driverp, |
2576 | void (*_fun)(struct bpf_if *, struct ifnet *, int, int)) | | 2575 | void (*_fun)(struct bpf_if *, struct ifnet *, int, int)) |
2577 | { | | 2576 | { |
2578 | struct bpf_if *bp; | | 2577 | struct bpf_if *bp; |
2579 | struct bpf_event_tracker *t = NULL; | | 2578 | struct bpf_event_tracker *t = NULL; |
2580 | int ret = ENOENT; | | 2579 | int ret = ENOENT; |
2581 | | | 2580 | |
2582 | mutex_enter(&bpf_mtx); | | 2581 | mutex_enter(&bpf_mtx); |
2583 | BPF_IFLIST_WRITER_FOREACH(bp) { | | 2582 | BPF_IFLIST_WRITER_FOREACH(bp) { |
2584 | if (bp->bif_driverp != driverp) | | 2583 | if (bp->bif_driverp != driverp) |
2585 | continue; | | 2584 | continue; |
2586 | SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) { | | 2585 | SLIST_FOREACH(t, &bp->bif_trackers, bet_entries) { |
2587 | if (t->bet_notify == _fun) { | | 2586 | if (t->bet_notify == _fun) { |
2588 | ret = 0; | | 2587 | ret = 0; |
2589 | break; | | 2588 | break; |
2590 | } | | 2589 | } |
2591 | } | | 2590 | } |
2592 | if (ret == 0) | | 2591 | if (ret == 0) |
2593 | break; | | 2592 | break; |
2594 | } | | 2593 | } |
2595 | if (ret == 0 && t && t->bet_notify == _fun) { | | 2594 | if (ret == 0 && t && t->bet_notify == _fun) { |
2596 | SLIST_REMOVE(&bp->bif_trackers, t, bpf_event_tracker, | | 2595 | SLIST_REMOVE(&bp->bif_trackers, t, bpf_event_tracker, |
2597 | bet_entries); | | 2596 | bet_entries); |
2598 | } | | 2597 | } |
2599 | mutex_exit(&bpf_mtx); | | 2598 | mutex_exit(&bpf_mtx); |
2600 | if (ret == 0) | | 2599 | if (ret == 0) |
2601 | kmem_free(t, sizeof(*t)); | | 2600 | kmem_free(t, sizeof(*t)); |
2602 | return ret; | | 2601 | return ret; |
2603 | } | | 2602 | } |
2604 | | | 2603 | |
2605 | struct bpf_ops bpf_ops_kernel = { | | 2604 | struct bpf_ops bpf_ops_kernel = { |
2606 | .bpf_attach = _bpfattach, | | 2605 | .bpf_attach = _bpfattach, |
2607 | .bpf_detach = _bpfdetach, | | 2606 | .bpf_detach = _bpfdetach, |
2608 | .bpf_change_type = _bpf_change_type, | | 2607 | .bpf_change_type = _bpf_change_type, |
2609 | .bpf_register_track_event = _bpf_register_track_event, | | 2608 | .bpf_register_track_event = _bpf_register_track_event, |
2610 | .bpf_deregister_track_event = _bpf_deregister_track_event, | | 2609 | .bpf_deregister_track_event = _bpf_deregister_track_event, |
2611 | | | 2610 | |
2612 | .bpf_mtap = _bpf_mtap, | | 2611 | .bpf_mtap = _bpf_mtap, |
2613 | .bpf_mtap2 = _bpf_mtap2, | | 2612 | .bpf_mtap2 = _bpf_mtap2, |
2614 | .bpf_mtap_af = _bpf_mtap_af, | | 2613 | .bpf_mtap_af = _bpf_mtap_af, |
2615 | .bpf_mtap_sl_in = _bpf_mtap_sl_in, | | 2614 | .bpf_mtap_sl_in = _bpf_mtap_sl_in, |
2616 | .bpf_mtap_sl_out = _bpf_mtap_sl_out, | | 2615 | .bpf_mtap_sl_out = _bpf_mtap_sl_out, |
2617 | | | 2616 | |
2618 | .bpf_mtap_softint = _bpf_mtap_softint, | | 2617 | .bpf_mtap_softint = _bpf_mtap_softint, |
2619 | .bpf_mtap_softint_init = _bpf_mtap_softint_init, | | 2618 | .bpf_mtap_softint_init = _bpf_mtap_softint_init, |
2620 | }; | | 2619 | }; |
2621 | | | 2620 | |
2622 | MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter"); | | 2621 | MODULE(MODULE_CLASS_DRIVER, bpf, "bpf_filter"); |
2623 | | | 2622 | |
2624 | static int | | 2623 | static int |
2625 | bpf_modcmd(modcmd_t cmd, void *arg) | | 2624 | bpf_modcmd(modcmd_t cmd, void *arg) |
2626 | { | | 2625 | { |
2627 | #ifdef _MODULE | | 2626 | #ifdef _MODULE |
2628 | devmajor_t bmajor, cmajor; | | 2627 | devmajor_t bmajor, cmajor; |
2629 | #endif | | 2628 | #endif |
2630 | int error = 0; | | 2629 | int error = 0; |
2631 | | | 2630 | |
2632 | switch (cmd) { | | 2631 | switch (cmd) { |
2633 | case MODULE_CMD_INIT: | | 2632 | case MODULE_CMD_INIT: |
2634 | bpf_init(); | | 2633 | bpf_init(); |
2635 | #ifdef _MODULE | | 2634 | #ifdef _MODULE |
2636 | bmajor = cmajor = NODEVMAJOR; | | 2635 | bmajor = cmajor = NODEVMAJOR; |
2637 | error = devsw_attach("bpf", NULL, &bmajor, | | 2636 | error = devsw_attach("bpf", NULL, &bmajor, |
2638 | &bpf_cdevsw, &cmajor); | | 2637 | &bpf_cdevsw, &cmajor); |
2639 | if (error) | | 2638 | if (error) |
2640 | break; | | 2639 | break; |
2641 | #endif | | 2640 | #endif |
2642 | | | 2641 | |
2643 | bpf_ops_handover_enter(&bpf_ops_kernel); | | 2642 | bpf_ops_handover_enter(&bpf_ops_kernel); |
2644 | atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); | | 2643 | atomic_swap_ptr(&bpf_ops, &bpf_ops_kernel); |
2645 | bpf_ops_handover_exit(); | | 2644 | bpf_ops_handover_exit(); |
2646 | break; | | 2645 | break; |
2647 | | | 2646 | |
2648 | case MODULE_CMD_FINI: | | 2647 | case MODULE_CMD_FINI: |
2649 | /* | | 2648 | /* |
2650 | * While there is no reference counting for bpf callers, | | 2649 | * While there is no reference counting for bpf callers, |
2651 | * unload could at least in theory be done similarly to | | 2650 | * unload could at least in theory be done similarly to |
2652 | * system call disestablishment. This should even be | | 2651 | * system call disestablishment. This should even be |
2653 | * a little simpler: | | 2652 | * a little simpler: |
2654 | * | | 2653 | * |
2655 | * 1) replace op vector with stubs | | 2654 | * 1) replace op vector with stubs |
2656 | * 2) post update to all cpus with xc | | 2655 | * 2) post update to all cpus with xc |
2657 | * 3) check that nobody is in bpf anymore | | 2656 | * 3) check that nobody is in bpf anymore |
2658 | * (it's doubtful we'd want something like l_sysent, | | 2657 | * (it's doubtful we'd want something like l_sysent, |
2659 | * but we could do something like *signed* percpu | | 2658 | * but we could do something like *signed* percpu |
2660 | * counters. if the sum is 0, we're good). | | 2659 | * counters. if the sum is 0, we're good). |
2661 | * 4) if fail, unroll changes | | 2660 | * 4) if fail, unroll changes |
2662 | * | | 2661 | * |
2663 | * NOTE: change won't be atomic to the outside. some | | 2662 | * NOTE: change won't be atomic to the outside. some |
2664 | * packets may be not captured even if unload is | | 2663 | * packets may be not captured even if unload is |
2665 | * not succesful. I think packet capture not working | | 2664 | * not succesful. I think packet capture not working |
2666 | * is a perfectly logical consequence of trying to | | 2665 | * is a perfectly logical consequence of trying to |
2667 | * disable packet capture. | | 2666 | * disable packet capture. |
2668 | */ | | 2667 | */ |
2669 | error = EOPNOTSUPP; | | 2668 | error = EOPNOTSUPP; |
2670 | break; | | 2669 | break; |
2671 | | | 2670 | |
2672 | default: | | 2671 | default: |
2673 | error = ENOTTY; | | 2672 | error = ENOTTY; |
2674 | break; | | 2673 | break; |
2675 | } | | 2674 | } |
2676 | | | 2675 | |
2677 | return error; | | 2676 | return error; |
2678 | } | | 2677 | } |