| @@ -1,2433 +1,2433 @@ | | | @@ -1,2433 +1,2433 @@ |
1 | /* $NetBSD: ix_txrx.c,v 1.79 2021/05/27 06:11:34 msaitoh Exp $ */ | | 1 | /* $NetBSD: ix_txrx.c,v 1.80 2021/07/07 08:32:51 msaitoh Exp $ */ |
2 | | | 2 | |
3 | /****************************************************************************** | | 3 | /****************************************************************************** |
4 | | | 4 | |
5 | Copyright (c) 2001-2017, Intel Corporation | | 5 | Copyright (c) 2001-2017, Intel Corporation |
6 | All rights reserved. | | 6 | All rights reserved. |
7 | | | 7 | |
8 | Redistribution and use in source and binary forms, with or without | | 8 | Redistribution and use in source and binary forms, with or without |
9 | modification, are permitted provided that the following conditions are met: | | 9 | modification, are permitted provided that the following conditions are met: |
10 | | | 10 | |
11 | 1. Redistributions of source code must retain the above copyright notice, | | 11 | 1. Redistributions of source code must retain the above copyright notice, |
12 | this list of conditions and the following disclaimer. | | 12 | this list of conditions and the following disclaimer. |
13 | | | 13 | |
14 | 2. Redistributions in binary form must reproduce the above copyright | | 14 | 2. Redistributions in binary form must reproduce the above copyright |
15 | notice, this list of conditions and the following disclaimer in the | | 15 | notice, this list of conditions and the following disclaimer in the |
16 | documentation and/or other materials provided with the distribution. | | 16 | documentation and/or other materials provided with the distribution. |
17 | | | 17 | |
18 | 3. Neither the name of the Intel Corporation nor the names of its | | 18 | 3. Neither the name of the Intel Corporation nor the names of its |
19 | contributors may be used to endorse or promote products derived from | | 19 | contributors may be used to endorse or promote products derived from |
20 | this software without specific prior written permission. | | 20 | this software without specific prior written permission. |
21 | | | 21 | |
22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
23 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | | 23 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
24 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | | 24 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
25 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | | 25 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE |
26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | | 26 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | | 27 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | | 28 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
29 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | | 29 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
30 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | | 30 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
31 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | | 31 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
32 | POSSIBILITY OF SUCH DAMAGE. | | 32 | POSSIBILITY OF SUCH DAMAGE. |
33 | | | 33 | |
34 | ******************************************************************************/ | | 34 | ******************************************************************************/ |
35 | /*$FreeBSD: head/sys/dev/ixgbe/ix_txrx.c 327031 2017-12-20 18:15:06Z erj $*/ | | 35 | /*$FreeBSD: head/sys/dev/ixgbe/ix_txrx.c 327031 2017-12-20 18:15:06Z erj $*/ |
36 | | | 36 | |
37 | /* | | 37 | /* |
38 | * Copyright (c) 2011 The NetBSD Foundation, Inc. | | 38 | * Copyright (c) 2011 The NetBSD Foundation, Inc. |
39 | * All rights reserved. | | 39 | * All rights reserved. |
40 | * | | 40 | * |
41 | * This code is derived from software contributed to The NetBSD Foundation | | 41 | * This code is derived from software contributed to The NetBSD Foundation |
42 | * by Coyote Point Systems, Inc. | | 42 | * by Coyote Point Systems, Inc. |
43 | * | | 43 | * |
44 | * Redistribution and use in source and binary forms, with or without | | 44 | * Redistribution and use in source and binary forms, with or without |
45 | * modification, are permitted provided that the following conditions | | 45 | * modification, are permitted provided that the following conditions |
46 | * are met: | | 46 | * are met: |
47 | * 1. Redistributions of source code must retain the above copyright | | 47 | * 1. Redistributions of source code must retain the above copyright |
48 | * notice, this list of conditions and the following disclaimer. | | 48 | * notice, this list of conditions and the following disclaimer. |
49 | * 2. Redistributions in binary form must reproduce the above copyright | | 49 | * 2. Redistributions in binary form must reproduce the above copyright |
50 | * notice, this list of conditions and the following disclaimer in the | | 50 | * notice, this list of conditions and the following disclaimer in the |
51 | * documentation and/or other materials provided with the distribution. | | 51 | * documentation and/or other materials provided with the distribution. |
52 | * | | 52 | * |
53 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS | | 53 | * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS |
54 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED | | 54 | * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
55 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | | 55 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
56 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS | | 56 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS |
57 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | | 57 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
58 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | | 58 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
59 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | | 59 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
60 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | | 60 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
61 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | | 61 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
62 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | | 62 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
63 | * POSSIBILITY OF SUCH DAMAGE. | | 63 | * POSSIBILITY OF SUCH DAMAGE. |
64 | */ | | 64 | */ |
65 | | | 65 | |
66 | #include <sys/cdefs.h> | | 66 | #include <sys/cdefs.h> |
67 | __KERNEL_RCSID(0, "$NetBSD: ix_txrx.c,v 1.79 2021/05/27 06:11:34 msaitoh Exp $"); | | 67 | __KERNEL_RCSID(0, "$NetBSD: ix_txrx.c,v 1.80 2021/07/07 08:32:51 msaitoh Exp $"); |
68 | | | 68 | |
69 | #include "opt_inet.h" | | 69 | #include "opt_inet.h" |
70 | #include "opt_inet6.h" | | 70 | #include "opt_inet6.h" |
71 | | | 71 | |
72 | #include "ixgbe.h" | | 72 | #include "ixgbe.h" |
73 | | | 73 | |
74 | /* | | 74 | /* |
75 | * HW RSC control: | | 75 | * HW RSC control: |
76 | * this feature only works with | | 76 | * this feature only works with |
77 | * IPv4, and only on 82599 and later. | | 77 | * IPv4, and only on 82599 and later. |
78 | * Also this will cause IP forwarding to | | 78 | * Also this will cause IP forwarding to |
79 | * fail and that can't be controlled by | | 79 | * fail and that can't be controlled by |
80 | * the stack as LRO can. For all these | | 80 | * the stack as LRO can. For all these |
81 | * reasons I've deemed it best to leave | | 81 | * reasons I've deemed it best to leave |
82 | * this off and not bother with a tuneable | | 82 | * this off and not bother with a tuneable |
83 | * interface, this would need to be compiled | | 83 | * interface, this would need to be compiled |
84 | * to enable. | | 84 | * to enable. |
85 | */ | | 85 | */ |
86 | static bool ixgbe_rsc_enable = FALSE; | | 86 | static bool ixgbe_rsc_enable = FALSE; |
87 | | | 87 | |
88 | /* | | 88 | /* |
89 | * For Flow Director: this is the | | 89 | * For Flow Director: this is the |
90 | * number of TX packets we sample | | 90 | * number of TX packets we sample |
91 | * for the filter pool, this means | | 91 | * for the filter pool, this means |
92 | * every 20th packet will be probed. | | 92 | * every 20th packet will be probed. |
93 | * | | 93 | * |
94 | * This feature can be disabled by | | 94 | * This feature can be disabled by |
95 | * setting this to 0. | | 95 | * setting this to 0. |
96 | */ | | 96 | */ |
97 | static int atr_sample_rate = 20; | | 97 | static int atr_sample_rate = 20; |
98 | | | 98 | |
99 | /************************************************************************ | | 99 | /************************************************************************ |
100 | * Local Function prototypes | | 100 | * Local Function prototypes |
101 | ************************************************************************/ | | 101 | ************************************************************************/ |
102 | static void ixgbe_setup_transmit_ring(struct tx_ring *); | | 102 | static void ixgbe_setup_transmit_ring(struct tx_ring *); |
103 | static void ixgbe_free_transmit_buffers(struct tx_ring *); | | 103 | static void ixgbe_free_transmit_buffers(struct tx_ring *); |
104 | static int ixgbe_setup_receive_ring(struct rx_ring *); | | 104 | static int ixgbe_setup_receive_ring(struct rx_ring *); |
105 | static void ixgbe_free_receive_buffers(struct rx_ring *); | | 105 | static void ixgbe_free_receive_buffers(struct rx_ring *); |
106 | static void ixgbe_rx_checksum(u32, struct mbuf *, u32, | | 106 | static void ixgbe_rx_checksum(u32, struct mbuf *, u32, |
107 | struct ixgbe_hw_stats *); | | 107 | struct ixgbe_hw_stats *); |
108 | static void ixgbe_refresh_mbufs(struct rx_ring *, int); | | 108 | static void ixgbe_refresh_mbufs(struct rx_ring *, int); |
109 | static void ixgbe_drain(struct ifnet *, struct tx_ring *); | | 109 | static void ixgbe_drain(struct ifnet *, struct tx_ring *); |
110 | static int ixgbe_xmit(struct tx_ring *, struct mbuf *); | | 110 | static int ixgbe_xmit(struct tx_ring *, struct mbuf *); |
111 | static int ixgbe_tx_ctx_setup(struct tx_ring *, | | 111 | static int ixgbe_tx_ctx_setup(struct tx_ring *, |
112 | struct mbuf *, u32 *, u32 *); | | 112 | struct mbuf *, u32 *, u32 *); |
113 | static int ixgbe_tso_setup(struct tx_ring *, | | 113 | static int ixgbe_tso_setup(struct tx_ring *, |
114 | struct mbuf *, u32 *, u32 *); | | 114 | struct mbuf *, u32 *, u32 *); |
115 | static __inline void ixgbe_rx_discard(struct rx_ring *, int); | | 115 | static __inline void ixgbe_rx_discard(struct rx_ring *, int); |
116 | static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *, | | 116 | static __inline void ixgbe_rx_input(struct rx_ring *, struct ifnet *, |
117 | struct mbuf *, u32); | | 117 | struct mbuf *, u32); |
118 | static int ixgbe_dma_malloc(struct adapter *, bus_size_t, | | 118 | static int ixgbe_dma_malloc(struct adapter *, bus_size_t, |
119 | struct ixgbe_dma_alloc *, int); | | 119 | struct ixgbe_dma_alloc *, int); |
120 | static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); | | 120 | static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); |
121 | | | 121 | |
122 | static void ixgbe_setup_hw_rsc(struct rx_ring *); | | 122 | static void ixgbe_setup_hw_rsc(struct rx_ring *); |
123 | | | 123 | |
124 | /************************************************************************ | | 124 | /************************************************************************ |
125 | * ixgbe_legacy_start_locked - Transmit entry point | | 125 | * ixgbe_legacy_start_locked - Transmit entry point |
126 | * | | 126 | * |
127 | * Called by the stack to initiate a transmit. | | 127 | * Called by the stack to initiate a transmit. |
128 | * The driver will remain in this routine as long as there are | | 128 | * The driver will remain in this routine as long as there are |
129 | * packets to transmit and transmit resources are available. | | 129 | * packets to transmit and transmit resources are available. |
130 | * In case resources are not available, the stack is notified | | 130 | * In case resources are not available, the stack is notified |
131 | * and the packet is requeued. | | 131 | * and the packet is requeued. |
132 | ************************************************************************/ | | 132 | ************************************************************************/ |
133 | int | | 133 | int |
134 | ixgbe_legacy_start_locked(struct ifnet *ifp, struct tx_ring *txr) | | 134 | ixgbe_legacy_start_locked(struct ifnet *ifp, struct tx_ring *txr) |
135 | { | | 135 | { |
136 | int rc; | | 136 | int rc; |
137 | struct mbuf *m_head; | | 137 | struct mbuf *m_head; |
138 | struct adapter *adapter = txr->adapter; | | 138 | struct adapter *adapter = txr->adapter; |
139 | | | 139 | |
140 | IXGBE_TX_LOCK_ASSERT(txr); | | 140 | IXGBE_TX_LOCK_ASSERT(txr); |
141 | | | 141 | |
142 | if (adapter->link_active != LINK_STATE_UP) { | | 142 | if (adapter->link_active != LINK_STATE_UP) { |
143 | /* | | 143 | /* |
144 | * discard all packets buffered in IFQ to avoid | | 144 | * discard all packets buffered in IFQ to avoid |
145 | * sending old packets at next link up timing. | | 145 | * sending old packets at next link up timing. |
146 | */ | | 146 | */ |
147 | ixgbe_drain(ifp, txr); | | 147 | ixgbe_drain(ifp, txr); |
148 | return (ENETDOWN); | | 148 | return (ENETDOWN); |
149 | } | | 149 | } |
150 | if ((ifp->if_flags & IFF_RUNNING) == 0) | | 150 | if ((ifp->if_flags & IFF_RUNNING) == 0) |
151 | return (ENETDOWN); | | 151 | return (ENETDOWN); |
152 | if (txr->txr_no_space) | | 152 | if (txr->txr_no_space) |
153 | return (ENETDOWN); | | 153 | return (ENETDOWN); |
154 | | | 154 | |
155 | while (!IFQ_IS_EMPTY(&ifp->if_snd)) { | | 155 | while (!IFQ_IS_EMPTY(&ifp->if_snd)) { |
156 | if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) | | 156 | if (txr->tx_avail <= IXGBE_QUEUE_MIN_FREE) |
157 | break; | | 157 | break; |
158 | | | 158 | |
159 | IFQ_POLL(&ifp->if_snd, m_head); | | 159 | IFQ_POLL(&ifp->if_snd, m_head); |
160 | if (m_head == NULL) | | 160 | if (m_head == NULL) |
161 | break; | | 161 | break; |
162 | | | 162 | |
163 | if ((rc = ixgbe_xmit(txr, m_head)) == EAGAIN) { | | 163 | if ((rc = ixgbe_xmit(txr, m_head)) == EAGAIN) { |
164 | break; | | 164 | break; |
165 | } | | 165 | } |
166 | IFQ_DEQUEUE(&ifp->if_snd, m_head); | | 166 | IFQ_DEQUEUE(&ifp->if_snd, m_head); |
167 | if (rc != 0) { | | 167 | if (rc != 0) { |
168 | m_freem(m_head); | | 168 | m_freem(m_head); |
169 | continue; | | 169 | continue; |
170 | } | | 170 | } |
171 | | | 171 | |
172 | /* Send a copy of the frame to the BPF listener */ | | 172 | /* Send a copy of the frame to the BPF listener */ |
173 | bpf_mtap(ifp, m_head, BPF_D_OUT); | | 173 | bpf_mtap(ifp, m_head, BPF_D_OUT); |
174 | } | | 174 | } |
175 | | | 175 | |
176 | return IXGBE_SUCCESS; | | 176 | return IXGBE_SUCCESS; |
177 | } /* ixgbe_legacy_start_locked */ | | 177 | } /* ixgbe_legacy_start_locked */ |
178 | | | 178 | |
179 | /************************************************************************ | | 179 | /************************************************************************ |
180 | * ixgbe_legacy_start | | 180 | * ixgbe_legacy_start |
181 | * | | 181 | * |
182 | * Called by the stack, this always uses the first tx ring, | | 182 | * Called by the stack, this always uses the first tx ring, |
183 | * and should not be used with multiqueue tx enabled. | | 183 | * and should not be used with multiqueue tx enabled. |
184 | ************************************************************************/ | | 184 | ************************************************************************/ |
185 | void | | 185 | void |
186 | ixgbe_legacy_start(struct ifnet *ifp) | | 186 | ixgbe_legacy_start(struct ifnet *ifp) |
187 | { | | 187 | { |
188 | struct adapter *adapter = ifp->if_softc; | | 188 | struct adapter *adapter = ifp->if_softc; |
189 | struct tx_ring *txr = adapter->tx_rings; | | 189 | struct tx_ring *txr = adapter->tx_rings; |
190 | | | 190 | |
191 | if (ifp->if_flags & IFF_RUNNING) { | | 191 | if (ifp->if_flags & IFF_RUNNING) { |
192 | IXGBE_TX_LOCK(txr); | | 192 | IXGBE_TX_LOCK(txr); |
193 | ixgbe_legacy_start_locked(ifp, txr); | | 193 | ixgbe_legacy_start_locked(ifp, txr); |
194 | IXGBE_TX_UNLOCK(txr); | | 194 | IXGBE_TX_UNLOCK(txr); |
195 | } | | 195 | } |
196 | } /* ixgbe_legacy_start */ | | 196 | } /* ixgbe_legacy_start */ |
197 | | | 197 | |
198 | /************************************************************************ | | 198 | /************************************************************************ |
199 | * ixgbe_mq_start - Multiqueue Transmit Entry Point | | 199 | * ixgbe_mq_start - Multiqueue Transmit Entry Point |
200 | * | | 200 | * |
201 | * (if_transmit function) | | 201 | * (if_transmit function) |
202 | ************************************************************************/ | | 202 | ************************************************************************/ |
203 | int | | 203 | int |
204 | ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) | | 204 | ixgbe_mq_start(struct ifnet *ifp, struct mbuf *m) |
205 | { | | 205 | { |
206 | struct adapter *adapter = ifp->if_softc; | | 206 | struct adapter *adapter = ifp->if_softc; |
207 | struct tx_ring *txr; | | 207 | struct tx_ring *txr; |
208 | int i; | | 208 | int i; |
209 | #ifdef RSS | | 209 | #ifdef RSS |
210 | uint32_t bucket_id; | | 210 | uint32_t bucket_id; |
211 | #endif | | 211 | #endif |
212 | | | 212 | |
213 | /* | | 213 | /* |
214 | * When doing RSS, map it to the same outbound queue | | 214 | * When doing RSS, map it to the same outbound queue |
215 | * as the incoming flow would be mapped to. | | 215 | * as the incoming flow would be mapped to. |
216 | * | | 216 | * |
217 | * If everything is setup correctly, it should be the | | 217 | * If everything is setup correctly, it should be the |
218 | * same bucket that the current CPU we're on is. | | 218 | * same bucket that the current CPU we're on is. |
219 | */ | | 219 | */ |
220 | #ifdef RSS | | 220 | #ifdef RSS |
221 | if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { | | 221 | if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) { |
222 | if ((adapter->feat_en & IXGBE_FEATURE_RSS) && | | 222 | if ((adapter->feat_en & IXGBE_FEATURE_RSS) && |
223 | (rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), | | 223 | (rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), |
224 | &bucket_id) == 0)) { | | 224 | &bucket_id) == 0)) { |
225 | i = bucket_id % adapter->num_queues; | | 225 | i = bucket_id % adapter->num_queues; |
226 | #ifdef IXGBE_DEBUG | | 226 | #ifdef IXGBE_DEBUG |
227 | if (bucket_id > adapter->num_queues) | | 227 | if (bucket_id > adapter->num_queues) |
228 | if_printf(ifp, | | 228 | if_printf(ifp, |
229 | "bucket_id (%d) > num_queues (%d)\n", | | 229 | "bucket_id (%d) > num_queues (%d)\n", |
230 | bucket_id, adapter->num_queues); | | 230 | bucket_id, adapter->num_queues); |
231 | #endif | | 231 | #endif |
232 | } else | | 232 | } else |
233 | i = m->m_pkthdr.flowid % adapter->num_queues; | | 233 | i = m->m_pkthdr.flowid % adapter->num_queues; |
234 | } else | | 234 | } else |
235 | #endif /* 0 */ | | 235 | #endif /* 0 */ |
236 | i = (cpu_index(curcpu()) % ncpu) % adapter->num_queues; | | 236 | i = (cpu_index(curcpu()) % ncpu) % adapter->num_queues; |
237 | | | 237 | |
238 | /* Check for a hung queue and pick alternative */ | | 238 | /* Check for a hung queue and pick alternative */ |
239 | if (((1ULL << i) & adapter->active_queues) == 0) | | 239 | if (((1ULL << i) & adapter->active_queues) == 0) |
240 | i = ffs64(adapter->active_queues); | | 240 | i = ffs64(adapter->active_queues); |
241 | | | 241 | |
242 | txr = &adapter->tx_rings[i]; | | 242 | txr = &adapter->tx_rings[i]; |
243 | | | 243 | |
244 | if (__predict_false(!pcq_put(txr->txr_interq, m))) { | | 244 | if (__predict_false(!pcq_put(txr->txr_interq, m))) { |
245 | m_freem(m); | | 245 | m_freem(m); |
246 | txr->pcq_drops.ev_count++; | | 246 | txr->pcq_drops.ev_count++; |
247 | return ENOBUFS; | | 247 | return ENOBUFS; |
248 | } | | 248 | } |
249 | if (IXGBE_TX_TRYLOCK(txr)) { | | 249 | if (IXGBE_TX_TRYLOCK(txr)) { |
250 | ixgbe_mq_start_locked(ifp, txr); | | 250 | ixgbe_mq_start_locked(ifp, txr); |
251 | IXGBE_TX_UNLOCK(txr); | | 251 | IXGBE_TX_UNLOCK(txr); |
252 | } else { | | 252 | } else { |
253 | if (adapter->txrx_use_workqueue) { | | 253 | if (adapter->txrx_use_workqueue) { |
254 | u_int *enqueued; | | 254 | u_int *enqueued; |
255 | | | 255 | |
256 | /* | | 256 | /* |
257 | * This function itself is not called in interrupt | | 257 | * This function itself is not called in interrupt |
258 | * context, however it can be called in fast softint | | 258 | * context, however it can be called in fast softint |
259 | * context right after receiving forwarding packets. | | 259 | * context right after receiving forwarding packets. |
260 | * So, it is required to protect workqueue from twice | | 260 | * So, it is required to protect workqueue from twice |
261 | * enqueuing when the machine uses both spontaneous | | 261 | * enqueuing when the machine uses both spontaneous |
262 | * packets and forwarding packets. | | 262 | * packets and forwarding packets. |
263 | */ | | 263 | */ |
264 | enqueued = percpu_getref(adapter->txr_wq_enqueued); | | 264 | enqueued = percpu_getref(adapter->txr_wq_enqueued); |
265 | if (*enqueued == 0) { | | 265 | if (*enqueued == 0) { |
266 | *enqueued = 1; | | 266 | *enqueued = 1; |
267 | percpu_putref(adapter->txr_wq_enqueued); | | 267 | percpu_putref(adapter->txr_wq_enqueued); |
268 | workqueue_enqueue(adapter->txr_wq, | | 268 | workqueue_enqueue(adapter->txr_wq, |
269 | &txr->wq_cookie, curcpu()); | | 269 | &txr->wq_cookie, curcpu()); |
270 | } else | | 270 | } else |
271 | percpu_putref(adapter->txr_wq_enqueued); | | 271 | percpu_putref(adapter->txr_wq_enqueued); |
272 | } else { | | 272 | } else { |
273 | kpreempt_disable(); | | 273 | kpreempt_disable(); |
274 | softint_schedule(txr->txr_si); | | 274 | softint_schedule(txr->txr_si); |
275 | kpreempt_enable(); | | 275 | kpreempt_enable(); |
276 | } | | 276 | } |
277 | } | | 277 | } |
278 | | | 278 | |
279 | return (0); | | 279 | return (0); |
280 | } /* ixgbe_mq_start */ | | 280 | } /* ixgbe_mq_start */ |
281 | | | 281 | |
282 | /************************************************************************ | | 282 | /************************************************************************ |
283 | * ixgbe_mq_start_locked | | 283 | * ixgbe_mq_start_locked |
284 | ************************************************************************/ | | 284 | ************************************************************************/ |
285 | int | | 285 | int |
286 | ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) | | 286 | ixgbe_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr) |
287 | { | | 287 | { |
288 | struct mbuf *next; | | 288 | struct mbuf *next; |
289 | int enqueued = 0, err = 0; | | 289 | int enqueued = 0, err = 0; |
290 | | | 290 | |
291 | if (txr->adapter->link_active != LINK_STATE_UP) { | | 291 | if (txr->adapter->link_active != LINK_STATE_UP) { |
292 | /* | | 292 | /* |
293 | * discard all packets buffered in txr_interq to avoid | | 293 | * discard all packets buffered in txr_interq to avoid |
294 | * sending old packets at next link up timing. | | 294 | * sending old packets at next link up timing. |
295 | */ | | 295 | */ |
296 | ixgbe_drain(ifp, txr); | | 296 | ixgbe_drain(ifp, txr); |
297 | return (ENETDOWN); | | 297 | return (ENETDOWN); |
298 | } | | 298 | } |
299 | if ((ifp->if_flags & IFF_RUNNING) == 0) | | 299 | if ((ifp->if_flags & IFF_RUNNING) == 0) |
300 | return (ENETDOWN); | | 300 | return (ENETDOWN); |
301 | if (txr->txr_no_space) | | 301 | if (txr->txr_no_space) |
302 | return (ENETDOWN); | | 302 | return (ENETDOWN); |
303 | | | 303 | |
304 | /* Process the queue */ | | 304 | /* Process the queue */ |
305 | while ((next = pcq_get(txr->txr_interq)) != NULL) { | | 305 | while ((next = pcq_get(txr->txr_interq)) != NULL) { |
306 | if ((err = ixgbe_xmit(txr, next)) != 0) { | | 306 | if ((err = ixgbe_xmit(txr, next)) != 0) { |
307 | m_freem(next); | | 307 | m_freem(next); |
308 | /* All errors are counted in ixgbe_xmit() */ | | 308 | /* All errors are counted in ixgbe_xmit() */ |
309 | break; | | 309 | break; |
310 | } | | 310 | } |
311 | enqueued++; | | 311 | enqueued++; |
312 | #if __FreeBSD_version >= 1100036 | | 312 | #if __FreeBSD_version >= 1100036 |
313 | /* | | 313 | /* |
314 | * Since we're looking at the tx ring, we can check | | 314 | * Since we're looking at the tx ring, we can check |
315 | * to see if we're a VF by examing our tail register | | 315 | * to see if we're a VF by examing our tail register |
316 | * address. | | 316 | * address. |
317 | */ | | 317 | */ |
318 | if ((txr->adapter->feat_en & IXGBE_FEATURE_VF) && | | 318 | if ((txr->adapter->feat_en & IXGBE_FEATURE_VF) && |
319 | (next->m_flags & M_MCAST)) | | 319 | (next->m_flags & M_MCAST)) |
320 | if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); | | 320 | if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); |
321 | #endif | | 321 | #endif |
322 | /* Send a copy of the frame to the BPF listener */ | | 322 | /* Send a copy of the frame to the BPF listener */ |
323 | bpf_mtap(ifp, next, BPF_D_OUT); | | 323 | bpf_mtap(ifp, next, BPF_D_OUT); |
324 | if ((ifp->if_flags & IFF_RUNNING) == 0) | | 324 | if ((ifp->if_flags & IFF_RUNNING) == 0) |
325 | break; | | 325 | break; |
326 | } | | 326 | } |
327 | | | 327 | |
328 | if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD(txr->adapter)) | | 328 | if (txr->tx_avail < IXGBE_TX_CLEANUP_THRESHOLD(txr->adapter)) |
329 | ixgbe_txeof(txr); | | 329 | ixgbe_txeof(txr); |
330 | | | 330 | |
331 | return (err); | | 331 | return (err); |
332 | } /* ixgbe_mq_start_locked */ | | 332 | } /* ixgbe_mq_start_locked */ |
333 | | | 333 | |
334 | /************************************************************************ | | 334 | /************************************************************************ |
335 | * ixgbe_deferred_mq_start | | 335 | * ixgbe_deferred_mq_start |
336 | * | | 336 | * |
337 | * Called from a softint and workqueue (indirectly) to drain queued | | 337 | * Called from a softint and workqueue (indirectly) to drain queued |
338 | * transmit packets. | | 338 | * transmit packets. |
339 | ************************************************************************/ | | 339 | ************************************************************************/ |
340 | void | | 340 | void |
341 | ixgbe_deferred_mq_start(void *arg) | | 341 | ixgbe_deferred_mq_start(void *arg) |
342 | { | | 342 | { |
343 | struct tx_ring *txr = arg; | | 343 | struct tx_ring *txr = arg; |
344 | struct adapter *adapter = txr->adapter; | | 344 | struct adapter *adapter = txr->adapter; |
345 | struct ifnet *ifp = adapter->ifp; | | 345 | struct ifnet *ifp = adapter->ifp; |
346 | | | 346 | |
347 | IXGBE_TX_LOCK(txr); | | 347 | IXGBE_TX_LOCK(txr); |
348 | if (pcq_peek(txr->txr_interq) != NULL) | | 348 | if (pcq_peek(txr->txr_interq) != NULL) |
349 | ixgbe_mq_start_locked(ifp, txr); | | 349 | ixgbe_mq_start_locked(ifp, txr); |
350 | IXGBE_TX_UNLOCK(txr); | | 350 | IXGBE_TX_UNLOCK(txr); |
351 | } /* ixgbe_deferred_mq_start */ | | 351 | } /* ixgbe_deferred_mq_start */ |
352 | | | 352 | |
353 | /************************************************************************ | | 353 | /************************************************************************ |
354 | * ixgbe_deferred_mq_start_work | | 354 | * ixgbe_deferred_mq_start_work |
355 | * | | 355 | * |
356 | * Called from a workqueue to drain queued transmit packets. | | 356 | * Called from a workqueue to drain queued transmit packets. |
357 | ************************************************************************/ | | 357 | ************************************************************************/ |
358 | void | | 358 | void |
359 | ixgbe_deferred_mq_start_work(struct work *wk, void *arg) | | 359 | ixgbe_deferred_mq_start_work(struct work *wk, void *arg) |
360 | { | | 360 | { |
361 | struct tx_ring *txr = container_of(wk, struct tx_ring, wq_cookie); | | 361 | struct tx_ring *txr = container_of(wk, struct tx_ring, wq_cookie); |
362 | struct adapter *adapter = txr->adapter; | | 362 | struct adapter *adapter = txr->adapter; |
363 | u_int *enqueued = percpu_getref(adapter->txr_wq_enqueued); | | 363 | u_int *enqueued = percpu_getref(adapter->txr_wq_enqueued); |
364 | *enqueued = 0; | | 364 | *enqueued = 0; |
365 | percpu_putref(adapter->txr_wq_enqueued); | | 365 | percpu_putref(adapter->txr_wq_enqueued); |
366 | | | 366 | |
367 | ixgbe_deferred_mq_start(txr); | | 367 | ixgbe_deferred_mq_start(txr); |
368 | } /* ixgbe_deferred_mq_start */ | | 368 | } /* ixgbe_deferred_mq_start */ |
369 | | | 369 | |
370 | /************************************************************************ | | 370 | /************************************************************************ |
371 | * ixgbe_drain_all | | 371 | * ixgbe_drain_all |
372 | ************************************************************************/ | | 372 | ************************************************************************/ |
373 | void | | 373 | void |
374 | ixgbe_drain_all(struct adapter *adapter) | | 374 | ixgbe_drain_all(struct adapter *adapter) |
375 | { | | 375 | { |
376 | struct ifnet *ifp = adapter->ifp; | | 376 | struct ifnet *ifp = adapter->ifp; |
377 | struct ix_queue *que = adapter->queues; | | 377 | struct ix_queue *que = adapter->queues; |
378 | | | 378 | |
379 | for (int i = 0; i < adapter->num_queues; i++, que++) { | | 379 | for (int i = 0; i < adapter->num_queues; i++, que++) { |
380 | struct tx_ring *txr = que->txr; | | 380 | struct tx_ring *txr = que->txr; |
381 | | | 381 | |
382 | IXGBE_TX_LOCK(txr); | | 382 | IXGBE_TX_LOCK(txr); |
383 | ixgbe_drain(ifp, txr); | | 383 | ixgbe_drain(ifp, txr); |
384 | IXGBE_TX_UNLOCK(txr); | | 384 | IXGBE_TX_UNLOCK(txr); |
385 | } | | 385 | } |
386 | } | | 386 | } |
387 | | | 387 | |
388 | /************************************************************************ | | 388 | /************************************************************************ |
389 | * ixgbe_xmit | | 389 | * ixgbe_xmit |
390 | * | | 390 | * |
391 | * Maps the mbufs to tx descriptors, allowing the | | 391 | * Maps the mbufs to tx descriptors, allowing the |
392 | * TX engine to transmit the packets. | | 392 | * TX engine to transmit the packets. |
393 | * | | 393 | * |
394 | * Return 0 on success, positive on failure | | 394 | * Return 0 on success, positive on failure |
395 | ************************************************************************/ | | 395 | ************************************************************************/ |
396 | static int | | 396 | static int |
397 | ixgbe_xmit(struct tx_ring *txr, struct mbuf *m_head) | | 397 | ixgbe_xmit(struct tx_ring *txr, struct mbuf *m_head) |
398 | { | | 398 | { |
399 | struct adapter *adapter = txr->adapter; | | 399 | struct adapter *adapter = txr->adapter; |
400 | struct ixgbe_tx_buf *txbuf; | | 400 | struct ixgbe_tx_buf *txbuf; |
401 | union ixgbe_adv_tx_desc *txd = NULL; | | 401 | union ixgbe_adv_tx_desc *txd = NULL; |
402 | struct ifnet *ifp = adapter->ifp; | | 402 | struct ifnet *ifp = adapter->ifp; |
403 | int i, j, error; | | 403 | int i, j, error; |
404 | int first; | | 404 | int first; |
405 | u32 olinfo_status = 0, cmd_type_len; | | 405 | u32 olinfo_status = 0, cmd_type_len; |
406 | bool remap = TRUE; | | 406 | bool remap = TRUE; |
407 | bus_dmamap_t map; | | 407 | bus_dmamap_t map; |
408 | | | 408 | |
409 | /* Basic descriptor defines */ | | 409 | /* Basic descriptor defines */ |
410 | cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | | | 410 | cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | |
411 | IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); | | 411 | IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); |
412 | | | 412 | |
413 | if (vlan_has_tag(m_head)) | | 413 | if (vlan_has_tag(m_head)) |
414 | cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; | | 414 | cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; |
415 | | | 415 | |
416 | /* | | 416 | /* |
417 | * Important to capture the first descriptor | | 417 | * Important to capture the first descriptor |
418 | * used because it will contain the index of | | 418 | * used because it will contain the index of |
419 | * the one we tell the hardware to report back | | 419 | * the one we tell the hardware to report back |
420 | */ | | 420 | */ |
421 | first = txr->next_avail_desc; | | 421 | first = txr->next_avail_desc; |
422 | txbuf = &txr->tx_buffers[first]; | | 422 | txbuf = &txr->tx_buffers[first]; |
423 | map = txbuf->map; | | 423 | map = txbuf->map; |
424 | | | 424 | |
425 | /* | | 425 | /* |
426 | * Map the packet for DMA. | | 426 | * Map the packet for DMA. |
427 | */ | | 427 | */ |
428 | retry: | | 428 | retry: |
429 | error = bus_dmamap_load_mbuf(txr->txtag->dt_dmat, map, m_head, | | 429 | error = bus_dmamap_load_mbuf(txr->txtag->dt_dmat, map, m_head, |
430 | BUS_DMA_NOWAIT); | | 430 | BUS_DMA_NOWAIT); |
431 | | | 431 | |
432 | if (__predict_false(error)) { | | 432 | if (__predict_false(error)) { |
433 | struct mbuf *m; | | 433 | struct mbuf *m; |
434 | | | 434 | |
435 | switch (error) { | | 435 | switch (error) { |
436 | case EAGAIN: | | 436 | case EAGAIN: |
437 | txr->q_eagain_tx_dma_setup++; | | 437 | txr->q_eagain_tx_dma_setup++; |
438 | return EAGAIN; | | 438 | return EAGAIN; |
439 | case ENOMEM: | | 439 | case ENOMEM: |
440 | txr->q_enomem_tx_dma_setup++; | | 440 | txr->q_enomem_tx_dma_setup++; |
441 | return EAGAIN; | | 441 | return EAGAIN; |
442 | case EFBIG: | | 442 | case EFBIG: |
443 | /* Try it again? - one try */ | | 443 | /* Try it again? - one try */ |
444 | if (remap == TRUE) { | | 444 | if (remap == TRUE) { |
445 | remap = FALSE; | | 445 | remap = FALSE; |
446 | /* | | 446 | /* |
447 | * XXX: m_defrag will choke on | | 447 | * XXX: m_defrag will choke on |
448 | * non-MCLBYTES-sized clusters | | 448 | * non-MCLBYTES-sized clusters |
449 | */ | | 449 | */ |
450 | txr->q_efbig_tx_dma_setup++; | | 450 | txr->q_efbig_tx_dma_setup++; |
451 | m = m_defrag(m_head, M_NOWAIT); | | 451 | m = m_defrag(m_head, M_NOWAIT); |
452 | if (m == NULL) { | | 452 | if (m == NULL) { |
453 | txr->q_mbuf_defrag_failed++; | | 453 | txr->q_mbuf_defrag_failed++; |
454 | return ENOBUFS; | | 454 | return ENOBUFS; |
455 | } | | 455 | } |
456 | m_head = m; | | 456 | m_head = m; |
457 | goto retry; | | 457 | goto retry; |
458 | } else { | | 458 | } else { |
459 | txr->q_efbig2_tx_dma_setup++; | | 459 | txr->q_efbig2_tx_dma_setup++; |
460 | return error; | | 460 | return error; |
461 | } | | 461 | } |
462 | case EINVAL: | | 462 | case EINVAL: |
463 | txr->q_einval_tx_dma_setup++; | | 463 | txr->q_einval_tx_dma_setup++; |
464 | return error; | | 464 | return error; |
465 | default: | | 465 | default: |
466 | txr->q_other_tx_dma_setup++; | | 466 | txr->q_other_tx_dma_setup++; |
467 | return error; | | 467 | return error; |
468 | } | | 468 | } |
469 | } | | 469 | } |
470 | | | 470 | |
471 | /* Make certain there are enough descriptors */ | | 471 | /* Make certain there are enough descriptors */ |
472 | if (txr->tx_avail < (map->dm_nsegs + 2)) { | | 472 | if (txr->tx_avail < (map->dm_nsegs + 2)) { |
473 | txr->txr_no_space = true; | | 473 | txr->txr_no_space = true; |
474 | txr->no_desc_avail.ev_count++; | | 474 | txr->no_desc_avail.ev_count++; |
475 | ixgbe_dmamap_unload(txr->txtag, txbuf->map); | | 475 | ixgbe_dmamap_unload(txr->txtag, txbuf->map); |
476 | return EAGAIN; | | 476 | return EAGAIN; |
477 | } | | 477 | } |
478 | | | 478 | |
479 | /* | | 479 | /* |
480 | * Set up the appropriate offload context | | 480 | * Set up the appropriate offload context |
481 | * this will consume the first descriptor | | 481 | * this will consume the first descriptor |
482 | */ | | 482 | */ |
483 | error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); | | 483 | error = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); |
484 | if (__predict_false(error)) { | | 484 | if (__predict_false(error)) { |
485 | return (error); | | 485 | return (error); |
486 | } | | 486 | } |
487 | | | 487 | |
488 | #ifdef IXGBE_FDIR | | 488 | #ifdef IXGBE_FDIR |
489 | /* Do the flow director magic */ | | 489 | /* Do the flow director magic */ |
490 | if ((adapter->feat_en & IXGBE_FEATURE_FDIR) && | | 490 | if ((adapter->feat_en & IXGBE_FEATURE_FDIR) && |
491 | (txr->atr_sample) && (!adapter->fdir_reinit)) { | | 491 | (txr->atr_sample) && (!adapter->fdir_reinit)) { |
492 | ++txr->atr_count; | | 492 | ++txr->atr_count; |
493 | if (txr->atr_count >= atr_sample_rate) { | | 493 | if (txr->atr_count >= atr_sample_rate) { |
494 | ixgbe_atr(txr, m_head); | | 494 | ixgbe_atr(txr, m_head); |
495 | txr->atr_count = 0; | | 495 | txr->atr_count = 0; |
496 | } | | 496 | } |
497 | } | | 497 | } |
498 | #endif | | 498 | #endif |
499 | | | 499 | |
500 | olinfo_status |= IXGBE_ADVTXD_CC; | | 500 | olinfo_status |= IXGBE_ADVTXD_CC; |
501 | i = txr->next_avail_desc; | | 501 | i = txr->next_avail_desc; |
502 | for (j = 0; j < map->dm_nsegs; j++) { | | 502 | for (j = 0; j < map->dm_nsegs; j++) { |
503 | bus_size_t seglen; | | 503 | bus_size_t seglen; |
504 | uint64_t segaddr; | | 504 | uint64_t segaddr; |
505 | | | 505 | |
506 | txbuf = &txr->tx_buffers[i]; | | 506 | txbuf = &txr->tx_buffers[i]; |
507 | txd = &txr->tx_base[i]; | | 507 | txd = &txr->tx_base[i]; |
508 | seglen = map->dm_segs[j].ds_len; | | 508 | seglen = map->dm_segs[j].ds_len; |
509 | segaddr = htole64(map->dm_segs[j].ds_addr); | | 509 | segaddr = htole64(map->dm_segs[j].ds_addr); |
510 | | | 510 | |
511 | txd->read.buffer_addr = segaddr; | | 511 | txd->read.buffer_addr = segaddr; |
512 | txd->read.cmd_type_len = htole32(cmd_type_len | seglen); | | 512 | txd->read.cmd_type_len = htole32(cmd_type_len | seglen); |
513 | txd->read.olinfo_status = htole32(olinfo_status); | | 513 | txd->read.olinfo_status = htole32(olinfo_status); |
514 | | | 514 | |
515 | if (++i == txr->num_desc) | | 515 | if (++i == txr->num_desc) |
516 | i = 0; | | 516 | i = 0; |
517 | } | | 517 | } |
518 | | | 518 | |
519 | txd->read.cmd_type_len |= htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); | | 519 | txd->read.cmd_type_len |= htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); |
520 | txr->tx_avail -= map->dm_nsegs; | | 520 | txr->tx_avail -= map->dm_nsegs; |
521 | txr->next_avail_desc = i; | | 521 | txr->next_avail_desc = i; |
522 | | | 522 | |
523 | txbuf->m_head = m_head; | | 523 | txbuf->m_head = m_head; |
524 | /* | | 524 | /* |
525 | * Here we swap the map so the last descriptor, | | 525 | * Here we swap the map so the last descriptor, |
526 | * which gets the completion interrupt has the | | 526 | * which gets the completion interrupt has the |
527 | * real map, and the first descriptor gets the | | 527 | * real map, and the first descriptor gets the |
528 | * unused map from this descriptor. | | 528 | * unused map from this descriptor. |
529 | */ | | 529 | */ |
530 | txr->tx_buffers[first].map = txbuf->map; | | 530 | txr->tx_buffers[first].map = txbuf->map; |
531 | txbuf->map = map; | | 531 | txbuf->map = map; |
532 | bus_dmamap_sync(txr->txtag->dt_dmat, map, 0, m_head->m_pkthdr.len, | | 532 | bus_dmamap_sync(txr->txtag->dt_dmat, map, 0, m_head->m_pkthdr.len, |
533 | BUS_DMASYNC_PREWRITE); | | 533 | BUS_DMASYNC_PREWRITE); |
534 | | | 534 | |
535 | /* Set the EOP descriptor that will be marked done */ | | 535 | /* Set the EOP descriptor that will be marked done */ |
536 | txbuf = &txr->tx_buffers[first]; | | 536 | txbuf = &txr->tx_buffers[first]; |
537 | txbuf->eop = txd; | | 537 | txbuf->eop = txd; |
538 | | | 538 | |
539 | ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, | | 539 | ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, |
540 | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | | 540 | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); |
541 | /* | | 541 | /* |
542 | * Advance the Transmit Descriptor Tail (Tdt), this tells the | | 542 | * Advance the Transmit Descriptor Tail (Tdt), this tells the |
543 | * hardware that this frame is available to transmit. | | 543 | * hardware that this frame is available to transmit. |
544 | */ | | 544 | */ |
545 | ++txr->total_packets.ev_count; | | 545 | ++txr->total_packets.ev_count; |
546 | IXGBE_WRITE_REG(&adapter->hw, txr->tail, i); | | 546 | IXGBE_WRITE_REG(&adapter->hw, txr->tail, i); |
547 | | | 547 | |
548 | net_stat_ref_t nsr = IF_STAT_GETREF(ifp); | | 548 | net_stat_ref_t nsr = IF_STAT_GETREF(ifp); |
549 | if_statadd_ref(nsr, if_obytes, m_head->m_pkthdr.len); | | 549 | if_statadd_ref(nsr, if_obytes, m_head->m_pkthdr.len); |
550 | if (m_head->m_flags & M_MCAST) | | 550 | if (m_head->m_flags & M_MCAST) |
551 | if_statinc_ref(nsr, if_omcasts); | | 551 | if_statinc_ref(nsr, if_omcasts); |
552 | IF_STAT_PUTREF(ifp); | | 552 | IF_STAT_PUTREF(ifp); |
553 | | | 553 | |
554 | /* Mark queue as having work */ | | 554 | /* Mark queue as having work */ |
555 | if (txr->busy == 0) | | 555 | if (txr->busy == 0) |
556 | txr->busy = 1; | | 556 | txr->busy = 1; |
557 | | | 557 | |
558 | return (0); | | 558 | return (0); |
559 | } /* ixgbe_xmit */ | | 559 | } /* ixgbe_xmit */ |
560 | | | 560 | |
561 | /************************************************************************ | | 561 | /************************************************************************ |
562 | * ixgbe_drain | | 562 | * ixgbe_drain |
563 | ************************************************************************/ | | 563 | ************************************************************************/ |
564 | static void | | 564 | static void |
565 | ixgbe_drain(struct ifnet *ifp, struct tx_ring *txr) | | 565 | ixgbe_drain(struct ifnet *ifp, struct tx_ring *txr) |
566 | { | | 566 | { |
567 | struct mbuf *m; | | 567 | struct mbuf *m; |
568 | | | 568 | |
569 | IXGBE_TX_LOCK_ASSERT(txr); | | 569 | IXGBE_TX_LOCK_ASSERT(txr); |
570 | | | 570 | |
571 | if (txr->me == 0) { | | 571 | if (txr->me == 0) { |
572 | while (!IFQ_IS_EMPTY(&ifp->if_snd)) { | | 572 | while (!IFQ_IS_EMPTY(&ifp->if_snd)) { |
573 | IFQ_DEQUEUE(&ifp->if_snd, m); | | 573 | IFQ_DEQUEUE(&ifp->if_snd, m); |
574 | m_freem(m); | | 574 | m_freem(m); |
575 | IF_DROP(&ifp->if_snd); | | 575 | IF_DROP(&ifp->if_snd); |
576 | } | | 576 | } |
577 | } | | 577 | } |
578 | | | 578 | |
579 | while ((m = pcq_get(txr->txr_interq)) != NULL) { | | 579 | while ((m = pcq_get(txr->txr_interq)) != NULL) { |
580 | m_freem(m); | | 580 | m_freem(m); |
581 | txr->pcq_drops.ev_count++; | | 581 | txr->pcq_drops.ev_count++; |
582 | } | | 582 | } |
583 | } | | 583 | } |
584 | | | 584 | |
585 | /************************************************************************ | | 585 | /************************************************************************ |
586 | * ixgbe_allocate_transmit_buffers | | 586 | * ixgbe_allocate_transmit_buffers |
587 | * | | 587 | * |
588 | * Allocate memory for tx_buffer structures. The tx_buffer stores all | | 588 | * Allocate memory for tx_buffer structures. The tx_buffer stores all |
589 | * the information needed to transmit a packet on the wire. This is | | 589 | * the information needed to transmit a packet on the wire. This is |
590 | * called only once at attach, setup is done every reset. | | 590 | * called only once at attach, setup is done every reset. |
591 | ************************************************************************/ | | 591 | ************************************************************************/ |
592 | static int | | 592 | static int |
593 | ixgbe_allocate_transmit_buffers(struct tx_ring *txr) | | 593 | ixgbe_allocate_transmit_buffers(struct tx_ring *txr) |
594 | { | | 594 | { |
595 | struct adapter *adapter = txr->adapter; | | 595 | struct adapter *adapter = txr->adapter; |
596 | device_t dev = adapter->dev; | | 596 | device_t dev = adapter->dev; |
597 | struct ixgbe_tx_buf *txbuf; | | 597 | struct ixgbe_tx_buf *txbuf; |
598 | int error, i; | | 598 | int error, i; |
599 | | | 599 | |
600 | /* | | 600 | /* |
601 | * Setup DMA descriptor areas. | | 601 | * Setup DMA descriptor areas. |
602 | */ | | 602 | */ |
603 | error = ixgbe_dma_tag_create( | | 603 | error = ixgbe_dma_tag_create( |
604 | /* parent */ adapter->osdep.dmat, | | 604 | /* parent */ adapter->osdep.dmat, |
605 | /* alignment */ 1, | | 605 | /* alignment */ 1, |
606 | /* bounds */ 0, | | 606 | /* bounds */ 0, |
607 | /* maxsize */ IXGBE_TSO_SIZE, | | 607 | /* maxsize */ IXGBE_TSO_SIZE, |
608 | /* nsegments */ adapter->num_segs, | | 608 | /* nsegments */ adapter->num_segs, |
609 | /* maxsegsize */ PAGE_SIZE, | | 609 | /* maxsegsize */ PAGE_SIZE, |
610 | /* flags */ 0, | | 610 | /* flags */ 0, |
611 | &txr->txtag); | | 611 | &txr->txtag); |
612 | if (error != 0) { | | 612 | if (error != 0) { |
613 | aprint_error_dev(dev,"Unable to allocate TX DMA tag\n"); | | 613 | aprint_error_dev(dev,"Unable to allocate TX DMA tag\n"); |
614 | goto fail; | | 614 | goto fail; |
615 | } | | 615 | } |
616 | | | 616 | |
617 | txr->tx_buffers = malloc(sizeof(struct ixgbe_tx_buf) * | | 617 | txr->tx_buffers = malloc(sizeof(struct ixgbe_tx_buf) * |
618 | adapter->num_tx_desc, M_DEVBUF, M_WAITOK | M_ZERO); | | 618 | adapter->num_tx_desc, M_DEVBUF, M_WAITOK | M_ZERO); |
619 | | | 619 | |
620 | /* Create the descriptor buffer dma maps */ | | 620 | /* Create the descriptor buffer dma maps */ |
621 | txbuf = txr->tx_buffers; | | 621 | txbuf = txr->tx_buffers; |
622 | for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { | | 622 | for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { |
623 | error = ixgbe_dmamap_create(txr->txtag, 0, &txbuf->map); | | 623 | error = ixgbe_dmamap_create(txr->txtag, 0, &txbuf->map); |
624 | if (error != 0) { | | 624 | if (error != 0) { |
625 | aprint_error_dev(dev, | | 625 | aprint_error_dev(dev, |
626 | "Unable to create TX DMA map (%d)\n", error); | | 626 | "Unable to create TX DMA map (%d)\n", error); |
627 | goto fail; | | 627 | goto fail; |
628 | } | | 628 | } |
629 | } | | 629 | } |
630 | | | 630 | |
631 | return 0; | | 631 | return 0; |
632 | fail: | | 632 | fail: |
633 | /* We free all, it handles case where we are in the middle */ | | 633 | /* We free all, it handles case where we are in the middle */ |
634 | #if 0 /* XXX was FreeBSD */ | | 634 | #if 0 /* XXX was FreeBSD */ |
635 | ixgbe_free_transmit_structures(adapter); | | 635 | ixgbe_free_transmit_structures(adapter); |
636 | #else | | 636 | #else |
637 | ixgbe_free_transmit_buffers(txr); | | 637 | ixgbe_free_transmit_buffers(txr); |
638 | #endif | | 638 | #endif |
639 | return (error); | | 639 | return (error); |
640 | } /* ixgbe_allocate_transmit_buffers */ | | 640 | } /* ixgbe_allocate_transmit_buffers */ |
641 | | | 641 | |
642 | /************************************************************************ | | 642 | /************************************************************************ |
643 | * ixgbe_setup_transmit_ring - Initialize a transmit ring. | | 643 | * ixgbe_setup_transmit_ring - Initialize a transmit ring. |
644 | ************************************************************************/ | | 644 | ************************************************************************/ |
645 | static void | | 645 | static void |
646 | ixgbe_setup_transmit_ring(struct tx_ring *txr) | | 646 | ixgbe_setup_transmit_ring(struct tx_ring *txr) |
647 | { | | 647 | { |
648 | struct adapter *adapter = txr->adapter; | | 648 | struct adapter *adapter = txr->adapter; |
649 | struct ixgbe_tx_buf *txbuf; | | 649 | struct ixgbe_tx_buf *txbuf; |
650 | #ifdef DEV_NETMAP | | 650 | #ifdef DEV_NETMAP |
651 | struct netmap_adapter *na = NA(adapter->ifp); | | 651 | struct netmap_adapter *na = NA(adapter->ifp); |
652 | struct netmap_slot *slot; | | 652 | struct netmap_slot *slot; |
653 | #endif /* DEV_NETMAP */ | | 653 | #endif /* DEV_NETMAP */ |
654 | | | 654 | |
655 | /* Clear the old ring contents */ | | 655 | /* Clear the old ring contents */ |
656 | IXGBE_TX_LOCK(txr); | | 656 | IXGBE_TX_LOCK(txr); |
657 | | | 657 | |
658 | #ifdef DEV_NETMAP | | 658 | #ifdef DEV_NETMAP |
659 | if (adapter->feat_en & IXGBE_FEATURE_NETMAP) { | | 659 | if (adapter->feat_en & IXGBE_FEATURE_NETMAP) { |
660 | /* | | 660 | /* |
661 | * (under lock): if in netmap mode, do some consistency | | 661 | * (under lock): if in netmap mode, do some consistency |
662 | * checks and set slot to entry 0 of the netmap ring. | | 662 | * checks and set slot to entry 0 of the netmap ring. |
663 | */ | | 663 | */ |
664 | slot = netmap_reset(na, NR_TX, txr->me, 0); | | 664 | slot = netmap_reset(na, NR_TX, txr->me, 0); |
665 | } | | 665 | } |
666 | #endif /* DEV_NETMAP */ | | 666 | #endif /* DEV_NETMAP */ |
667 | | | 667 | |
668 | bzero((void *)txr->tx_base, | | 668 | bzero((void *)txr->tx_base, |
669 | (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); | | 669 | (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); |
670 | /* Reset indices */ | | 670 | /* Reset indices */ |
671 | txr->next_avail_desc = 0; | | 671 | txr->next_avail_desc = 0; |
672 | txr->next_to_clean = 0; | | 672 | txr->next_to_clean = 0; |
673 | | | 673 | |
674 | /* Free any existing tx buffers. */ | | 674 | /* Free any existing tx buffers. */ |
675 | txbuf = txr->tx_buffers; | | 675 | txbuf = txr->tx_buffers; |
676 | for (int i = 0; i < txr->num_desc; i++, txbuf++) { | | 676 | for (int i = 0; i < txr->num_desc; i++, txbuf++) { |
677 | if (txbuf->m_head != NULL) { | | 677 | if (txbuf->m_head != NULL) { |
678 | bus_dmamap_sync(txr->txtag->dt_dmat, txbuf->map, | | 678 | bus_dmamap_sync(txr->txtag->dt_dmat, txbuf->map, |
679 | 0, txbuf->m_head->m_pkthdr.len, | | 679 | 0, txbuf->m_head->m_pkthdr.len, |
680 | BUS_DMASYNC_POSTWRITE); | | 680 | BUS_DMASYNC_POSTWRITE); |
681 | ixgbe_dmamap_unload(txr->txtag, txbuf->map); | | 681 | ixgbe_dmamap_unload(txr->txtag, txbuf->map); |
682 | m_freem(txbuf->m_head); | | 682 | m_freem(txbuf->m_head); |
683 | txbuf->m_head = NULL; | | 683 | txbuf->m_head = NULL; |
684 | } | | 684 | } |
685 | | | 685 | |
686 | #ifdef DEV_NETMAP | | 686 | #ifdef DEV_NETMAP |
687 | /* | | 687 | /* |
688 | * In netmap mode, set the map for the packet buffer. | | 688 | * In netmap mode, set the map for the packet buffer. |
689 | * NOTE: Some drivers (not this one) also need to set | | 689 | * NOTE: Some drivers (not this one) also need to set |
690 | * the physical buffer address in the NIC ring. | | 690 | * the physical buffer address in the NIC ring. |
691 | * Slots in the netmap ring (indexed by "si") are | | 691 | * Slots in the netmap ring (indexed by "si") are |
692 | * kring->nkr_hwofs positions "ahead" wrt the | | 692 | * kring->nkr_hwofs positions "ahead" wrt the |
693 | * corresponding slot in the NIC ring. In some drivers | | 693 | * corresponding slot in the NIC ring. In some drivers |
694 | * (not here) nkr_hwofs can be negative. Function | | 694 | * (not here) nkr_hwofs can be negative. Function |
695 | * netmap_idx_n2k() handles wraparounds properly. | | 695 | * netmap_idx_n2k() handles wraparounds properly. |
696 | */ | | 696 | */ |
697 | if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && slot) { | | 697 | if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && slot) { |
698 | int si = netmap_idx_n2k(na->tx_rings[txr->me], i); | | 698 | int si = netmap_idx_n2k(na->tx_rings[txr->me], i); |
699 | netmap_load_map(na, txr->txtag, | | 699 | netmap_load_map(na, txr->txtag, |
700 | txbuf->map, NMB(na, slot + si)); | | 700 | txbuf->map, NMB(na, slot + si)); |
701 | } | | 701 | } |
702 | #endif /* DEV_NETMAP */ | | 702 | #endif /* DEV_NETMAP */ |
703 | | | 703 | |
704 | /* Clear the EOP descriptor pointer */ | | 704 | /* Clear the EOP descriptor pointer */ |
705 | txbuf->eop = NULL; | | 705 | txbuf->eop = NULL; |
706 | } | | 706 | } |
707 | | | 707 | |
708 | /* Set the rate at which we sample packets */ | | 708 | /* Set the rate at which we sample packets */ |
709 | if (adapter->feat_en & IXGBE_FEATURE_FDIR) | | 709 | if (adapter->feat_en & IXGBE_FEATURE_FDIR) |
710 | txr->atr_sample = atr_sample_rate; | | 710 | txr->atr_sample = atr_sample_rate; |
711 | | | 711 | |
712 | /* Set number of descriptors available */ | | 712 | /* Set number of descriptors available */ |
713 | txr->tx_avail = adapter->num_tx_desc; | | 713 | txr->tx_avail = adapter->num_tx_desc; |
714 | | | 714 | |
715 | ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, | | 715 | ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, |
716 | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | | 716 | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); |
717 | IXGBE_TX_UNLOCK(txr); | | 717 | IXGBE_TX_UNLOCK(txr); |
718 | } /* ixgbe_setup_transmit_ring */ | | 718 | } /* ixgbe_setup_transmit_ring */ |
719 | | | 719 | |
720 | /************************************************************************ | | 720 | /************************************************************************ |
721 | * ixgbe_setup_transmit_structures - Initialize all transmit rings. | | 721 | * ixgbe_setup_transmit_structures - Initialize all transmit rings. |
722 | ************************************************************************/ | | 722 | ************************************************************************/ |
723 | int | | 723 | int |
724 | ixgbe_setup_transmit_structures(struct adapter *adapter) | | 724 | ixgbe_setup_transmit_structures(struct adapter *adapter) |
725 | { | | 725 | { |
726 | struct tx_ring *txr = adapter->tx_rings; | | 726 | struct tx_ring *txr = adapter->tx_rings; |
727 | | | 727 | |
728 | for (int i = 0; i < adapter->num_queues; i++, txr++) | | 728 | for (int i = 0; i < adapter->num_queues; i++, txr++) |
729 | ixgbe_setup_transmit_ring(txr); | | 729 | ixgbe_setup_transmit_ring(txr); |
730 | | | 730 | |
731 | return (0); | | 731 | return (0); |
732 | } /* ixgbe_setup_transmit_structures */ | | 732 | } /* ixgbe_setup_transmit_structures */ |
733 | | | 733 | |
734 | /************************************************************************ | | 734 | /************************************************************************ |
735 | * ixgbe_free_transmit_structures - Free all transmit rings. | | 735 | * ixgbe_free_transmit_structures - Free all transmit rings. |
736 | ************************************************************************/ | | 736 | ************************************************************************/ |
737 | void | | 737 | void |
738 | ixgbe_free_transmit_structures(struct adapter *adapter) | | 738 | ixgbe_free_transmit_structures(struct adapter *adapter) |
739 | { | | 739 | { |
740 | struct tx_ring *txr = adapter->tx_rings; | | 740 | struct tx_ring *txr = adapter->tx_rings; |
741 | | | 741 | |
742 | for (int i = 0; i < adapter->num_queues; i++, txr++) { | | 742 | for (int i = 0; i < adapter->num_queues; i++, txr++) { |
743 | ixgbe_free_transmit_buffers(txr); | | 743 | ixgbe_free_transmit_buffers(txr); |
744 | ixgbe_dma_free(adapter, &txr->txdma); | | 744 | ixgbe_dma_free(adapter, &txr->txdma); |
745 | IXGBE_TX_LOCK_DESTROY(txr); | | 745 | IXGBE_TX_LOCK_DESTROY(txr); |
746 | } | | 746 | } |
747 | free(adapter->tx_rings, M_DEVBUF); | | 747 | free(adapter->tx_rings, M_DEVBUF); |
748 | } /* ixgbe_free_transmit_structures */ | | 748 | } /* ixgbe_free_transmit_structures */ |
749 | | | 749 | |
750 | /************************************************************************ | | 750 | /************************************************************************ |
751 | * ixgbe_free_transmit_buffers | | 751 | * ixgbe_free_transmit_buffers |
752 | * | | 752 | * |
753 | * Free transmit ring related data structures. | | 753 | * Free transmit ring related data structures. |
754 | ************************************************************************/ | | 754 | ************************************************************************/ |
755 | static void | | 755 | static void |
756 | ixgbe_free_transmit_buffers(struct tx_ring *txr) | | 756 | ixgbe_free_transmit_buffers(struct tx_ring *txr) |
757 | { | | 757 | { |
758 | struct adapter *adapter = txr->adapter; | | 758 | struct adapter *adapter = txr->adapter; |
759 | struct ixgbe_tx_buf *tx_buffer; | | 759 | struct ixgbe_tx_buf *tx_buffer; |
760 | int i; | | 760 | int i; |
761 | | | 761 | |
762 | INIT_DEBUGOUT("ixgbe_free_transmit_buffers: begin"); | | 762 | INIT_DEBUGOUT("ixgbe_free_transmit_buffers: begin"); |
763 | | | 763 | |
764 | if (txr->tx_buffers == NULL) | | 764 | if (txr->tx_buffers == NULL) |
765 | return; | | 765 | return; |
766 | | | 766 | |
767 | tx_buffer = txr->tx_buffers; | | 767 | tx_buffer = txr->tx_buffers; |
768 | for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { | | 768 | for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { |
769 | if (tx_buffer->m_head != NULL) { | | 769 | if (tx_buffer->m_head != NULL) { |
770 | bus_dmamap_sync(txr->txtag->dt_dmat, tx_buffer->map, | | 770 | bus_dmamap_sync(txr->txtag->dt_dmat, tx_buffer->map, |
771 | 0, tx_buffer->m_head->m_pkthdr.len, | | 771 | 0, tx_buffer->m_head->m_pkthdr.len, |
772 | BUS_DMASYNC_POSTWRITE); | | 772 | BUS_DMASYNC_POSTWRITE); |
773 | ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); | | 773 | ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); |
774 | m_freem(tx_buffer->m_head); | | 774 | m_freem(tx_buffer->m_head); |
775 | tx_buffer->m_head = NULL; | | 775 | tx_buffer->m_head = NULL; |
776 | if (tx_buffer->map != NULL) { | | 776 | if (tx_buffer->map != NULL) { |
777 | ixgbe_dmamap_destroy(txr->txtag, | | 777 | ixgbe_dmamap_destroy(txr->txtag, |
778 | tx_buffer->map); | | 778 | tx_buffer->map); |
779 | tx_buffer->map = NULL; | | 779 | tx_buffer->map = NULL; |
780 | } | | 780 | } |
781 | } else if (tx_buffer->map != NULL) { | | 781 | } else if (tx_buffer->map != NULL) { |
782 | ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); | | 782 | ixgbe_dmamap_unload(txr->txtag, tx_buffer->map); |
783 | ixgbe_dmamap_destroy(txr->txtag, tx_buffer->map); | | 783 | ixgbe_dmamap_destroy(txr->txtag, tx_buffer->map); |
784 | tx_buffer->map = NULL; | | 784 | tx_buffer->map = NULL; |
785 | } | | 785 | } |
786 | } | | 786 | } |
787 | if (txr->txr_interq != NULL) { | | 787 | if (txr->txr_interq != NULL) { |
788 | struct mbuf *m; | | 788 | struct mbuf *m; |
789 | | | 789 | |
790 | while ((m = pcq_get(txr->txr_interq)) != NULL) | | 790 | while ((m = pcq_get(txr->txr_interq)) != NULL) |
791 | m_freem(m); | | 791 | m_freem(m); |
792 | pcq_destroy(txr->txr_interq); | | 792 | pcq_destroy(txr->txr_interq); |
793 | } | | 793 | } |
794 | if (txr->tx_buffers != NULL) { | | 794 | if (txr->tx_buffers != NULL) { |
795 | free(txr->tx_buffers, M_DEVBUF); | | 795 | free(txr->tx_buffers, M_DEVBUF); |
796 | txr->tx_buffers = NULL; | | 796 | txr->tx_buffers = NULL; |
797 | } | | 797 | } |
798 | if (txr->txtag != NULL) { | | 798 | if (txr->txtag != NULL) { |
799 | ixgbe_dma_tag_destroy(txr->txtag); | | 799 | ixgbe_dma_tag_destroy(txr->txtag); |
800 | txr->txtag = NULL; | | 800 | txr->txtag = NULL; |
801 | } | | 801 | } |
802 | } /* ixgbe_free_transmit_buffers */ | | 802 | } /* ixgbe_free_transmit_buffers */ |
803 | | | 803 | |
804 | /************************************************************************ | | 804 | /************************************************************************ |
805 | * ixgbe_tx_ctx_setup | | 805 | * ixgbe_tx_ctx_setup |
806 | * | | 806 | * |
807 | * Advanced Context Descriptor setup for VLAN, CSUM or TSO | | 807 | * Advanced Context Descriptor setup for VLAN, CSUM or TSO |
808 | ************************************************************************/ | | 808 | ************************************************************************/ |
809 | static int | | 809 | static int |
810 | ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, | | 810 | ixgbe_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp, |
811 | u32 *cmd_type_len, u32 *olinfo_status) | | 811 | u32 *cmd_type_len, u32 *olinfo_status) |
812 | { | | 812 | { |
813 | struct adapter *adapter = txr->adapter; | | 813 | struct adapter *adapter = txr->adapter; |
814 | struct ixgbe_adv_tx_context_desc *TXD; | | 814 | struct ixgbe_adv_tx_context_desc *TXD; |
815 | struct ether_vlan_header *eh; | | 815 | struct ether_vlan_header *eh; |
816 | #ifdef INET | | 816 | #ifdef INET |
817 | struct ip *ip; | | 817 | struct ip *ip; |
818 | #endif | | 818 | #endif |
819 | #ifdef INET6 | | 819 | #ifdef INET6 |
820 | struct ip6_hdr *ip6; | | 820 | struct ip6_hdr *ip6; |
821 | #endif | | 821 | #endif |
822 | int ehdrlen, ip_hlen = 0; | | 822 | int ehdrlen, ip_hlen = 0; |
823 | int offload = TRUE; | | 823 | int offload = TRUE; |
824 | int ctxd = txr->next_avail_desc; | | 824 | int ctxd = txr->next_avail_desc; |
825 | u32 vlan_macip_lens = 0; | | 825 | u32 vlan_macip_lens = 0; |
826 | u32 type_tucmd_mlhl = 0; | | 826 | u32 type_tucmd_mlhl = 0; |
827 | u16 vtag = 0; | | 827 | u16 vtag = 0; |
828 | u16 etype; | | 828 | u16 etype; |
829 | u8 ipproto = 0; | | 829 | u8 ipproto = 0; |
830 | char *l3d; | | 830 | char *l3d; |
831 | | | 831 | |
832 | | | 832 | |
833 | /* First check if TSO is to be used */ | | 833 | /* First check if TSO is to be used */ |
834 | if (mp->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) { | | 834 | if (mp->m_pkthdr.csum_flags & (M_CSUM_TSOv4 | M_CSUM_TSOv6)) { |
835 | int rv = ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status); | | 835 | int rv = ixgbe_tso_setup(txr, mp, cmd_type_len, olinfo_status); |
836 | | | 836 | |
837 | if (rv != 0) | | 837 | if (rv != 0) |
838 | ++adapter->tso_err.ev_count; | | 838 | ++adapter->tso_err.ev_count; |
839 | return rv; | | 839 | return rv; |
840 | } | | 840 | } |
841 | | | 841 | |
842 | if ((mp->m_pkthdr.csum_flags & M_CSUM_OFFLOAD) == 0) | | 842 | if ((mp->m_pkthdr.csum_flags & M_CSUM_OFFLOAD) == 0) |
843 | offload = FALSE; | | 843 | offload = FALSE; |
844 | | | 844 | |
845 | /* Indicate the whole packet as payload when not doing TSO */ | | 845 | /* Indicate the whole packet as payload when not doing TSO */ |
846 | *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; | | 846 | *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; |
847 | | | 847 | |
848 | /* Now ready a context descriptor */ | | 848 | /* Now ready a context descriptor */ |
849 | TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd]; | | 849 | TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd]; |
850 | | | 850 | |
851 | /* | | 851 | /* |
852 | * In advanced descriptors the vlan tag must | | 852 | * In advanced descriptors the vlan tag must |
853 | * be placed into the context descriptor. Hence | | 853 | * be placed into the context descriptor. Hence |
854 | * we need to make one even if not doing offloads. | | 854 | * we need to make one even if not doing offloads. |
855 | */ | | 855 | */ |
856 | if (vlan_has_tag(mp)) { | | 856 | if (vlan_has_tag(mp)) { |
857 | vtag = htole16(vlan_get_tag(mp)); | | 857 | vtag = htole16(vlan_get_tag(mp)); |
858 | vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); | | 858 | vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); |
859 | } else if (!(txr->adapter->feat_en & IXGBE_FEATURE_NEEDS_CTXD) && | | 859 | } else if (!(txr->adapter->feat_en & IXGBE_FEATURE_NEEDS_CTXD) && |
860 | (offload == FALSE)) | | 860 | (offload == FALSE)) |
861 | return (0); | | 861 | return (0); |
862 | | | 862 | |
863 | /* | | 863 | /* |
864 | * Determine where frame payload starts. | | 864 | * Determine where frame payload starts. |
865 | * Jump over vlan headers if already present, | | 865 | * Jump over vlan headers if already present, |
866 | * helpful for QinQ too. | | 866 | * helpful for QinQ too. |
867 | */ | | 867 | */ |
868 | KASSERT(mp->m_len >= offsetof(struct ether_vlan_header, evl_tag)); | | 868 | KASSERT(mp->m_len >= offsetof(struct ether_vlan_header, evl_tag)); |
869 | eh = mtod(mp, struct ether_vlan_header *); | | 869 | eh = mtod(mp, struct ether_vlan_header *); |
870 | if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { | | 870 | if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { |
871 | KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); | | 871 | KASSERT(mp->m_len >= sizeof(struct ether_vlan_header)); |
872 | etype = ntohs(eh->evl_proto); | | 872 | etype = ntohs(eh->evl_proto); |
873 | ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; | | 873 | ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; |
874 | } else { | | 874 | } else { |
875 | etype = ntohs(eh->evl_encap_proto); | | 875 | etype = ntohs(eh->evl_encap_proto); |
876 | ehdrlen = ETHER_HDR_LEN; | | 876 | ehdrlen = ETHER_HDR_LEN; |
877 | } | | 877 | } |
878 | | | 878 | |
879 | /* Set the ether header length */ | | 879 | /* Set the ether header length */ |
880 | vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; | | 880 | vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; |
881 | | | 881 | |
882 | if (offload == FALSE) | | 882 | if (offload == FALSE) |
883 | goto no_offloads; | | 883 | goto no_offloads; |
884 | | | 884 | |
885 | /* | | 885 | /* |
886 | * If the first mbuf only includes the ethernet header, | | 886 | * If the first mbuf only includes the ethernet header, |
887 | * jump to the next one | | 887 | * jump to the next one |
888 | * XXX: This assumes the stack splits mbufs containing headers | | 888 | * XXX: This assumes the stack splits mbufs containing headers |
889 | * on header boundaries | | 889 | * on header boundaries |
890 | * XXX: And assumes the entire IP header is contained in one mbuf | | 890 | * XXX: And assumes the entire IP header is contained in one mbuf |
891 | */ | | 891 | */ |
892 | if (mp->m_len == ehdrlen && mp->m_next) | | 892 | if (mp->m_len == ehdrlen && mp->m_next) |
893 | l3d = mtod(mp->m_next, char *); | | 893 | l3d = mtod(mp->m_next, char *); |
894 | else | | 894 | else |
895 | l3d = mtod(mp, char *) + ehdrlen; | | 895 | l3d = mtod(mp, char *) + ehdrlen; |
896 | | | 896 | |
897 | switch (etype) { | | 897 | switch (etype) { |
898 | #ifdef INET | | 898 | #ifdef INET |
899 | case ETHERTYPE_IP: | | 899 | case ETHERTYPE_IP: |
900 | ip = (struct ip *)(l3d); | | 900 | ip = (struct ip *)(l3d); |
901 | ip_hlen = ip->ip_hl << 2; | | 901 | ip_hlen = ip->ip_hl << 2; |
902 | ipproto = ip->ip_p; | | 902 | ipproto = ip->ip_p; |
903 | type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; | | 903 | type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; |
904 | KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || | | 904 | KASSERT((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) == 0 || |
905 | ip->ip_sum == 0); | | 905 | ip->ip_sum == 0); |
906 | break; | | 906 | break; |
907 | #endif | | 907 | #endif |
908 | #ifdef INET6 | | 908 | #ifdef INET6 |
909 | case ETHERTYPE_IPV6: | | 909 | case ETHERTYPE_IPV6: |
910 | ip6 = (struct ip6_hdr *)(l3d); | | 910 | ip6 = (struct ip6_hdr *)(l3d); |
911 | ip_hlen = sizeof(struct ip6_hdr); | | 911 | ip_hlen = sizeof(struct ip6_hdr); |
912 | ipproto = ip6->ip6_nxt; | | 912 | ipproto = ip6->ip6_nxt; |
913 | type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; | | 913 | type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; |
914 | break; | | 914 | break; |
915 | #endif | | 915 | #endif |
916 | default: | | 916 | default: |
917 | offload = false; | | 917 | offload = false; |
918 | break; | | 918 | break; |
919 | } | | 919 | } |
920 | | | 920 | |
921 | if ((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) != 0) | | 921 | if ((mp->m_pkthdr.csum_flags & M_CSUM_IPv4) != 0) |
922 | *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; | | 922 | *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; |
923 | | | 923 | |
924 | vlan_macip_lens |= ip_hlen; | | 924 | vlan_macip_lens |= ip_hlen; |
925 | | | 925 | |
926 | /* No support for offloads for non-L4 next headers */ | | 926 | /* No support for offloads for non-L4 next headers */ |
927 | switch (ipproto) { | | 927 | switch (ipproto) { |
928 | case IPPROTO_TCP: | | 928 | case IPPROTO_TCP: |
929 | if (mp->m_pkthdr.csum_flags & | | 929 | if (mp->m_pkthdr.csum_flags & |
930 | (M_CSUM_TCPv4 | M_CSUM_TCPv6)) | | 930 | (M_CSUM_TCPv4 | M_CSUM_TCPv6)) |
931 | type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; | | 931 | type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; |
932 | else | | 932 | else |
933 | offload = false; | | 933 | offload = false; |
934 | break; | | 934 | break; |
935 | case IPPROTO_UDP: | | 935 | case IPPROTO_UDP: |
936 | if (mp->m_pkthdr.csum_flags & | | 936 | if (mp->m_pkthdr.csum_flags & |
937 | (M_CSUM_UDPv4 | M_CSUM_UDPv6)) | | 937 | (M_CSUM_UDPv4 | M_CSUM_UDPv6)) |
938 | type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; | | 938 | type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; |
939 | else | | 939 | else |
940 | offload = false; | | 940 | offload = false; |
941 | break; | | 941 | break; |
942 | default: | | 942 | default: |
943 | offload = false; | | 943 | offload = false; |
944 | break; | | 944 | break; |
945 | } | | 945 | } |
946 | | | 946 | |
947 | if (offload) /* Insert L4 checksum into data descriptors */ | | 947 | if (offload) /* Insert L4 checksum into data descriptors */ |
948 | *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; | | 948 | *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; |
949 | | | 949 | |
950 | no_offloads: | | 950 | no_offloads: |
951 | type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; | | 951 | type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; |
952 | | | 952 | |
953 | /* Now copy bits into descriptor */ | | 953 | /* Now copy bits into descriptor */ |
954 | TXD->vlan_macip_lens = htole32(vlan_macip_lens); | | 954 | TXD->vlan_macip_lens = htole32(vlan_macip_lens); |
955 | TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); | | 955 | TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); |
956 | TXD->seqnum_seed = htole32(0); | | 956 | TXD->seqnum_seed = htole32(0); |
957 | TXD->mss_l4len_idx = htole32(0); | | 957 | TXD->mss_l4len_idx = htole32(0); |
958 | | | 958 | |
959 | /* We've consumed the first desc, adjust counters */ | | 959 | /* We've consumed the first desc, adjust counters */ |
960 | if (++ctxd == txr->num_desc) | | 960 | if (++ctxd == txr->num_desc) |
961 | ctxd = 0; | | 961 | ctxd = 0; |
962 | txr->next_avail_desc = ctxd; | | 962 | txr->next_avail_desc = ctxd; |
963 | --txr->tx_avail; | | 963 | --txr->tx_avail; |
964 | | | 964 | |
965 | return (0); | | 965 | return (0); |
966 | } /* ixgbe_tx_ctx_setup */ | | 966 | } /* ixgbe_tx_ctx_setup */ |
967 | | | 967 | |
968 | /************************************************************************ | | 968 | /************************************************************************ |
969 | * ixgbe_tso_setup | | 969 | * ixgbe_tso_setup |
970 | * | | 970 | * |
971 | * Setup work for hardware segmentation offload (TSO) on | | 971 | * Setup work for hardware segmentation offload (TSO) on |
972 | * adapters using advanced tx descriptors | | 972 | * adapters using advanced tx descriptors |
973 | ************************************************************************/ | | 973 | ************************************************************************/ |
974 | static int | | 974 | static int |
975 | ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *cmd_type_len, | | 975 | ixgbe_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *cmd_type_len, |
976 | u32 *olinfo_status) | | 976 | u32 *olinfo_status) |
977 | { | | 977 | { |
978 | struct ixgbe_adv_tx_context_desc *TXD; | | 978 | struct ixgbe_adv_tx_context_desc *TXD; |
979 | struct ether_vlan_header *eh; | | 979 | struct ether_vlan_header *eh; |
980 | #ifdef INET6 | | 980 | #ifdef INET6 |
981 | struct ip6_hdr *ip6; | | 981 | struct ip6_hdr *ip6; |
982 | #endif | | 982 | #endif |
983 | #ifdef INET | | 983 | #ifdef INET |
984 | struct ip *ip; | | 984 | struct ip *ip; |
985 | #endif | | 985 | #endif |
986 | struct tcphdr *th; | | 986 | struct tcphdr *th; |
987 | int ctxd, ehdrlen, ip_hlen, tcp_hlen; | | 987 | int ctxd, ehdrlen, ip_hlen, tcp_hlen; |
988 | u32 vlan_macip_lens = 0; | | 988 | u32 vlan_macip_lens = 0; |
989 | u32 type_tucmd_mlhl = 0; | | 989 | u32 type_tucmd_mlhl = 0; |
990 | u32 mss_l4len_idx = 0, paylen; | | 990 | u32 mss_l4len_idx = 0, paylen; |
991 | u16 vtag = 0, eh_type; | | 991 | u16 vtag = 0, eh_type; |
992 | | | 992 | |
993 | /* | | 993 | /* |
994 | * Determine where frame payload starts. | | 994 | * Determine where frame payload starts. |
995 | * Jump over vlan headers if already present | | 995 | * Jump over vlan headers if already present |
996 | */ | | 996 | */ |
997 | eh = mtod(mp, struct ether_vlan_header *); | | 997 | eh = mtod(mp, struct ether_vlan_header *); |
998 | if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { | | 998 | if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { |
999 | ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; | | 999 | ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; |
1000 | eh_type = eh->evl_proto; | | 1000 | eh_type = eh->evl_proto; |
1001 | } else { | | 1001 | } else { |
1002 | ehdrlen = ETHER_HDR_LEN; | | 1002 | ehdrlen = ETHER_HDR_LEN; |
1003 | eh_type = eh->evl_encap_proto; | | 1003 | eh_type = eh->evl_encap_proto; |
1004 | } | | 1004 | } |
1005 | | | 1005 | |
1006 | switch (ntohs(eh_type)) { | | 1006 | switch (ntohs(eh_type)) { |
1007 | #ifdef INET | | 1007 | #ifdef INET |
1008 | case ETHERTYPE_IP: | | 1008 | case ETHERTYPE_IP: |
1009 | ip = (struct ip *)(mp->m_data + ehdrlen); | | 1009 | ip = (struct ip *)(mp->m_data + ehdrlen); |
1010 | if (ip->ip_p != IPPROTO_TCP) | | 1010 | if (ip->ip_p != IPPROTO_TCP) |
1011 | return (ENXIO); | | 1011 | return (ENXIO); |
1012 | ip->ip_sum = 0; | | 1012 | ip->ip_sum = 0; |
1013 | ip_hlen = ip->ip_hl << 2; | | 1013 | ip_hlen = ip->ip_hl << 2; |
1014 | th = (struct tcphdr *)((char *)ip + ip_hlen); | | 1014 | th = (struct tcphdr *)((char *)ip + ip_hlen); |
1015 | th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, | | 1015 | th->th_sum = in_cksum_phdr(ip->ip_src.s_addr, |
1016 | ip->ip_dst.s_addr, htons(IPPROTO_TCP)); | | 1016 | ip->ip_dst.s_addr, htons(IPPROTO_TCP)); |
1017 | type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; | | 1017 | type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; |
1018 | /* Tell transmit desc to also do IPv4 checksum. */ | | 1018 | /* Tell transmit desc to also do IPv4 checksum. */ |
1019 | *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; | | 1019 | *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; |
1020 | break; | | 1020 | break; |
1021 | #endif | | 1021 | #endif |
1022 | #ifdef INET6 | | 1022 | #ifdef INET6 |
1023 | case ETHERTYPE_IPV6: | | 1023 | case ETHERTYPE_IPV6: |
1024 | ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); | | 1024 | ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); |
1025 | /* XXX-BZ For now we do not pretend to support ext. hdrs. */ | | 1025 | /* XXX-BZ For now we do not pretend to support ext. hdrs. */ |
1026 | if (ip6->ip6_nxt != IPPROTO_TCP) | | 1026 | if (ip6->ip6_nxt != IPPROTO_TCP) |
1027 | return (ENXIO); | | 1027 | return (ENXIO); |
1028 | ip_hlen = sizeof(struct ip6_hdr); | | 1028 | ip_hlen = sizeof(struct ip6_hdr); |
1029 | ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); | | 1029 | ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); |
1030 | th = (struct tcphdr *)((char *)ip6 + ip_hlen); | | 1030 | th = (struct tcphdr *)((char *)ip6 + ip_hlen); |
1031 | th->th_sum = in6_cksum_phdr(&ip6->ip6_src, | | 1031 | th->th_sum = in6_cksum_phdr(&ip6->ip6_src, |
1032 | &ip6->ip6_dst, 0, htonl(IPPROTO_TCP)); | | 1032 | &ip6->ip6_dst, 0, htonl(IPPROTO_TCP)); |
1033 | type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; | | 1033 | type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; |
1034 | break; | | 1034 | break; |
1035 | #endif | | 1035 | #endif |
1036 | default: | | 1036 | default: |
1037 | panic("%s: CSUM_TSO but no supported IP version (0x%04x)", | | 1037 | panic("%s: CSUM_TSO but no supported IP version (0x%04x)", |
1038 | __func__, ntohs(eh_type)); | | 1038 | __func__, ntohs(eh_type)); |
1039 | break; | | 1039 | break; |
1040 | } | | 1040 | } |
1041 | | | 1041 | |
1042 | ctxd = txr->next_avail_desc; | | 1042 | ctxd = txr->next_avail_desc; |
1043 | TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd]; | | 1043 | TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd]; |
1044 | | | 1044 | |
1045 | tcp_hlen = th->th_off << 2; | | 1045 | tcp_hlen = th->th_off << 2; |
1046 | | | 1046 | |
1047 | /* This is used in the transmit desc in encap */ | | 1047 | /* This is used in the transmit desc in encap */ |
1048 | paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; | | 1048 | paylen = mp->m_pkthdr.len - ehdrlen - ip_hlen - tcp_hlen; |
1049 | | | 1049 | |
1050 | /* VLAN MACLEN IPLEN */ | | 1050 | /* VLAN MACLEN IPLEN */ |
1051 | if (vlan_has_tag(mp)) { | | 1051 | if (vlan_has_tag(mp)) { |
1052 | vtag = htole16(vlan_get_tag(mp)); | | 1052 | vtag = htole16(vlan_get_tag(mp)); |
1053 | vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); | | 1053 | vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); |
1054 | } | | 1054 | } |
1055 | | | 1055 | |
1056 | vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; | | 1056 | vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; |
1057 | vlan_macip_lens |= ip_hlen; | | 1057 | vlan_macip_lens |= ip_hlen; |
1058 | TXD->vlan_macip_lens = htole32(vlan_macip_lens); | | 1058 | TXD->vlan_macip_lens = htole32(vlan_macip_lens); |
1059 | | | 1059 | |
1060 | /* ADV DTYPE TUCMD */ | | 1060 | /* ADV DTYPE TUCMD */ |
1061 | type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; | | 1061 | type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; |
1062 | type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; | | 1062 | type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; |
1063 | TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); | | 1063 | TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); |
1064 | | | 1064 | |
1065 | /* MSS L4LEN IDX */ | | 1065 | /* MSS L4LEN IDX */ |
1066 | mss_l4len_idx |= (mp->m_pkthdr.segsz << IXGBE_ADVTXD_MSS_SHIFT); | | 1066 | mss_l4len_idx |= (mp->m_pkthdr.segsz << IXGBE_ADVTXD_MSS_SHIFT); |
1067 | mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); | | 1067 | mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); |
1068 | TXD->mss_l4len_idx = htole32(mss_l4len_idx); | | 1068 | TXD->mss_l4len_idx = htole32(mss_l4len_idx); |
1069 | | | 1069 | |
1070 | TXD->seqnum_seed = htole32(0); | | 1070 | TXD->seqnum_seed = htole32(0); |
1071 | | | 1071 | |
1072 | if (++ctxd == txr->num_desc) | | 1072 | if (++ctxd == txr->num_desc) |
1073 | ctxd = 0; | | 1073 | ctxd = 0; |
1074 | | | 1074 | |
1075 | txr->tx_avail--; | | 1075 | txr->tx_avail--; |
1076 | txr->next_avail_desc = ctxd; | | 1076 | txr->next_avail_desc = ctxd; |
1077 | *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; | | 1077 | *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; |
1078 | *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; | | 1078 | *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; |
1079 | *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; | | 1079 | *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; |
1080 | ++txr->tso_tx.ev_count; | | 1080 | ++txr->tso_tx.ev_count; |
1081 | | | 1081 | |
1082 | return (0); | | 1082 | return (0); |
1083 | } /* ixgbe_tso_setup */ | | 1083 | } /* ixgbe_tso_setup */ |
1084 | | | 1084 | |
1085 | | | 1085 | |
1086 | /************************************************************************ | | 1086 | /************************************************************************ |
1087 | * ixgbe_txeof | | 1087 | * ixgbe_txeof |
1088 | * | | 1088 | * |
1089 | * Examine each tx_buffer in the used queue. If the hardware is done | | 1089 | * Examine each tx_buffer in the used queue. If the hardware is done |
1090 | * processing the packet then free associated resources. The | | 1090 | * processing the packet then free associated resources. The |
1091 | * tx_buffer is put back on the free queue. | | 1091 | * tx_buffer is put back on the free queue. |
1092 | ************************************************************************/ | | 1092 | ************************************************************************/ |
1093 | bool | | 1093 | bool |
1094 | ixgbe_txeof(struct tx_ring *txr) | | 1094 | ixgbe_txeof(struct tx_ring *txr) |
1095 | { | | 1095 | { |
1096 | struct adapter *adapter = txr->adapter; | | 1096 | struct adapter *adapter = txr->adapter; |
1097 | struct ifnet *ifp = adapter->ifp; | | 1097 | struct ifnet *ifp = adapter->ifp; |
1098 | struct ixgbe_tx_buf *buf; | | 1098 | struct ixgbe_tx_buf *buf; |
1099 | union ixgbe_adv_tx_desc *txd; | | 1099 | union ixgbe_adv_tx_desc *txd; |
1100 | u32 work, processed = 0; | | 1100 | u32 work, processed = 0; |
1101 | u32 limit = adapter->tx_process_limit; | | 1101 | u32 limit = adapter->tx_process_limit; |
1102 | | | 1102 | |
1103 | KASSERT(mutex_owned(&txr->tx_mtx)); | | 1103 | KASSERT(mutex_owned(&txr->tx_mtx)); |
1104 | | | 1104 | |
1105 | #ifdef DEV_NETMAP | | 1105 | #ifdef DEV_NETMAP |
1106 | if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && | | 1106 | if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && |
1107 | (adapter->ifp->if_capenable & IFCAP_NETMAP)) { | | 1107 | (adapter->ifp->if_capenable & IFCAP_NETMAP)) { |
1108 | struct netmap_adapter *na = NA(adapter->ifp); | | 1108 | struct netmap_adapter *na = NA(adapter->ifp); |
1109 | struct netmap_kring *kring = na->tx_rings[txr->me]; | | 1109 | struct netmap_kring *kring = na->tx_rings[txr->me]; |
1110 | txd = txr->tx_base; | | 1110 | txd = txr->tx_base; |
1111 | bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, | | 1111 | bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, |
1112 | BUS_DMASYNC_POSTREAD); | | 1112 | BUS_DMASYNC_POSTREAD); |
1113 | /* | | 1113 | /* |
1114 | * In netmap mode, all the work is done in the context | | 1114 | * In netmap mode, all the work is done in the context |
1115 | * of the client thread. Interrupt handlers only wake up | | 1115 | * of the client thread. Interrupt handlers only wake up |
1116 | * clients, which may be sleeping on individual rings | | 1116 | * clients, which may be sleeping on individual rings |
1117 | * or on a global resource for all rings. | | 1117 | * or on a global resource for all rings. |
1118 | * To implement tx interrupt mitigation, we wake up the client | | 1118 | * To implement tx interrupt mitigation, we wake up the client |
1119 | * thread roughly every half ring, even if the NIC interrupts | | 1119 | * thread roughly every half ring, even if the NIC interrupts |
1120 | * more frequently. This is implemented as follows: | | 1120 | * more frequently. This is implemented as follows: |
1121 | * - ixgbe_txsync() sets kring->nr_kflags with the index of | | 1121 | * - ixgbe_txsync() sets kring->nr_kflags with the index of |
1122 | * the slot that should wake up the thread (nkr_num_slots | | 1122 | * the slot that should wake up the thread (nkr_num_slots |
1123 | * means the user thread should not be woken up); | | 1123 | * means the user thread should not be woken up); |
1124 | * - the driver ignores tx interrupts unless netmap_mitigate=0 | | 1124 | * - the driver ignores tx interrupts unless netmap_mitigate=0 |
1125 | * or the slot has the DD bit set. | | 1125 | * or the slot has the DD bit set. |
1126 | */ | | 1126 | */ |
1127 | if (kring->nr_kflags < kring->nkr_num_slots && | | 1127 | if (kring->nr_kflags < kring->nkr_num_slots && |
1128 | le32toh(txd[kring->nr_kflags].wb.status) & IXGBE_TXD_STAT_DD) { | | 1128 | le32toh(txd[kring->nr_kflags].wb.status) & IXGBE_TXD_STAT_DD) { |
1129 | netmap_tx_irq(ifp, txr->me); | | 1129 | netmap_tx_irq(ifp, txr->me); |
1130 | } | | 1130 | } |
1131 | return false; | | 1131 | return false; |
1132 | } | | 1132 | } |
1133 | #endif /* DEV_NETMAP */ | | 1133 | #endif /* DEV_NETMAP */ |
1134 | | | 1134 | |
1135 | if (txr->tx_avail == txr->num_desc) { | | 1135 | if (txr->tx_avail == txr->num_desc) { |
1136 | txr->busy = 0; | | 1136 | txr->busy = 0; |
1137 | return false; | | 1137 | return false; |
1138 | } | | 1138 | } |
1139 | | | 1139 | |
1140 | /* Get work starting point */ | | 1140 | /* Get work starting point */ |
1141 | work = txr->next_to_clean; | | 1141 | work = txr->next_to_clean; |
1142 | buf = &txr->tx_buffers[work]; | | 1142 | buf = &txr->tx_buffers[work]; |
1143 | txd = &txr->tx_base[work]; | | 1143 | txd = &txr->tx_base[work]; |
1144 | work -= txr->num_desc; /* The distance to ring end */ | | 1144 | work -= txr->num_desc; /* The distance to ring end */ |
1145 | ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, | | 1145 | ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, |
1146 | BUS_DMASYNC_POSTREAD); | | 1146 | BUS_DMASYNC_POSTREAD); |
1147 | | | 1147 | |
1148 | do { | | 1148 | do { |
1149 | union ixgbe_adv_tx_desc *eop = buf->eop; | | 1149 | union ixgbe_adv_tx_desc *eop = buf->eop; |
1150 | if (eop == NULL) /* No work */ | | 1150 | if (eop == NULL) /* No work */ |
1151 | break; | | 1151 | break; |
1152 | | | 1152 | |
1153 | if ((le32toh(eop->wb.status) & IXGBE_TXD_STAT_DD) == 0) | | 1153 | if ((le32toh(eop->wb.status) & IXGBE_TXD_STAT_DD) == 0) |
1154 | break; /* I/O not complete */ | | 1154 | break; /* I/O not complete */ |
1155 | | | 1155 | |
1156 | if (buf->m_head) { | | 1156 | if (buf->m_head) { |
1157 | txr->bytes += buf->m_head->m_pkthdr.len; | | 1157 | txr->bytes += buf->m_head->m_pkthdr.len; |
1158 | bus_dmamap_sync(txr->txtag->dt_dmat, buf->map, | | 1158 | bus_dmamap_sync(txr->txtag->dt_dmat, buf->map, |
1159 | 0, buf->m_head->m_pkthdr.len, | | 1159 | 0, buf->m_head->m_pkthdr.len, |
1160 | BUS_DMASYNC_POSTWRITE); | | 1160 | BUS_DMASYNC_POSTWRITE); |
1161 | ixgbe_dmamap_unload(txr->txtag, buf->map); | | 1161 | ixgbe_dmamap_unload(txr->txtag, buf->map); |
1162 | m_freem(buf->m_head); | | 1162 | m_freem(buf->m_head); |
1163 | buf->m_head = NULL; | | 1163 | buf->m_head = NULL; |
1164 | } | | 1164 | } |
1165 | buf->eop = NULL; | | 1165 | buf->eop = NULL; |
1166 | txr->txr_no_space = false; | | 1166 | txr->txr_no_space = false; |
1167 | ++txr->tx_avail; | | 1167 | ++txr->tx_avail; |
1168 | | | 1168 | |
1169 | /* We clean the range if multi segment */ | | 1169 | /* We clean the range if multi segment */ |
1170 | while (txd != eop) { | | 1170 | while (txd != eop) { |
1171 | ++txd; | | 1171 | ++txd; |
1172 | ++buf; | | 1172 | ++buf; |
1173 | ++work; | | 1173 | ++work; |
1174 | /* wrap the ring? */ | | 1174 | /* wrap the ring? */ |
1175 | if (__predict_false(!work)) { | | 1175 | if (__predict_false(!work)) { |
1176 | work -= txr->num_desc; | | 1176 | work -= txr->num_desc; |
1177 | buf = txr->tx_buffers; | | 1177 | buf = txr->tx_buffers; |
1178 | txd = txr->tx_base; | | 1178 | txd = txr->tx_base; |
1179 | } | | 1179 | } |
1180 | if (buf->m_head) { | | 1180 | if (buf->m_head) { |
1181 | txr->bytes += | | 1181 | txr->bytes += |
1182 | buf->m_head->m_pkthdr.len; | | 1182 | buf->m_head->m_pkthdr.len; |
1183 | bus_dmamap_sync(txr->txtag->dt_dmat, | | 1183 | bus_dmamap_sync(txr->txtag->dt_dmat, |
1184 | buf->map, | | 1184 | buf->map, |
1185 | 0, buf->m_head->m_pkthdr.len, | | 1185 | 0, buf->m_head->m_pkthdr.len, |
1186 | BUS_DMASYNC_POSTWRITE); | | 1186 | BUS_DMASYNC_POSTWRITE); |
1187 | ixgbe_dmamap_unload(txr->txtag, | | 1187 | ixgbe_dmamap_unload(txr->txtag, |
1188 | buf->map); | | 1188 | buf->map); |
1189 | m_freem(buf->m_head); | | 1189 | m_freem(buf->m_head); |
1190 | buf->m_head = NULL; | | 1190 | buf->m_head = NULL; |
1191 | } | | 1191 | } |
1192 | ++txr->tx_avail; | | 1192 | ++txr->tx_avail; |
1193 | buf->eop = NULL; | | 1193 | buf->eop = NULL; |
1194 | | | 1194 | |
1195 | } | | 1195 | } |
1196 | ++txr->packets; | | 1196 | ++txr->packets; |
1197 | ++processed; | | 1197 | ++processed; |
1198 | if_statinc(ifp, if_opackets); | | 1198 | if_statinc(ifp, if_opackets); |
1199 | | | 1199 | |
1200 | /* Try the next packet */ | | 1200 | /* Try the next packet */ |
1201 | ++txd; | | 1201 | ++txd; |
1202 | ++buf; | | 1202 | ++buf; |
1203 | ++work; | | 1203 | ++work; |
1204 | /* reset with a wrap */ | | 1204 | /* reset with a wrap */ |
1205 | if (__predict_false(!work)) { | | 1205 | if (__predict_false(!work)) { |
1206 | work -= txr->num_desc; | | 1206 | work -= txr->num_desc; |
1207 | buf = txr->tx_buffers; | | 1207 | buf = txr->tx_buffers; |
1208 | txd = txr->tx_base; | | 1208 | txd = txr->tx_base; |
1209 | } | | 1209 | } |
1210 | prefetch(txd); | | 1210 | prefetch(txd); |
1211 | } while (__predict_true(--limit)); | | 1211 | } while (__predict_true(--limit)); |
1212 | | | 1212 | |
1213 | ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, | | 1213 | ixgbe_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, |
1214 | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | | 1214 | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); |
1215 | | | 1215 | |
1216 | work += txr->num_desc; | | 1216 | work += txr->num_desc; |
1217 | txr->next_to_clean = work; | | 1217 | txr->next_to_clean = work; |
1218 | | | 1218 | |
1219 | /* | | 1219 | /* |
1220 | * Queue Hang detection, we know there's | | 1220 | * Queue Hang detection, we know there's |
1221 | * work outstanding or the first return | | 1221 | * work outstanding or the first return |
1222 | * would have been taken, so increment busy | | 1222 | * would have been taken, so increment busy |
1223 | * if nothing managed to get cleaned, then | | 1223 | * if nothing managed to get cleaned, then |
1224 | * in local_timer it will be checked and | | 1224 | * in local_timer it will be checked and |
1225 | * marked as HUNG if it exceeds a MAX attempt. | | 1225 | * marked as HUNG if it exceeds a MAX attempt. |
1226 | */ | | 1226 | */ |
1227 | if ((processed == 0) && (txr->busy != IXGBE_QUEUE_HUNG)) | | 1227 | if ((processed == 0) && (txr->busy != IXGBE_QUEUE_HUNG)) |
1228 | ++txr->busy; | | 1228 | ++txr->busy; |
1229 | /* | | 1229 | /* |
1230 | * If anything gets cleaned we reset state to 1, | | 1230 | * If anything gets cleaned we reset state to 1, |
1231 | * note this will turn off HUNG if its set. | | 1231 | * note this will turn off HUNG if its set. |
1232 | */ | | 1232 | */ |
1233 | if (processed) | | 1233 | if (processed) |
1234 | txr->busy = 1; | | 1234 | txr->busy = 1; |
1235 | | | 1235 | |
1236 | if (txr->tx_avail == txr->num_desc) | | 1236 | if (txr->tx_avail == txr->num_desc) |
1237 | txr->busy = 0; | | 1237 | txr->busy = 0; |
1238 | | | 1238 | |
1239 | return ((limit > 0) ? false : true); | | 1239 | return ((limit > 0) ? false : true); |
1240 | } /* ixgbe_txeof */ | | 1240 | } /* ixgbe_txeof */ |
1241 | | | 1241 | |
1242 | /************************************************************************ | | 1242 | /************************************************************************ |
1243 | * ixgbe_rsc_count | | 1243 | * ixgbe_rsc_count |
1244 | * | | 1244 | * |
1245 | * Used to detect a descriptor that has been merged by Hardware RSC. | | 1245 | * Used to detect a descriptor that has been merged by Hardware RSC. |
1246 | ************************************************************************/ | | 1246 | ************************************************************************/ |
1247 | static inline u32 | | 1247 | static inline u32 |
1248 | ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx) | | 1248 | ixgbe_rsc_count(union ixgbe_adv_rx_desc *rx) |
1249 | { | | 1249 | { |
1250 | return (le32toh(rx->wb.lower.lo_dword.data) & | | 1250 | return (le32toh(rx->wb.lower.lo_dword.data) & |
1251 | IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; | | 1251 | IXGBE_RXDADV_RSCCNT_MASK) >> IXGBE_RXDADV_RSCCNT_SHIFT; |
1252 | } /* ixgbe_rsc_count */ | | 1252 | } /* ixgbe_rsc_count */ |
1253 | | | 1253 | |
1254 | /************************************************************************ | | 1254 | /************************************************************************ |
1255 | * ixgbe_setup_hw_rsc | | 1255 | * ixgbe_setup_hw_rsc |
1256 | * | | 1256 | * |
1257 | * Initialize Hardware RSC (LRO) feature on 82599 | | 1257 | * Initialize Hardware RSC (LRO) feature on 82599 |
1258 | * for an RX ring, this is toggled by the LRO capability | | 1258 | * for an RX ring, this is toggled by the LRO capability |
1259 | * even though it is transparent to the stack. | | 1259 | * even though it is transparent to the stack. |
1260 | * | | 1260 | * |
1261 | * NOTE: Since this HW feature only works with IPv4 and | | 1261 | * NOTE: Since this HW feature only works with IPv4 and |
1262 | * testing has shown soft LRO to be as effective, | | 1262 | * testing has shown soft LRO to be as effective, |
1263 | * this feature will be disabled by default. | | 1263 | * this feature will be disabled by default. |
1264 | ************************************************************************/ | | 1264 | ************************************************************************/ |
1265 | static void | | 1265 | static void |
1266 | ixgbe_setup_hw_rsc(struct rx_ring *rxr) | | 1266 | ixgbe_setup_hw_rsc(struct rx_ring *rxr) |
1267 | { | | 1267 | { |
1268 | struct adapter *adapter = rxr->adapter; | | 1268 | struct adapter *adapter = rxr->adapter; |
1269 | struct ixgbe_hw *hw = &adapter->hw; | | 1269 | struct ixgbe_hw *hw = &adapter->hw; |
1270 | u32 rscctrl, rdrxctl; | | 1270 | u32 rscctrl, rdrxctl; |
1271 | | | 1271 | |
1272 | /* If turning LRO/RSC off we need to disable it */ | | 1272 | /* If turning LRO/RSC off we need to disable it */ |
1273 | if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) { | | 1273 | if ((adapter->ifp->if_capenable & IFCAP_LRO) == 0) { |
1274 | rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); | | 1274 | rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); |
1275 | rscctrl &= ~IXGBE_RSCCTL_RSCEN; | | 1275 | rscctrl &= ~IXGBE_RSCCTL_RSCEN; |
1276 | return; | | 1276 | return; |
1277 | } | | 1277 | } |
1278 | | | 1278 | |
1279 | rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); | | 1279 | rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); |
1280 | rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; | | 1280 | rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; |
1281 | #ifdef DEV_NETMAP | | 1281 | #ifdef DEV_NETMAP |
1282 | /* Always strip CRC unless Netmap disabled it */ | | 1282 | /* Always strip CRC unless Netmap disabled it */ |
1283 | if (!(adapter->feat_en & IXGBE_FEATURE_NETMAP) || | | 1283 | if (!(adapter->feat_en & IXGBE_FEATURE_NETMAP) || |
1284 | !(adapter->ifp->if_capenable & IFCAP_NETMAP) || | | 1284 | !(adapter->ifp->if_capenable & IFCAP_NETMAP) || |
1285 | ix_crcstrip) | | 1285 | ix_crcstrip) |
1286 | #endif /* DEV_NETMAP */ | | 1286 | #endif /* DEV_NETMAP */ |
1287 | rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; | | 1287 | rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP; |
1288 | rdrxctl |= IXGBE_RDRXCTL_RSCACKC; | | 1288 | rdrxctl |= IXGBE_RDRXCTL_RSCACKC; |
1289 | IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); | | 1289 | IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); |
1290 | | | 1290 | |
1291 | rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); | | 1291 | rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(rxr->me)); |
1292 | rscctrl |= IXGBE_RSCCTL_RSCEN; | | 1292 | rscctrl |= IXGBE_RSCCTL_RSCEN; |
1293 | /* | | 1293 | /* |
1294 | * Limit the total number of descriptors that | | 1294 | * Limit the total number of descriptors that |
1295 | * can be combined, so it does not exceed 64K | | 1295 | * can be combined, so it does not exceed 64K |
1296 | */ | | 1296 | */ |
1297 | if (rxr->mbuf_sz == MCLBYTES) | | 1297 | if (rxr->mbuf_sz == MCLBYTES) |
1298 | rscctrl |= IXGBE_RSCCTL_MAXDESC_16; | | 1298 | rscctrl |= IXGBE_RSCCTL_MAXDESC_16; |
1299 | else if (rxr->mbuf_sz == MJUMPAGESIZE) | | 1299 | else if (rxr->mbuf_sz == MJUMPAGESIZE) |
1300 | rscctrl |= IXGBE_RSCCTL_MAXDESC_8; | | 1300 | rscctrl |= IXGBE_RSCCTL_MAXDESC_8; |
1301 | else if (rxr->mbuf_sz == MJUM9BYTES) | | 1301 | else if (rxr->mbuf_sz == MJUM9BYTES) |
1302 | rscctrl |= IXGBE_RSCCTL_MAXDESC_4; | | 1302 | rscctrl |= IXGBE_RSCCTL_MAXDESC_4; |
1303 | else /* Using 16K cluster */ | | 1303 | else /* Using 16K cluster */ |
1304 | rscctrl |= IXGBE_RSCCTL_MAXDESC_1; | | 1304 | rscctrl |= IXGBE_RSCCTL_MAXDESC_1; |
1305 | | | 1305 | |
1306 | IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl); | | 1306 | IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(rxr->me), rscctrl); |
1307 | | | 1307 | |
1308 | /* Enable TCP header recognition */ | | 1308 | /* Enable TCP header recognition */ |
1309 | IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), | | 1309 | IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), |
1310 | (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | IXGBE_PSRTYPE_TCPHDR)); | | 1310 | (IXGBE_READ_REG(hw, IXGBE_PSRTYPE(0)) | IXGBE_PSRTYPE_TCPHDR)); |
1311 | | | 1311 | |
1312 | /* Disable RSC for ACK packets */ | | 1312 | /* Disable RSC for ACK packets */ |
1313 | IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, | | 1313 | IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, |
1314 | (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); | | 1314 | (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU))); |
1315 | | | 1315 | |
1316 | rxr->hw_rsc = TRUE; | | 1316 | rxr->hw_rsc = TRUE; |
1317 | } /* ixgbe_setup_hw_rsc */ | | 1317 | } /* ixgbe_setup_hw_rsc */ |
1318 | | | 1318 | |
1319 | /************************************************************************ | | 1319 | /************************************************************************ |
1320 | * ixgbe_refresh_mbufs | | 1320 | * ixgbe_refresh_mbufs |
1321 | * | | 1321 | * |
1322 | * Refresh mbuf buffers for RX descriptor rings | | 1322 | * Refresh mbuf buffers for RX descriptor rings |
1323 | * - now keeps its own state so discards due to resource | | 1323 | * - now keeps its own state so discards due to resource |
1324 | * exhaustion are unnecessary, if an mbuf cannot be obtained | | 1324 | * exhaustion are unnecessary, if an mbuf cannot be obtained |
1325 | * it just returns, keeping its placeholder, thus it can simply | | 1325 | * it just returns, keeping its placeholder, thus it can simply |
1326 | * be recalled to try again. | | 1326 | * be recalled to try again. |
1327 | * | | 1327 | * |
1328 | * XXX NetBSD TODO: | | 1328 | * XXX NetBSD TODO: |
1329 | * - The ixgbe_rxeof() function always preallocates mbuf cluster (jcl), | | 1329 | * - The ixgbe_rxeof() function always preallocates mbuf cluster (jcl), |
1330 | * so the ixgbe_refresh_mbufs() function can be simplified. | | 1330 | * so the ixgbe_refresh_mbufs() function can be simplified. |
1331 | * | | 1331 | * |
1332 | ************************************************************************/ | | 1332 | ************************************************************************/ |
1333 | static void | | 1333 | static void |
1334 | ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit) | | 1334 | ixgbe_refresh_mbufs(struct rx_ring *rxr, int limit) |
1335 | { | | 1335 | { |
1336 | struct adapter *adapter = rxr->adapter; | | 1336 | struct adapter *adapter = rxr->adapter; |
1337 | struct ixgbe_rx_buf *rxbuf; | | 1337 | struct ixgbe_rx_buf *rxbuf; |
1338 | struct mbuf *mp; | | 1338 | struct mbuf *mp; |
1339 | int i, j, error; | | 1339 | int i, j, error; |
1340 | bool refreshed = false; | | 1340 | bool refreshed = false; |
1341 | | | 1341 | |
1342 | i = j = rxr->next_to_refresh; | | 1342 | i = j = rxr->next_to_refresh; |
1343 | /* Control the loop with one beyond */ | | 1343 | /* Control the loop with one beyond */ |
1344 | if (++j == rxr->num_desc) | | 1344 | if (++j == rxr->num_desc) |
1345 | j = 0; | | 1345 | j = 0; |
1346 | | | 1346 | |
1347 | while (j != limit) { | | 1347 | while (j != limit) { |
1348 | rxbuf = &rxr->rx_buffers[i]; | | 1348 | rxbuf = &rxr->rx_buffers[i]; |
1349 | if (rxbuf->buf == NULL) { | | 1349 | if (rxbuf->buf == NULL) { |
1350 | mp = ixgbe_getjcl(&rxr->jcl_head, M_NOWAIT, | | 1350 | mp = ixgbe_getjcl(&rxr->jcl_head, M_NOWAIT, |
1351 | MT_DATA, M_PKTHDR, rxr->mbuf_sz); | | 1351 | MT_DATA, M_PKTHDR, rxr->mbuf_sz); |
1352 | if (mp == NULL) { | | 1352 | if (mp == NULL) { |
1353 | rxr->no_jmbuf.ev_count++; | | 1353 | rxr->no_jmbuf.ev_count++; |
1354 | goto update; | | 1354 | goto update; |
1355 | } | | 1355 | } |
1356 | if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) | | 1356 | if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) |
1357 | m_adj(mp, ETHER_ALIGN); | | 1357 | m_adj(mp, ETHER_ALIGN); |
1358 | } else | | 1358 | } else |
1359 | mp = rxbuf->buf; | | 1359 | mp = rxbuf->buf; |
1360 | | | 1360 | |
1361 | mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; | | 1361 | mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; |
1362 | | | 1362 | |
1363 | /* If we're dealing with an mbuf that was copied rather | | 1363 | /* If we're dealing with an mbuf that was copied rather |
1364 | * than replaced, there's no need to go through busdma. | | 1364 | * than replaced, there's no need to go through busdma. |
1365 | */ | | 1365 | */ |
1366 | if ((rxbuf->flags & IXGBE_RX_COPY) == 0) { | | 1366 | if ((rxbuf->flags & IXGBE_RX_COPY) == 0) { |
1367 | /* Get the memory mapping */ | | 1367 | /* Get the memory mapping */ |
1368 | ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); | | 1368 | ixgbe_dmamap_unload(rxr->ptag, rxbuf->pmap); |
1369 | error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, | | 1369 | error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, |
1370 | rxbuf->pmap, mp, BUS_DMA_NOWAIT); | | 1370 | rxbuf->pmap, mp, BUS_DMA_NOWAIT); |
1371 | if (error != 0) { | | 1371 | if (error != 0) { |
1372 | device_printf(adapter->dev, "Refresh mbufs: " | | 1372 | device_printf(adapter->dev, "Refresh mbufs: " |
1373 | "payload dmamap load failure - %d\n", | | 1373 | "payload dmamap load failure - %d\n", |
1374 | error); | | 1374 | error); |
1375 | m_free(mp); | | 1375 | m_free(mp); |
1376 | rxbuf->buf = NULL; | | 1376 | rxbuf->buf = NULL; |
1377 | goto update; | | 1377 | goto update; |
1378 | } | | 1378 | } |
1379 | rxbuf->buf = mp; | | 1379 | rxbuf->buf = mp; |
1380 | bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, | | 1380 | bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, |
1381 | 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD); | | 1381 | 0, mp->m_pkthdr.len, BUS_DMASYNC_PREREAD); |
1382 | rxbuf->addr = rxr->rx_base[i].read.pkt_addr = | | 1382 | rxbuf->addr = rxr->rx_base[i].read.pkt_addr = |
1383 | htole64(rxbuf->pmap->dm_segs[0].ds_addr); | | 1383 | htole64(rxbuf->pmap->dm_segs[0].ds_addr); |
1384 | } else { | | 1384 | } else { |
1385 | rxr->rx_base[i].read.pkt_addr = rxbuf->addr; | | 1385 | rxr->rx_base[i].read.pkt_addr = rxbuf->addr; |
1386 | rxbuf->flags &= ~IXGBE_RX_COPY; | | 1386 | rxbuf->flags &= ~IXGBE_RX_COPY; |
1387 | } | | 1387 | } |
1388 | | | 1388 | |
1389 | refreshed = true; | | 1389 | refreshed = true; |
1390 | /* Next is precalculated */ | | 1390 | /* Next is precalculated */ |
1391 | i = j; | | 1391 | i = j; |
1392 | rxr->next_to_refresh = i; | | 1392 | rxr->next_to_refresh = i; |
1393 | if (++j == rxr->num_desc) | | 1393 | if (++j == rxr->num_desc) |
1394 | j = 0; | | 1394 | j = 0; |
1395 | } | | 1395 | } |
1396 | | | 1396 | |
1397 | update: | | 1397 | update: |
1398 | if (refreshed) /* Update hardware tail index */ | | 1398 | if (refreshed) /* Update hardware tail index */ |
1399 | IXGBE_WRITE_REG(&adapter->hw, rxr->tail, rxr->next_to_refresh); | | 1399 | IXGBE_WRITE_REG(&adapter->hw, rxr->tail, rxr->next_to_refresh); |
1400 | | | 1400 | |
1401 | return; | | 1401 | return; |
1402 | } /* ixgbe_refresh_mbufs */ | | 1402 | } /* ixgbe_refresh_mbufs */ |
1403 | | | 1403 | |
1404 | /************************************************************************ | | 1404 | /************************************************************************ |
1405 | * ixgbe_allocate_receive_buffers | | 1405 | * ixgbe_allocate_receive_buffers |
1406 | * | | 1406 | * |
1407 | * Allocate memory for rx_buffer structures. Since we use one | | 1407 | * Allocate memory for rx_buffer structures. Since we use one |
1408 | * rx_buffer per received packet, the maximum number of rx_buffer's | | 1408 | * rx_buffer per received packet, the maximum number of rx_buffer's |
1409 | * that we'll need is equal to the number of receive descriptors | | 1409 | * that we'll need is equal to the number of receive descriptors |
1410 | * that we've allocated. | | 1410 | * that we've allocated. |
1411 | ************************************************************************/ | | 1411 | ************************************************************************/ |
1412 | static int | | 1412 | static int |
1413 | ixgbe_allocate_receive_buffers(struct rx_ring *rxr) | | 1413 | ixgbe_allocate_receive_buffers(struct rx_ring *rxr) |
1414 | { | | 1414 | { |
1415 | struct adapter *adapter = rxr->adapter; | | 1415 | struct adapter *adapter = rxr->adapter; |
1416 | device_t dev = adapter->dev; | | 1416 | device_t dev = adapter->dev; |
1417 | struct ixgbe_rx_buf *rxbuf; | | 1417 | struct ixgbe_rx_buf *rxbuf; |
1418 | int bsize, error; | | 1418 | int bsize, error; |
1419 | | | 1419 | |
1420 | bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc; | | 1420 | bsize = sizeof(struct ixgbe_rx_buf) * rxr->num_desc; |
1421 | rxr->rx_buffers = malloc(bsize, M_DEVBUF, M_WAITOK | M_ZERO); | | 1421 | rxr->rx_buffers = malloc(bsize, M_DEVBUF, M_WAITOK | M_ZERO); |
1422 | | | 1422 | |
1423 | error = ixgbe_dma_tag_create( | | 1423 | error = ixgbe_dma_tag_create( |
1424 | /* parent */ adapter->osdep.dmat, | | 1424 | /* parent */ adapter->osdep.dmat, |
1425 | /* alignment */ 1, | | 1425 | /* alignment */ 1, |
1426 | /* bounds */ 0, | | 1426 | /* bounds */ 0, |
1427 | /* maxsize */ MJUM16BYTES, | | 1427 | /* maxsize */ MJUM16BYTES, |
1428 | /* nsegments */ 1, | | 1428 | /* nsegments */ 1, |
1429 | /* maxsegsize */ MJUM16BYTES, | | 1429 | /* maxsegsize */ MJUM16BYTES, |
1430 | /* flags */ 0, | | 1430 | /* flags */ 0, |
1431 | &rxr->ptag); | | 1431 | &rxr->ptag); |
1432 | if (error != 0) { | | 1432 | if (error != 0) { |
1433 | aprint_error_dev(dev, "Unable to create RX DMA tag\n"); | | 1433 | aprint_error_dev(dev, "Unable to create RX DMA tag\n"); |
1434 | goto fail; | | 1434 | goto fail; |
1435 | } | | 1435 | } |
1436 | | | 1436 | |
1437 | for (int i = 0; i < rxr->num_desc; i++, rxbuf++) { | | 1437 | for (int i = 0; i < rxr->num_desc; i++, rxbuf++) { |
1438 | rxbuf = &rxr->rx_buffers[i]; | | 1438 | rxbuf = &rxr->rx_buffers[i]; |
1439 | error = ixgbe_dmamap_create(rxr->ptag, 0, &rxbuf->pmap); | | 1439 | error = ixgbe_dmamap_create(rxr->ptag, 0, &rxbuf->pmap); |
1440 | if (error) { | | 1440 | if (error) { |
1441 | aprint_error_dev(dev, "Unable to create RX dma map\n"); | | 1441 | aprint_error_dev(dev, "Unable to create RX dma map\n"); |
1442 | goto fail; | | 1442 | goto fail; |
1443 | } | | 1443 | } |
1444 | } | | 1444 | } |
1445 | | | 1445 | |
1446 | return (0); | | 1446 | return (0); |
1447 | | | 1447 | |
1448 | fail: | | 1448 | fail: |
1449 | /* Frees all, but can handle partial completion */ | | 1449 | /* Frees all, but can handle partial completion */ |
1450 | ixgbe_free_receive_structures(adapter); | | 1450 | ixgbe_free_receive_structures(adapter); |
1451 | | | 1451 | |
1452 | return (error); | | 1452 | return (error); |
1453 | } /* ixgbe_allocate_receive_buffers */ | | 1453 | } /* ixgbe_allocate_receive_buffers */ |
1454 | | | 1454 | |
1455 | /************************************************************************ | | 1455 | /************************************************************************ |
1456 | * ixgbe_free_receive_ring | | 1456 | * ixgbe_free_receive_ring |
1457 | ************************************************************************/ | | 1457 | ************************************************************************/ |
1458 | static void | | 1458 | static void |
1459 | ixgbe_free_receive_ring(struct rx_ring *rxr) | | 1459 | ixgbe_free_receive_ring(struct rx_ring *rxr) |
1460 | { | | 1460 | { |
1461 | for (int i = 0; i < rxr->num_desc; i++) { | | 1461 | for (int i = 0; i < rxr->num_desc; i++) { |
1462 | ixgbe_rx_discard(rxr, i); | | 1462 | ixgbe_rx_discard(rxr, i); |
1463 | } | | 1463 | } |
1464 | } /* ixgbe_free_receive_ring */ | | 1464 | } /* ixgbe_free_receive_ring */ |
1465 | | | 1465 | |
1466 | /************************************************************************ | | 1466 | /************************************************************************ |
1467 | * ixgbe_setup_receive_ring | | 1467 | * ixgbe_setup_receive_ring |
1468 | * | | 1468 | * |
1469 | * Initialize a receive ring and its buffers. | | 1469 | * Initialize a receive ring and its buffers. |
1470 | ************************************************************************/ | | 1470 | ************************************************************************/ |
1471 | static int | | 1471 | static int |
1472 | ixgbe_setup_receive_ring(struct rx_ring *rxr) | | 1472 | ixgbe_setup_receive_ring(struct rx_ring *rxr) |
1473 | { | | 1473 | { |
1474 | struct adapter *adapter; | | 1474 | struct adapter *adapter; |
1475 | struct ixgbe_rx_buf *rxbuf; | | 1475 | struct ixgbe_rx_buf *rxbuf; |
1476 | #ifdef LRO | | 1476 | #ifdef LRO |
1477 | struct ifnet *ifp; | | 1477 | struct ifnet *ifp; |
1478 | struct lro_ctrl *lro = &rxr->lro; | | 1478 | struct lro_ctrl *lro = &rxr->lro; |
1479 | #endif /* LRO */ | | 1479 | #endif /* LRO */ |
1480 | #ifdef DEV_NETMAP | | 1480 | #ifdef DEV_NETMAP |
1481 | struct netmap_adapter *na = NA(rxr->adapter->ifp); | | 1481 | struct netmap_adapter *na = NA(rxr->adapter->ifp); |
1482 | struct netmap_slot *slot; | | 1482 | struct netmap_slot *slot; |
1483 | #endif /* DEV_NETMAP */ | | 1483 | #endif /* DEV_NETMAP */ |
1484 | int rsize, error = 0; | | 1484 | int rsize, error = 0; |
1485 | | | 1485 | |
1486 | adapter = rxr->adapter; | | 1486 | adapter = rxr->adapter; |
1487 | #ifdef LRO | | 1487 | #ifdef LRO |
1488 | ifp = adapter->ifp; | | 1488 | ifp = adapter->ifp; |
1489 | #endif /* LRO */ | | 1489 | #endif /* LRO */ |
1490 | | | 1490 | |
1491 | /* Clear the ring contents */ | | 1491 | /* Clear the ring contents */ |
1492 | IXGBE_RX_LOCK(rxr); | | 1492 | IXGBE_RX_LOCK(rxr); |
1493 | | | 1493 | |
1494 | #ifdef DEV_NETMAP | | 1494 | #ifdef DEV_NETMAP |
1495 | if (adapter->feat_en & IXGBE_FEATURE_NETMAP) | | 1495 | if (adapter->feat_en & IXGBE_FEATURE_NETMAP) |
1496 | slot = netmap_reset(na, NR_RX, rxr->me, 0); | | 1496 | slot = netmap_reset(na, NR_RX, rxr->me, 0); |
1497 | #endif /* DEV_NETMAP */ | | 1497 | #endif /* DEV_NETMAP */ |
1498 | | | 1498 | |
1499 | rsize = roundup2(adapter->num_rx_desc * | | 1499 | rsize = roundup2(adapter->num_rx_desc * |
1500 | sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); | | 1500 | sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); |
1501 | bzero((void *)rxr->rx_base, rsize); | | 1501 | bzero((void *)rxr->rx_base, rsize); |
1502 | /* Cache the size */ | | 1502 | /* Cache the size */ |
1503 | rxr->mbuf_sz = adapter->rx_mbuf_sz; | | 1503 | rxr->mbuf_sz = adapter->rx_mbuf_sz; |
1504 | | | 1504 | |
1505 | /* Free current RX buffer structs and their mbufs */ | | 1505 | /* Free current RX buffer structs and their mbufs */ |
1506 | ixgbe_free_receive_ring(rxr); | | 1506 | ixgbe_free_receive_ring(rxr); |
1507 | | | 1507 | |
1508 | IXGBE_RX_UNLOCK(rxr); | | 1508 | IXGBE_RX_UNLOCK(rxr); |
1509 | /* | | 1509 | /* |
1510 | * Now reinitialize our supply of jumbo mbufs. The number | | 1510 | * Now reinitialize our supply of jumbo mbufs. The number |
1511 | * or size of jumbo mbufs may have changed. | | 1511 | * or size of jumbo mbufs may have changed. |
1512 | * Assume all of rxr->ptag are the same. | | 1512 | * Assume all of rxr->ptag are the same. |
1513 | */ | | 1513 | */ |
1514 | ixgbe_jcl_reinit(adapter, rxr->ptag->dt_dmat, rxr, | | 1514 | ixgbe_jcl_reinit(adapter, rxr->ptag->dt_dmat, rxr, |
1515 | adapter->num_jcl, adapter->rx_mbuf_sz); | | 1515 | adapter->num_jcl, adapter->rx_mbuf_sz); |
1516 | | | 1516 | |
1517 | IXGBE_RX_LOCK(rxr); | | 1517 | IXGBE_RX_LOCK(rxr); |
1518 | | | 1518 | |
1519 | /* Now replenish the mbufs */ | | 1519 | /* Now replenish the mbufs */ |
1520 | for (int j = 0; j != rxr->num_desc; ++j) { | | 1520 | for (int j = 0; j != rxr->num_desc; ++j) { |
1521 | struct mbuf *mp; | | 1521 | struct mbuf *mp; |
1522 | | | 1522 | |
1523 | rxbuf = &rxr->rx_buffers[j]; | | 1523 | rxbuf = &rxr->rx_buffers[j]; |
1524 | | | 1524 | |
1525 | #ifdef DEV_NETMAP | | 1525 | #ifdef DEV_NETMAP |
1526 | /* | | 1526 | /* |
1527 | * In netmap mode, fill the map and set the buffer | | 1527 | * In netmap mode, fill the map and set the buffer |
1528 | * address in the NIC ring, considering the offset | | 1528 | * address in the NIC ring, considering the offset |
1529 | * between the netmap and NIC rings (see comment in | | 1529 | * between the netmap and NIC rings (see comment in |
1530 | * ixgbe_setup_transmit_ring() ). No need to allocate | | 1530 | * ixgbe_setup_transmit_ring() ). No need to allocate |
1531 | * an mbuf, so end the block with a continue; | | 1531 | * an mbuf, so end the block with a continue; |
1532 | */ | | 1532 | */ |
1533 | if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && slot) { | | 1533 | if ((adapter->feat_en & IXGBE_FEATURE_NETMAP) && slot) { |
1534 | int sj = netmap_idx_n2k(na->rx_rings[rxr->me], j); | | 1534 | int sj = netmap_idx_n2k(na->rx_rings[rxr->me], j); |
1535 | uint64_t paddr; | | 1535 | uint64_t paddr; |
1536 | void *addr; | | 1536 | void *addr; |
1537 | | | 1537 | |
1538 | addr = PNMB(na, slot + sj, &paddr); | | 1538 | addr = PNMB(na, slot + sj, &paddr); |
1539 | netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr); | | 1539 | netmap_load_map(na, rxr->ptag, rxbuf->pmap, addr); |
1540 | /* Update descriptor and the cached value */ | | 1540 | /* Update descriptor and the cached value */ |
1541 | rxr->rx_base[j].read.pkt_addr = htole64(paddr); | | 1541 | rxr->rx_base[j].read.pkt_addr = htole64(paddr); |
1542 | rxbuf->addr = htole64(paddr); | | 1542 | rxbuf->addr = htole64(paddr); |
1543 | continue; | | 1543 | continue; |
1544 | } | | 1544 | } |
1545 | #endif /* DEV_NETMAP */ | | 1545 | #endif /* DEV_NETMAP */ |
1546 | | | 1546 | |
1547 | rxbuf->flags = 0; | | 1547 | rxbuf->flags = 0; |
1548 | rxbuf->buf = ixgbe_getjcl(&rxr->jcl_head, M_NOWAIT, | | 1548 | rxbuf->buf = ixgbe_getjcl(&rxr->jcl_head, M_NOWAIT, |
1549 | MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); | | 1549 | MT_DATA, M_PKTHDR, adapter->rx_mbuf_sz); |
1550 | if (rxbuf->buf == NULL) { | | 1550 | if (rxbuf->buf == NULL) { |
1551 | rxr->no_jmbuf.ev_count++; | | 1551 | rxr->no_jmbuf.ev_count++; |
1552 | error = ENOBUFS; | | 1552 | error = ENOBUFS; |
1553 | goto fail; | | 1553 | goto fail; |
1554 | } | | 1554 | } |
1555 | mp = rxbuf->buf; | | 1555 | mp = rxbuf->buf; |
1556 | mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; | | 1556 | mp->m_pkthdr.len = mp->m_len = rxr->mbuf_sz; |
1557 | /* Get the memory mapping */ | | 1557 | /* Get the memory mapping */ |
1558 | error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, rxbuf->pmap, | | 1558 | error = bus_dmamap_load_mbuf(rxr->ptag->dt_dmat, rxbuf->pmap, |
1559 | mp, BUS_DMA_NOWAIT); | | 1559 | mp, BUS_DMA_NOWAIT); |
1560 | if (error != 0) { | | 1560 | if (error != 0) { |
1561 | /* | | 1561 | /* |
1562 | * Clear this entry for later cleanup in | | 1562 | * Clear this entry for later cleanup in |
1563 | * ixgbe_discard() which is called via | | 1563 | * ixgbe_discard() which is called via |
1564 | * ixgbe_free_receive_ring(). | | 1564 | * ixgbe_free_receive_ring(). |
1565 | */ | | 1565 | */ |
1566 | m_freem(mp); | | 1566 | m_freem(mp); |
1567 | rxbuf->buf = NULL; | | 1567 | rxbuf->buf = NULL; |
1568 | goto fail; | | 1568 | goto fail; |
1569 | } | | 1569 | } |
1570 | bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, | | 1570 | bus_dmamap_sync(rxr->ptag->dt_dmat, rxbuf->pmap, |
1571 | 0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD); | | 1571 | 0, adapter->rx_mbuf_sz, BUS_DMASYNC_PREREAD); |
1572 | /* Update the descriptor and the cached value */ | | 1572 | /* Update the descriptor and the cached value */ |
1573 | rxr->rx_base[j].read.pkt_addr = | | 1573 | rxr->rx_base[j].read.pkt_addr = |
1574 | htole64(rxbuf->pmap->dm_segs[0].ds_addr); | | 1574 | htole64(rxbuf->pmap->dm_segs[0].ds_addr); |
1575 | rxbuf->addr = htole64(rxbuf->pmap->dm_segs[0].ds_addr); | | 1575 | rxbuf->addr = htole64(rxbuf->pmap->dm_segs[0].ds_addr); |
1576 | } | | 1576 | } |
1577 | | | 1577 | |
1578 | /* Setup our descriptor indices */ | | 1578 | /* Setup our descriptor indices */ |
1579 | rxr->next_to_check = 0; | | 1579 | rxr->next_to_check = 0; |
1580 | rxr->next_to_refresh = 0; | | 1580 | rxr->next_to_refresh = adapter->num_rx_desc - 1; /* Fully allocated */ |
1581 | rxr->lro_enabled = FALSE; | | 1581 | rxr->lro_enabled = FALSE; |
1582 | rxr->rx_copies.ev_count = 0; | | 1582 | rxr->rx_copies.ev_count = 0; |
1583 | #if 0 /* NetBSD */ | | 1583 | #if 0 /* NetBSD */ |
1584 | rxr->rx_bytes.ev_count = 0; | | 1584 | rxr->rx_bytes.ev_count = 0; |
1585 | #if 1 /* Fix inconsistency */ | | 1585 | #if 1 /* Fix inconsistency */ |
1586 | rxr->rx_packets.ev_count = 0; | | 1586 | rxr->rx_packets.ev_count = 0; |
1587 | #endif | | 1587 | #endif |
1588 | #endif | | 1588 | #endif |
1589 | rxr->vtag_strip = FALSE; | | 1589 | rxr->vtag_strip = FALSE; |
1590 | | | 1590 | |
1591 | ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, | | 1591 | ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, |
1592 | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | | 1592 | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); |
1593 | | | 1593 | |
1594 | /* | | 1594 | /* |
1595 | * Now set up the LRO interface | | 1595 | * Now set up the LRO interface |
1596 | */ | | 1596 | */ |
1597 | if (ixgbe_rsc_enable) | | 1597 | if (ixgbe_rsc_enable) |
1598 | ixgbe_setup_hw_rsc(rxr); | | 1598 | ixgbe_setup_hw_rsc(rxr); |
1599 | #ifdef LRO | | 1599 | #ifdef LRO |
1600 | else if (ifp->if_capenable & IFCAP_LRO) { | | 1600 | else if (ifp->if_capenable & IFCAP_LRO) { |
1601 | device_t dev = adapter->dev; | | 1601 | device_t dev = adapter->dev; |
1602 | int err = tcp_lro_init(lro); | | 1602 | int err = tcp_lro_init(lro); |
1603 | if (err) { | | 1603 | if (err) { |
1604 | device_printf(dev, "LRO Initialization failed!\n"); | | 1604 | device_printf(dev, "LRO Initialization failed!\n"); |
1605 | goto fail; | | 1605 | goto fail; |
1606 | } | | 1606 | } |
1607 | INIT_DEBUGOUT("RX Soft LRO Initialized\n"); | | 1607 | INIT_DEBUGOUT("RX Soft LRO Initialized\n"); |
1608 | rxr->lro_enabled = TRUE; | | 1608 | rxr->lro_enabled = TRUE; |
1609 | lro->ifp = adapter->ifp; | | 1609 | lro->ifp = adapter->ifp; |
1610 | } | | 1610 | } |
1611 | #endif /* LRO */ | | 1611 | #endif /* LRO */ |
1612 | | | 1612 | |
1613 | IXGBE_RX_UNLOCK(rxr); | | 1613 | IXGBE_RX_UNLOCK(rxr); |
1614 | | | 1614 | |
1615 | return (0); | | 1615 | return (0); |
1616 | | | 1616 | |
1617 | fail: | | 1617 | fail: |
1618 | ixgbe_free_receive_ring(rxr); | | 1618 | ixgbe_free_receive_ring(rxr); |
1619 | IXGBE_RX_UNLOCK(rxr); | | 1619 | IXGBE_RX_UNLOCK(rxr); |
1620 | | | 1620 | |
1621 | return (error); | | 1621 | return (error); |
1622 | } /* ixgbe_setup_receive_ring */ | | 1622 | } /* ixgbe_setup_receive_ring */ |
1623 | | | 1623 | |
1624 | /************************************************************************ | | 1624 | /************************************************************************ |
1625 | * ixgbe_setup_receive_structures - Initialize all receive rings. | | 1625 | * ixgbe_setup_receive_structures - Initialize all receive rings. |
1626 | ************************************************************************/ | | 1626 | ************************************************************************/ |
1627 | int | | 1627 | int |
1628 | ixgbe_setup_receive_structures(struct adapter *adapter) | | 1628 | ixgbe_setup_receive_structures(struct adapter *adapter) |
1629 | { | | 1629 | { |
1630 | struct rx_ring *rxr = adapter->rx_rings; | | 1630 | struct rx_ring *rxr = adapter->rx_rings; |
1631 | int j; | | 1631 | int j; |
1632 | | | 1632 | |
1633 | INIT_DEBUGOUT("ixgbe_setup_receive_structures"); | | 1633 | INIT_DEBUGOUT("ixgbe_setup_receive_structures"); |
1634 | for (j = 0; j < adapter->num_queues; j++, rxr++) | | 1634 | for (j = 0; j < adapter->num_queues; j++, rxr++) |
1635 | if (ixgbe_setup_receive_ring(rxr)) | | 1635 | if (ixgbe_setup_receive_ring(rxr)) |
1636 | goto fail; | | 1636 | goto fail; |
1637 | | | 1637 | |
1638 | return (0); | | 1638 | return (0); |
1639 | fail: | | 1639 | fail: |
1640 | /* | | 1640 | /* |
1641 | * Free RX buffers allocated so far, we will only handle | | 1641 | * Free RX buffers allocated so far, we will only handle |
1642 | * the rings that completed, the failing case will have | | 1642 | * the rings that completed, the failing case will have |
1643 | * cleaned up for itself. 'j' failed, so its the terminus. | | 1643 | * cleaned up for itself. 'j' failed, so its the terminus. |
1644 | */ | | 1644 | */ |
1645 | for (int i = 0; i < j; ++i) { | | 1645 | for (int i = 0; i < j; ++i) { |
1646 | rxr = &adapter->rx_rings[i]; | | 1646 | rxr = &adapter->rx_rings[i]; |
1647 | IXGBE_RX_LOCK(rxr); | | 1647 | IXGBE_RX_LOCK(rxr); |
1648 | ixgbe_free_receive_ring(rxr); | | 1648 | ixgbe_free_receive_ring(rxr); |
1649 | IXGBE_RX_UNLOCK(rxr); | | 1649 | IXGBE_RX_UNLOCK(rxr); |
1650 | } | | 1650 | } |
1651 | | | 1651 | |
1652 | return (ENOBUFS); | | 1652 | return (ENOBUFS); |
1653 | } /* ixgbe_setup_receive_structures */ | | 1653 | } /* ixgbe_setup_receive_structures */ |
1654 | | | 1654 | |
1655 | | | 1655 | |
1656 | /************************************************************************ | | 1656 | /************************************************************************ |
1657 | * ixgbe_free_receive_structures - Free all receive rings. | | 1657 | * ixgbe_free_receive_structures - Free all receive rings. |
1658 | ************************************************************************/ | | 1658 | ************************************************************************/ |
1659 | void | | 1659 | void |
1660 | ixgbe_free_receive_structures(struct adapter *adapter) | | 1660 | ixgbe_free_receive_structures(struct adapter *adapter) |
1661 | { | | 1661 | { |
1662 | struct rx_ring *rxr = adapter->rx_rings; | | 1662 | struct rx_ring *rxr = adapter->rx_rings; |
1663 | | | 1663 | |
1664 | INIT_DEBUGOUT("ixgbe_free_receive_structures: begin"); | | 1664 | INIT_DEBUGOUT("ixgbe_free_receive_structures: begin"); |
1665 | | | 1665 | |
1666 | for (int i = 0; i < adapter->num_queues; i++, rxr++) { | | 1666 | for (int i = 0; i < adapter->num_queues; i++, rxr++) { |
1667 | ixgbe_free_receive_buffers(rxr); | | 1667 | ixgbe_free_receive_buffers(rxr); |
1668 | #ifdef LRO | | 1668 | #ifdef LRO |
1669 | /* Free LRO memory */ | | 1669 | /* Free LRO memory */ |
1670 | tcp_lro_free(&rxr->lro); | | 1670 | tcp_lro_free(&rxr->lro); |
1671 | #endif /* LRO */ | | 1671 | #endif /* LRO */ |
1672 | /* Free the ring memory as well */ | | 1672 | /* Free the ring memory as well */ |
1673 | ixgbe_dma_free(adapter, &rxr->rxdma); | | 1673 | ixgbe_dma_free(adapter, &rxr->rxdma); |
1674 | IXGBE_RX_LOCK_DESTROY(rxr); | | 1674 | IXGBE_RX_LOCK_DESTROY(rxr); |
1675 | } | | 1675 | } |
1676 | | | 1676 | |
1677 | free(adapter->rx_rings, M_DEVBUF); | | 1677 | free(adapter->rx_rings, M_DEVBUF); |
1678 | } /* ixgbe_free_receive_structures */ | | 1678 | } /* ixgbe_free_receive_structures */ |
1679 | | | 1679 | |
1680 | | | 1680 | |
1681 | /************************************************************************ | | 1681 | /************************************************************************ |
1682 | * ixgbe_free_receive_buffers - Free receive ring data structures | | 1682 | * ixgbe_free_receive_buffers - Free receive ring data structures |
1683 | ************************************************************************/ | | 1683 | ************************************************************************/ |
1684 | static void | | 1684 | static void |
1685 | ixgbe_free_receive_buffers(struct rx_ring *rxr) | | 1685 | ixgbe_free_receive_buffers(struct rx_ring *rxr) |
1686 | { | | 1686 | { |
1687 | struct adapter *adapter = rxr->adapter; | | 1687 | struct adapter *adapter = rxr->adapter; |
1688 | struct ixgbe_rx_buf *rxbuf; | | 1688 | struct ixgbe_rx_buf *rxbuf; |
1689 | | | 1689 | |
1690 | INIT_DEBUGOUT("ixgbe_free_receive_buffers: begin"); | | 1690 | INIT_DEBUGOUT("ixgbe_free_receive_buffers: begin"); |
1691 | | | 1691 | |
1692 | /* Cleanup any existing buffers */ | | 1692 | /* Cleanup any existing buffers */ |
1693 | if (rxr->rx_buffers != NULL) { | | 1693 | if (rxr->rx_buffers != NULL) { |
1694 | for (int i = 0; i < adapter->num_rx_desc; i++) { | | 1694 | for (int i = 0; i < adapter->num_rx_desc; i++) { |
1695 | rxbuf = &rxr->rx_buffers[i]; | | 1695 | rxbuf = &rxr->rx_buffers[i]; |
1696 | ixgbe_rx_discard(rxr, i); | | 1696 | ixgbe_rx_discard(rxr, i); |
1697 | if (rxbuf->pmap != NULL) { | | 1697 | if (rxbuf->pmap != NULL) { |
1698 | ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap); | | 1698 | ixgbe_dmamap_destroy(rxr->ptag, rxbuf->pmap); |
1699 | rxbuf->pmap = NULL; | | 1699 | rxbuf->pmap = NULL; |
1700 | } | | 1700 | } |
1701 | } | | 1701 | } |
1702 | | | 1702 | |
1703 | /* NetBSD specific. See ixgbe_netbsd.c */ | | 1703 | /* NetBSD specific. See ixgbe_netbsd.c */ |
1704 | ixgbe_jcl_destroy(adapter, rxr); | | 1704 | ixgbe_jcl_destroy(adapter, rxr); |
1705 | | | 1705 | |
1706 | if (rxr->rx_buffers != NULL) { | | 1706 | if (rxr->rx_buffers != NULL) { |
1707 | free(rxr->rx_buffers, M_DEVBUF); | | 1707 | free(rxr->rx_buffers, M_DEVBUF); |
1708 | rxr->rx_buffers = NULL; | | 1708 | rxr->rx_buffers = NULL; |
1709 | } | | 1709 | } |
1710 | } | | 1710 | } |
1711 | | | 1711 | |
1712 | if (rxr->ptag != NULL) { | | 1712 | if (rxr->ptag != NULL) { |
1713 | ixgbe_dma_tag_destroy(rxr->ptag); | | 1713 | ixgbe_dma_tag_destroy(rxr->ptag); |
1714 | rxr->ptag = NULL; | | 1714 | rxr->ptag = NULL; |
1715 | } | | 1715 | } |
1716 | | | 1716 | |
1717 | return; | | 1717 | return; |
1718 | } /* ixgbe_free_receive_buffers */ | | 1718 | } /* ixgbe_free_receive_buffers */ |
1719 | | | 1719 | |
1720 | /************************************************************************ | | 1720 | /************************************************************************ |
1721 | * ixgbe_rx_input | | 1721 | * ixgbe_rx_input |
1722 | ************************************************************************/ | | 1722 | ************************************************************************/ |
1723 | static __inline void | | 1723 | static __inline void |
1724 | ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, | | 1724 | ixgbe_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, |
1725 | u32 ptype) | | 1725 | u32 ptype) |
1726 | { | | 1726 | { |
1727 | struct adapter *adapter = ifp->if_softc; | | 1727 | struct adapter *adapter = ifp->if_softc; |
1728 | | | 1728 | |
1729 | #ifdef LRO | | 1729 | #ifdef LRO |
1730 | struct ethercom *ec = &adapter->osdep.ec; | | 1730 | struct ethercom *ec = &adapter->osdep.ec; |
1731 | | | 1731 | |
1732 | /* | | 1732 | /* |
1733 | * ATM LRO is only for IP/TCP packets and TCP checksum of the packet | | 1733 | * ATM LRO is only for IP/TCP packets and TCP checksum of the packet |
1734 | * should be computed by hardware. Also it should not have VLAN tag in | | 1734 | * should be computed by hardware. Also it should not have VLAN tag in |
1735 | * ethernet header. In case of IPv6 we do not yet support ext. hdrs. | | 1735 | * ethernet header. In case of IPv6 we do not yet support ext. hdrs. |
1736 | */ | | 1736 | */ |
1737 | if (rxr->lro_enabled && | | 1737 | if (rxr->lro_enabled && |
1738 | (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) != 0 && | | 1738 | (ec->ec_capenable & ETHERCAP_VLAN_HWTAGGING) != 0 && |
1739 | (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && | | 1739 | (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && |
1740 | ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == | | 1740 | ((ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == |
1741 | (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) || | | 1741 | (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) || |
1742 | (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) == | | 1742 | (ptype & (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) == |
1743 | (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) && | | 1743 | (IXGBE_RXDADV_PKTTYPE_IPV6 | IXGBE_RXDADV_PKTTYPE_TCP)) && |
1744 | (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == | | 1744 | (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == |
1745 | (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { | | 1745 | (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { |
1746 | /* | | 1746 | /* |
1747 | * Send to the stack if: | | 1747 | * Send to the stack if: |
1748 | ** - LRO not enabled, or | | 1748 | ** - LRO not enabled, or |
1749 | ** - no LRO resources, or | | 1749 | ** - no LRO resources, or |
1750 | ** - lro enqueue fails | | 1750 | ** - lro enqueue fails |
1751 | */ | | 1751 | */ |
1752 | if (rxr->lro.lro_cnt != 0) | | 1752 | if (rxr->lro.lro_cnt != 0) |
1753 | if (tcp_lro_rx(&rxr->lro, m, 0) == 0) | | 1753 | if (tcp_lro_rx(&rxr->lro, m, 0) == 0) |
1754 | return; | | 1754 | return; |
1755 | } | | 1755 | } |
1756 | #endif /* LRO */ | | 1756 | #endif /* LRO */ |
1757 | | | 1757 | |
1758 | if_percpuq_enqueue(adapter->ipq, m); | | 1758 | if_percpuq_enqueue(adapter->ipq, m); |
1759 | } /* ixgbe_rx_input */ | | 1759 | } /* ixgbe_rx_input */ |
1760 | | | 1760 | |
1761 | /************************************************************************ | | 1761 | /************************************************************************ |
1762 | * ixgbe_rx_discard | | 1762 | * ixgbe_rx_discard |
1763 | ************************************************************************/ | | 1763 | ************************************************************************/ |
1764 | static __inline void | | 1764 | static __inline void |
1765 | ixgbe_rx_discard(struct rx_ring *rxr, int i) | | 1765 | ixgbe_rx_discard(struct rx_ring *rxr, int i) |
1766 | { | | 1766 | { |
1767 | struct ixgbe_rx_buf *rbuf; | | 1767 | struct ixgbe_rx_buf *rbuf; |
1768 | | | 1768 | |
1769 | rbuf = &rxr->rx_buffers[i]; | | 1769 | rbuf = &rxr->rx_buffers[i]; |
1770 | | | 1770 | |
1771 | /* | | 1771 | /* |
1772 | * With advanced descriptors the writeback clobbers the buffer addrs, | | 1772 | * With advanced descriptors the writeback clobbers the buffer addrs, |
1773 | * so its easier to just free the existing mbufs and take the normal | | 1773 | * so its easier to just free the existing mbufs and take the normal |
1774 | * refresh path to get new buffers and mapping. | | 1774 | * refresh path to get new buffers and mapping. |
1775 | */ | | 1775 | */ |
1776 | | | 1776 | |
1777 | if (rbuf->fmp != NULL) {/* Partial chain ? */ | | 1777 | if (rbuf->fmp != NULL) {/* Partial chain ? */ |
1778 | bus_dmamap_sync(rxr->ptag->dt_dmat, rbuf->pmap, 0, | | 1778 | bus_dmamap_sync(rxr->ptag->dt_dmat, rbuf->pmap, 0, |
1779 | rbuf->buf->m_pkthdr.len, BUS_DMASYNC_POSTREAD); | | 1779 | rbuf->buf->m_pkthdr.len, BUS_DMASYNC_POSTREAD); |
1780 | ixgbe_dmamap_unload(rxr->ptag, rbuf->pmap); | | 1780 | ixgbe_dmamap_unload(rxr->ptag, rbuf->pmap); |
1781 | m_freem(rbuf->fmp); | | 1781 | m_freem(rbuf->fmp); |
1782 | rbuf->fmp = NULL; | | 1782 | rbuf->fmp = NULL; |
1783 | rbuf->buf = NULL; /* rbuf->buf is part of fmp's chain */ | | 1783 | rbuf->buf = NULL; /* rbuf->buf is part of fmp's chain */ |
1784 | } else if (rbuf->buf) { | | 1784 | } else if (rbuf->buf) { |
1785 | bus_dmamap_sync(rxr->ptag->dt_dmat, rbuf->pmap, 0, | | 1785 | bus_dmamap_sync(rxr->ptag->dt_dmat, rbuf->pmap, 0, |
1786 | rbuf->buf->m_pkthdr.len, BUS_DMASYNC_POSTREAD); | | 1786 | rbuf->buf->m_pkthdr.len, BUS_DMASYNC_POSTREAD); |
1787 | ixgbe_dmamap_unload(rxr->ptag, rbuf->pmap); | | 1787 | ixgbe_dmamap_unload(rxr->ptag, rbuf->pmap); |
1788 | m_free(rbuf->buf); | | 1788 | m_free(rbuf->buf); |
1789 | rbuf->buf = NULL; | | 1789 | rbuf->buf = NULL; |
1790 | } | | 1790 | } |
1791 | | | 1791 | |
1792 | rbuf->flags = 0; | | 1792 | rbuf->flags = 0; |
1793 | | | 1793 | |
1794 | return; | | 1794 | return; |
1795 | } /* ixgbe_rx_discard */ | | 1795 | } /* ixgbe_rx_discard */ |
1796 | | | 1796 | |
1797 | | | 1797 | |
1798 | /************************************************************************ | | 1798 | /************************************************************************ |
1799 | * ixgbe_rxeof | | 1799 | * ixgbe_rxeof |
1800 | * | | 1800 | * |
1801 | * Executes in interrupt context. It replenishes the | | 1801 | * Executes in interrupt context. It replenishes the |
1802 | * mbufs in the descriptor and sends data which has | | 1802 | * mbufs in the descriptor and sends data which has |
1803 | * been dma'ed into host memory to upper layer. | | 1803 | * been dma'ed into host memory to upper layer. |
1804 | * | | 1804 | * |
1805 | * Return TRUE for more work, FALSE for all clean. | | 1805 | * Return TRUE for more work, FALSE for all clean. |
1806 | ************************************************************************/ | | 1806 | ************************************************************************/ |
1807 | bool | | 1807 | bool |
1808 | ixgbe_rxeof(struct ix_queue *que) | | 1808 | ixgbe_rxeof(struct ix_queue *que) |
1809 | { | | 1809 | { |
1810 | struct adapter *adapter = que->adapter; | | 1810 | struct adapter *adapter = que->adapter; |
1811 | struct rx_ring *rxr = que->rxr; | | 1811 | struct rx_ring *rxr = que->rxr; |
1812 | struct ifnet *ifp = adapter->ifp; | | 1812 | struct ifnet *ifp = adapter->ifp; |
1813 | #ifdef LRO | | 1813 | #ifdef LRO |
1814 | struct lro_ctrl *lro = &rxr->lro; | | 1814 | struct lro_ctrl *lro = &rxr->lro; |
1815 | #endif /* LRO */ | | 1815 | #endif /* LRO */ |
1816 | union ixgbe_adv_rx_desc *cur; | | 1816 | union ixgbe_adv_rx_desc *cur; |
1817 | struct ixgbe_rx_buf *rbuf, *nbuf; | | 1817 | struct ixgbe_rx_buf *rbuf, *nbuf; |
1818 | int i, nextp, processed = 0; | | 1818 | int i, nextp, processed = 0; |
1819 | u32 staterr = 0; | | 1819 | u32 staterr = 0; |
1820 | u32 count = 0; | | 1820 | u32 count = 0; |
1821 | u32 limit = adapter->rx_process_limit; | | 1821 | u32 limit = adapter->rx_process_limit; |
1822 | bool discard_multidesc = false; | | 1822 | bool discard_multidesc = false; |
1823 | #ifdef RSS | | 1823 | #ifdef RSS |
1824 | u16 pkt_info; | | 1824 | u16 pkt_info; |
1825 | #endif | | 1825 | #endif |
1826 | | | 1826 | |
1827 | IXGBE_RX_LOCK(rxr); | | 1827 | IXGBE_RX_LOCK(rxr); |
1828 | | | 1828 | |
1829 | #ifdef DEV_NETMAP | | 1829 | #ifdef DEV_NETMAP |
1830 | if (adapter->feat_en & IXGBE_FEATURE_NETMAP) { | | 1830 | if (adapter->feat_en & IXGBE_FEATURE_NETMAP) { |
1831 | /* Same as the txeof routine: wakeup clients on intr. */ | | 1831 | /* Same as the txeof routine: wakeup clients on intr. */ |
1832 | if (netmap_rx_irq(ifp, rxr->me, &processed)) { | | 1832 | if (netmap_rx_irq(ifp, rxr->me, &processed)) { |
1833 | IXGBE_RX_UNLOCK(rxr); | | 1833 | IXGBE_RX_UNLOCK(rxr); |
1834 | return (FALSE); | | 1834 | return (FALSE); |
1835 | } | | 1835 | } |
1836 | } | | 1836 | } |
1837 | #endif /* DEV_NETMAP */ | | 1837 | #endif /* DEV_NETMAP */ |
1838 | | | 1838 | |
1839 | /* | | 1839 | /* |
1840 | * The max number of loop is rx_process_limit. If discard_multidesc is | | 1840 | * The max number of loop is rx_process_limit. If discard_multidesc is |
1841 | * true, continue processing to not to send broken packet to the upper | | 1841 | * true, continue processing to not to send broken packet to the upper |
1842 | * layer. | | 1842 | * layer. |
1843 | */ | | 1843 | */ |
1844 | for (i = rxr->next_to_check; | | 1844 | for (i = rxr->next_to_check; |
1845 | (count < limit) || (discard_multidesc == true);) { | | 1845 | (count < limit) || (discard_multidesc == true);) { |
1846 | | | 1846 | |
1847 | struct mbuf *sendmp, *mp; | | 1847 | struct mbuf *sendmp, *mp; |
1848 | struct mbuf *newmp; | | 1848 | struct mbuf *newmp; |
1849 | u32 rsc, ptype; | | 1849 | u32 rsc, ptype; |
1850 | u16 len; | | 1850 | u16 len; |
1851 | u16 vtag = 0; | | 1851 | u16 vtag = 0; |
1852 | bool eop; | | 1852 | bool eop; |
1853 | | | 1853 | |
1854 | /* Sync the ring. */ | | 1854 | /* Sync the ring. */ |
1855 | ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, | | 1855 | ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, |
1856 | BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | | 1856 | BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); |
1857 | | | 1857 | |
1858 | cur = &rxr->rx_base[i]; | | 1858 | cur = &rxr->rx_base[i]; |
1859 | staterr = le32toh(cur->wb.upper.status_error); | | 1859 | staterr = le32toh(cur->wb.upper.status_error); |
1860 | #ifdef RSS | | 1860 | #ifdef RSS |
1861 | pkt_info = le16toh(cur->wb.lower.lo_dword.hs_rss.pkt_info); | | 1861 | pkt_info = le16toh(cur->wb.lower.lo_dword.hs_rss.pkt_info); |
1862 | #endif | | 1862 | #endif |
1863 | | | 1863 | |
1864 | if ((staterr & IXGBE_RXD_STAT_DD) == 0) | | 1864 | if ((staterr & IXGBE_RXD_STAT_DD) == 0) |
1865 | break; | | 1865 | break; |
1866 | | | 1866 | |
1867 | count++; | | 1867 | count++; |
1868 | sendmp = NULL; | | 1868 | sendmp = NULL; |
1869 | nbuf = NULL; | | 1869 | nbuf = NULL; |
1870 | rsc = 0; | | 1870 | rsc = 0; |
1871 | cur->wb.upper.status_error = 0; | | 1871 | cur->wb.upper.status_error = 0; |
1872 | rbuf = &rxr->rx_buffers[i]; | | 1872 | rbuf = &rxr->rx_buffers[i]; |
1873 | mp = rbuf->buf; | | 1873 | mp = rbuf->buf; |
1874 | | | 1874 | |
1875 | len = le16toh(cur->wb.upper.length); | | 1875 | len = le16toh(cur->wb.upper.length); |
1876 | ptype = le32toh(cur->wb.lower.lo_dword.data) & | | 1876 | ptype = le32toh(cur->wb.lower.lo_dword.data) & |
1877 | IXGBE_RXDADV_PKTTYPE_MASK; | | 1877 | IXGBE_RXDADV_PKTTYPE_MASK; |
1878 | eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); | | 1878 | eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); |
1879 | | | 1879 | |
1880 | /* Make sure bad packets are discarded */ | | 1880 | /* Make sure bad packets are discarded */ |
1881 | if (eop && (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) { | | 1881 | if (eop && (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) { |
1882 | #if __FreeBSD_version >= 1100036 | | 1882 | #if __FreeBSD_version >= 1100036 |
1883 | if (adapter->feat_en & IXGBE_FEATURE_VF) | | 1883 | if (adapter->feat_en & IXGBE_FEATURE_VF) |
1884 | if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); | | 1884 | if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); |
1885 | #endif | | 1885 | #endif |
1886 | rxr->rx_discarded.ev_count++; | | 1886 | rxr->rx_discarded.ev_count++; |
1887 | ixgbe_rx_discard(rxr, i); | | 1887 | ixgbe_rx_discard(rxr, i); |
1888 | discard_multidesc = false; | | 1888 | discard_multidesc = false; |
1889 | goto next_desc; | | 1889 | goto next_desc; |
1890 | } | | 1890 | } |
1891 | | | 1891 | |
1892 | /* pre-alloc new mbuf */ | | 1892 | /* pre-alloc new mbuf */ |
1893 | if (!discard_multidesc) | | 1893 | if (!discard_multidesc) |
1894 | newmp = ixgbe_getjcl(&rxr->jcl_head, M_NOWAIT, MT_DATA, | | 1894 | newmp = ixgbe_getjcl(&rxr->jcl_head, M_NOWAIT, MT_DATA, |
1895 | M_PKTHDR, rxr->mbuf_sz); | | 1895 | M_PKTHDR, rxr->mbuf_sz); |
1896 | else | | 1896 | else |
1897 | newmp = NULL; | | 1897 | newmp = NULL; |
1898 | if (newmp == NULL) { | | 1898 | if (newmp == NULL) { |
1899 | rxr->no_jmbuf.ev_count++; | | 1899 | rxr->no_jmbuf.ev_count++; |
1900 | /* | | 1900 | /* |
1901 | * Descriptor initialization is already done by the | | 1901 | * Descriptor initialization is already done by the |
1902 | * above code (cur->wb.upper.status_error = 0). | | 1902 | * above code (cur->wb.upper.status_error = 0). |
1903 | * So, we can reuse current rbuf->buf for new packet. | | 1903 | * So, we can reuse current rbuf->buf for new packet. |
1904 | * | | 1904 | * |
1905 | * Rewrite the buffer addr, see comment in | | 1905 | * Rewrite the buffer addr, see comment in |
1906 | * ixgbe_rx_discard(). | | 1906 | * ixgbe_rx_discard(). |
1907 | */ | | 1907 | */ |
1908 | cur->read.pkt_addr = rbuf->addr; | | 1908 | cur->read.pkt_addr = rbuf->addr; |
1909 | m_freem(rbuf->fmp); | | 1909 | m_freem(rbuf->fmp); |
1910 | rbuf->fmp = NULL; | | 1910 | rbuf->fmp = NULL; |
1911 | if (!eop) { | | 1911 | if (!eop) { |
1912 | /* Discard the entire packet. */ | | 1912 | /* Discard the entire packet. */ |
1913 | discard_multidesc = true; | | 1913 | discard_multidesc = true; |
1914 | } else | | 1914 | } else |
1915 | discard_multidesc = false; | | 1915 | discard_multidesc = false; |
1916 | goto next_desc; | | 1916 | goto next_desc; |
1917 | } | | 1917 | } |
1918 | discard_multidesc = false; | | 1918 | discard_multidesc = false; |
1919 | | | 1919 | |
1920 | bus_dmamap_sync(rxr->ptag->dt_dmat, rbuf->pmap, 0, | | 1920 | bus_dmamap_sync(rxr->ptag->dt_dmat, rbuf->pmap, 0, |
1921 | rbuf->buf->m_pkthdr.len, BUS_DMASYNC_POSTREAD); | | 1921 | rbuf->buf->m_pkthdr.len, BUS_DMASYNC_POSTREAD); |
1922 | | | 1922 | |
1923 | /* | | 1923 | /* |
1924 | * On 82599 which supports a hardware | | 1924 | * On 82599 which supports a hardware |
1925 | * LRO (called HW RSC), packets need | | 1925 | * LRO (called HW RSC), packets need |
1926 | * not be fragmented across sequential | | 1926 | * not be fragmented across sequential |
1927 | * descriptors, rather the next descriptor | | 1927 | * descriptors, rather the next descriptor |
1928 | * is indicated in bits of the descriptor. | | 1928 | * is indicated in bits of the descriptor. |
1929 | * This also means that we might proceses | | 1929 | * This also means that we might proceses |
1930 | * more than one packet at a time, something | | 1930 | * more than one packet at a time, something |
1931 | * that has never been true before, it | | 1931 | * that has never been true before, it |
1932 | * required eliminating global chain pointers | | 1932 | * required eliminating global chain pointers |
1933 | * in favor of what we are doing here. -jfv | | 1933 | * in favor of what we are doing here. -jfv |
1934 | */ | | 1934 | */ |
1935 | if (!eop) { | | 1935 | if (!eop) { |
1936 | /* | | 1936 | /* |
1937 | * Figure out the next descriptor | | 1937 | * Figure out the next descriptor |
1938 | * of this frame. | | 1938 | * of this frame. |
1939 | */ | | 1939 | */ |
1940 | if (rxr->hw_rsc == TRUE) { | | 1940 | if (rxr->hw_rsc == TRUE) { |
1941 | rsc = ixgbe_rsc_count(cur); | | 1941 | rsc = ixgbe_rsc_count(cur); |
1942 | rxr->rsc_num += (rsc - 1); | | 1942 | rxr->rsc_num += (rsc - 1); |
1943 | } | | 1943 | } |
1944 | if (rsc) { /* Get hardware index */ | | 1944 | if (rsc) { /* Get hardware index */ |
1945 | nextp = ((staterr & IXGBE_RXDADV_NEXTP_MASK) >> | | 1945 | nextp = ((staterr & IXGBE_RXDADV_NEXTP_MASK) >> |
1946 | IXGBE_RXDADV_NEXTP_SHIFT); | | 1946 | IXGBE_RXDADV_NEXTP_SHIFT); |
1947 | } else { /* Just sequential */ | | 1947 | } else { /* Just sequential */ |
1948 | nextp = i + 1; | | 1948 | nextp = i + 1; |
1949 | if (nextp == adapter->num_rx_desc) | | 1949 | if (nextp == adapter->num_rx_desc) |
1950 | nextp = 0; | | 1950 | nextp = 0; |
1951 | } | | 1951 | } |
1952 | nbuf = &rxr->rx_buffers[nextp]; | | 1952 | nbuf = &rxr->rx_buffers[nextp]; |
1953 | prefetch(nbuf); | | 1953 | prefetch(nbuf); |
1954 | } | | 1954 | } |
1955 | /* | | 1955 | /* |
1956 | * Rather than using the fmp/lmp global pointers | | 1956 | * Rather than using the fmp/lmp global pointers |
1957 | * we now keep the head of a packet chain in the | | 1957 | * we now keep the head of a packet chain in the |
1958 | * buffer struct and pass this along from one | | 1958 | * buffer struct and pass this along from one |
1959 | * descriptor to the next, until we get EOP. | | 1959 | * descriptor to the next, until we get EOP. |
1960 | */ | | 1960 | */ |
1961 | /* | | 1961 | /* |
1962 | * See if there is a stored head | | 1962 | * See if there is a stored head |
1963 | * that determines what we are | | 1963 | * that determines what we are |
1964 | */ | | 1964 | */ |
1965 | sendmp = rbuf->fmp; | | 1965 | sendmp = rbuf->fmp; |
1966 | if (sendmp != NULL) { /* secondary frag */ | | 1966 | if (sendmp != NULL) { /* secondary frag */ |
1967 | rbuf->buf = newmp; | | 1967 | rbuf->buf = newmp; |
1968 | rbuf->fmp = NULL; | | 1968 | rbuf->fmp = NULL; |
1969 | mp->m_len = len; | | 1969 | mp->m_len = len; |
1970 | mp->m_flags &= ~M_PKTHDR; | | 1970 | mp->m_flags &= ~M_PKTHDR; |
1971 | sendmp->m_pkthdr.len += mp->m_len; | | 1971 | sendmp->m_pkthdr.len += mp->m_len; |
1972 | } else { | | 1972 | } else { |
1973 | /* | | 1973 | /* |
1974 | * Optimize. This might be a small packet, | | 1974 | * Optimize. This might be a small packet, |
1975 | * maybe just a TCP ACK. Do a fast copy that | | 1975 | * maybe just a TCP ACK. Do a fast copy that |
1976 | * is cache aligned into a new mbuf, and | | 1976 | * is cache aligned into a new mbuf, and |
1977 | * leave the old mbuf+cluster for re-use. | | 1977 | * leave the old mbuf+cluster for re-use. |
1978 | */ | | 1978 | */ |
1979 | if (eop && len <= IXGBE_RX_COPY_LEN) { | | 1979 | if (eop && len <= IXGBE_RX_COPY_LEN) { |
1980 | sendmp = m_gethdr(M_NOWAIT, MT_DATA); | | 1980 | sendmp = m_gethdr(M_NOWAIT, MT_DATA); |
1981 | if (sendmp != NULL) { | | 1981 | if (sendmp != NULL) { |
1982 | sendmp->m_data += IXGBE_RX_COPY_ALIGN; | | 1982 | sendmp->m_data += IXGBE_RX_COPY_ALIGN; |
1983 | ixgbe_bcopy(mp->m_data, sendmp->m_data, | | 1983 | ixgbe_bcopy(mp->m_data, sendmp->m_data, |
1984 | len); | | 1984 | len); |
1985 | sendmp->m_len = len; | | 1985 | sendmp->m_len = len; |
1986 | rxr->rx_copies.ev_count++; | | 1986 | rxr->rx_copies.ev_count++; |
1987 | rbuf->flags |= IXGBE_RX_COPY; | | 1987 | rbuf->flags |= IXGBE_RX_COPY; |
1988 | | | 1988 | |
1989 | m_freem(newmp); | | 1989 | m_freem(newmp); |
1990 | } | | 1990 | } |
1991 | } | | 1991 | } |
1992 | if (sendmp == NULL) { | | 1992 | if (sendmp == NULL) { |
1993 | rbuf->buf = newmp; | | 1993 | rbuf->buf = newmp; |
1994 | rbuf->fmp = NULL; | | 1994 | rbuf->fmp = NULL; |
1995 | mp->m_len = len; | | 1995 | mp->m_len = len; |
1996 | sendmp = mp; | | 1996 | sendmp = mp; |
1997 | } | | 1997 | } |
1998 | | | 1998 | |
1999 | /* first desc of a non-ps chain */ | | 1999 | /* first desc of a non-ps chain */ |
2000 | sendmp->m_flags |= M_PKTHDR; | | 2000 | sendmp->m_flags |= M_PKTHDR; |
2001 | sendmp->m_pkthdr.len = len; | | 2001 | sendmp->m_pkthdr.len = len; |
2002 | } | | 2002 | } |
2003 | ++processed; | | 2003 | ++processed; |
2004 | | | 2004 | |
2005 | /* Pass the head pointer on */ | | 2005 | /* Pass the head pointer on */ |
2006 | if (eop == 0) { | | 2006 | if (eop == 0) { |
2007 | nbuf->fmp = sendmp; | | 2007 | nbuf->fmp = sendmp; |
2008 | sendmp = NULL; | | 2008 | sendmp = NULL; |
2009 | mp->m_next = nbuf->buf; | | 2009 | mp->m_next = nbuf->buf; |
2010 | } else { /* Sending this frame */ | | 2010 | } else { /* Sending this frame */ |
2011 | m_set_rcvif(sendmp, ifp); | | 2011 | m_set_rcvif(sendmp, ifp); |
2012 | ++rxr->packets; | | 2012 | ++rxr->packets; |
2013 | rxr->rx_packets.ev_count++; | | 2013 | rxr->rx_packets.ev_count++; |
2014 | /* capture data for AIM */ | | 2014 | /* capture data for AIM */ |
2015 | rxr->bytes += sendmp->m_pkthdr.len; | | 2015 | rxr->bytes += sendmp->m_pkthdr.len; |
2016 | rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len; | | 2016 | rxr->rx_bytes.ev_count += sendmp->m_pkthdr.len; |
2017 | /* Process vlan info */ | | 2017 | /* Process vlan info */ |
2018 | if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP)) | | 2018 | if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP)) |
2019 | vtag = le16toh(cur->wb.upper.vlan); | | 2019 | vtag = le16toh(cur->wb.upper.vlan); |
2020 | if (vtag) { | | 2020 | if (vtag) { |
2021 | vlan_set_tag(sendmp, vtag); | | 2021 | vlan_set_tag(sendmp, vtag); |
2022 | } | | 2022 | } |
2023 | if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { | | 2023 | if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { |
2024 | ixgbe_rx_checksum(staterr, sendmp, ptype, | | 2024 | ixgbe_rx_checksum(staterr, sendmp, ptype, |
2025 | &adapter->stats.pf); | | 2025 | &adapter->stats.pf); |
2026 | } | | 2026 | } |
2027 | | | 2027 | |
2028 | #if 0 /* FreeBSD */ | | 2028 | #if 0 /* FreeBSD */ |
2029 | /* | | 2029 | /* |
2030 | * In case of multiqueue, we have RXCSUM.PCSD bit set | | 2030 | * In case of multiqueue, we have RXCSUM.PCSD bit set |
2031 | * and never cleared. This means we have RSS hash | | 2031 | * and never cleared. This means we have RSS hash |
2032 | * available to be used. | | 2032 | * available to be used. |
2033 | */ | | 2033 | */ |
2034 | if (adapter->num_queues > 1) { | | 2034 | if (adapter->num_queues > 1) { |
2035 | sendmp->m_pkthdr.flowid = | | 2035 | sendmp->m_pkthdr.flowid = |
2036 | le32toh(cur->wb.lower.hi_dword.rss); | | 2036 | le32toh(cur->wb.lower.hi_dword.rss); |
2037 | switch (pkt_info & IXGBE_RXDADV_RSSTYPE_MASK) { | | 2037 | switch (pkt_info & IXGBE_RXDADV_RSSTYPE_MASK) { |
2038 | case IXGBE_RXDADV_RSSTYPE_IPV4: | | 2038 | case IXGBE_RXDADV_RSSTYPE_IPV4: |
2039 | M_HASHTYPE_SET(sendmp, | | 2039 | M_HASHTYPE_SET(sendmp, |
2040 | M_HASHTYPE_RSS_IPV4); | | 2040 | M_HASHTYPE_RSS_IPV4); |
2041 | break; | | 2041 | break; |
2042 | case IXGBE_RXDADV_RSSTYPE_IPV4_TCP: | | 2042 | case IXGBE_RXDADV_RSSTYPE_IPV4_TCP: |
2043 | M_HASHTYPE_SET(sendmp, | | 2043 | M_HASHTYPE_SET(sendmp, |
2044 | M_HASHTYPE_RSS_TCP_IPV4); | | 2044 | M_HASHTYPE_RSS_TCP_IPV4); |
2045 | break; | | 2045 | break; |
2046 | case IXGBE_RXDADV_RSSTYPE_IPV6: | | 2046 | case IXGBE_RXDADV_RSSTYPE_IPV6: |
2047 | M_HASHTYPE_SET(sendmp, | | 2047 | M_HASHTYPE_SET(sendmp, |
2048 | M_HASHTYPE_RSS_IPV6); | | 2048 | M_HASHTYPE_RSS_IPV6); |
2049 | break; | | 2049 | break; |
2050 | case IXGBE_RXDADV_RSSTYPE_IPV6_TCP: | | 2050 | case IXGBE_RXDADV_RSSTYPE_IPV6_TCP: |
2051 | M_HASHTYPE_SET(sendmp, | | 2051 | M_HASHTYPE_SET(sendmp, |
2052 | M_HASHTYPE_RSS_TCP_IPV6); | | 2052 | M_HASHTYPE_RSS_TCP_IPV6); |
2053 | break; | | 2053 | break; |
2054 | case IXGBE_RXDADV_RSSTYPE_IPV6_EX: | | 2054 | case IXGBE_RXDADV_RSSTYPE_IPV6_EX: |
2055 | M_HASHTYPE_SET(sendmp, | | 2055 | M_HASHTYPE_SET(sendmp, |
2056 | M_HASHTYPE_RSS_IPV6_EX); | | 2056 | M_HASHTYPE_RSS_IPV6_EX); |
2057 | break; | | 2057 | break; |
2058 | case IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX: | | 2058 | case IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX: |
2059 | M_HASHTYPE_SET(sendmp, | | 2059 | M_HASHTYPE_SET(sendmp, |
2060 | M_HASHTYPE_RSS_TCP_IPV6_EX); | | 2060 | M_HASHTYPE_RSS_TCP_IPV6_EX); |
2061 | break; | | 2061 | break; |
2062 | #if __FreeBSD_version > 1100000 | | 2062 | #if __FreeBSD_version > 1100000 |
2063 | case IXGBE_RXDADV_RSSTYPE_IPV4_UDP: | | 2063 | case IXGBE_RXDADV_RSSTYPE_IPV4_UDP: |
2064 | M_HASHTYPE_SET(sendmp, | | 2064 | M_HASHTYPE_SET(sendmp, |
2065 | M_HASHTYPE_RSS_UDP_IPV4); | | 2065 | M_HASHTYPE_RSS_UDP_IPV4); |
2066 | break; | | 2066 | break; |
2067 | case IXGBE_RXDADV_RSSTYPE_IPV6_UDP: | | 2067 | case IXGBE_RXDADV_RSSTYPE_IPV6_UDP: |
2068 | M_HASHTYPE_SET(sendmp, | | 2068 | M_HASHTYPE_SET(sendmp, |
2069 | M_HASHTYPE_RSS_UDP_IPV6); | | 2069 | M_HASHTYPE_RSS_UDP_IPV6); |
2070 | break; | | 2070 | break; |
2071 | case IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX: | | 2071 | case IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX: |
2072 | M_HASHTYPE_SET(sendmp, | | 2072 | M_HASHTYPE_SET(sendmp, |
2073 | M_HASHTYPE_RSS_UDP_IPV6_EX); | | 2073 | M_HASHTYPE_RSS_UDP_IPV6_EX); |
2074 | break; | | 2074 | break; |
2075 | #endif | | 2075 | #endif |
2076 | default: | | 2076 | default: |
2077 | M_HASHTYPE_SET(sendmp, | | 2077 | M_HASHTYPE_SET(sendmp, |
2078 | M_HASHTYPE_OPAQUE_HASH); | | 2078 | M_HASHTYPE_OPAQUE_HASH); |
2079 | } | | 2079 | } |
2080 | } else { | | 2080 | } else { |
2081 | sendmp->m_pkthdr.flowid = que->msix; | | 2081 | sendmp->m_pkthdr.flowid = que->msix; |
2082 | M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE); | | 2082 | M_HASHTYPE_SET(sendmp, M_HASHTYPE_OPAQUE); |
2083 | } | | 2083 | } |
2084 | #endif | | 2084 | #endif |
2085 | } | | 2085 | } |
2086 | next_desc: | | 2086 | next_desc: |
2087 | ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, | | 2087 | ixgbe_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, |
2088 | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); | | 2088 | BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); |
2089 | | | 2089 | |
2090 | /* Advance our pointers to the next descriptor. */ | | 2090 | /* Advance our pointers to the next descriptor. */ |
2091 | if (++i == rxr->num_desc) | | 2091 | if (++i == rxr->num_desc) |
2092 | i = 0; | | 2092 | i = 0; |
2093 | | | 2093 | |
2094 | /* Now send to the stack or do LRO */ | | 2094 | /* Now send to the stack or do LRO */ |
2095 | if (sendmp != NULL) { | | 2095 | if (sendmp != NULL) { |
2096 | ixgbe_rx_input(rxr, ifp, sendmp, ptype); | | 2096 | ixgbe_rx_input(rxr, ifp, sendmp, ptype); |
2097 | } | | 2097 | } |
2098 | | | 2098 | |
2099 | /* Every 8 descriptors we go to refresh mbufs */ | | 2099 | /* Every 8 descriptors we go to refresh mbufs */ |
2100 | if (processed == 8) { | | 2100 | if (processed == 8) { |
2101 | ixgbe_refresh_mbufs(rxr, i); | | 2101 | ixgbe_refresh_mbufs(rxr, i); |
2102 | processed = 0; | | 2102 | processed = 0; |
2103 | } | | 2103 | } |
2104 | } | | 2104 | } |
2105 | | | 2105 | |
2106 | /* Refresh any remaining buf structs */ | | 2106 | /* Refresh any remaining buf structs */ |
2107 | if (ixgbe_rx_unrefreshed(rxr)) | | 2107 | if (ixgbe_rx_unrefreshed(rxr)) |
2108 | ixgbe_refresh_mbufs(rxr, i); | | 2108 | ixgbe_refresh_mbufs(rxr, i); |
2109 | | | 2109 | |
2110 | rxr->next_to_check = i; | | 2110 | rxr->next_to_check = i; |
2111 | | | 2111 | |
2112 | IXGBE_RX_UNLOCK(rxr); | | 2112 | IXGBE_RX_UNLOCK(rxr); |
2113 | | | 2113 | |
2114 | #ifdef LRO | | 2114 | #ifdef LRO |
2115 | /* | | 2115 | /* |
2116 | * Flush any outstanding LRO work | | 2116 | * Flush any outstanding LRO work |
2117 | */ | | 2117 | */ |
2118 | tcp_lro_flush_all(lro); | | 2118 | tcp_lro_flush_all(lro); |
2119 | #endif /* LRO */ | | 2119 | #endif /* LRO */ |
2120 | | | 2120 | |
2121 | /* | | 2121 | /* |
2122 | * Still have cleaning to do? | | 2122 | * Still have cleaning to do? |
2123 | */ | | 2123 | */ |
2124 | if ((staterr & IXGBE_RXD_STAT_DD) != 0) | | 2124 | if ((staterr & IXGBE_RXD_STAT_DD) != 0) |
2125 | return (TRUE); | | 2125 | return (TRUE); |
2126 | | | 2126 | |
2127 | return (FALSE); | | 2127 | return (FALSE); |
2128 | } /* ixgbe_rxeof */ | | 2128 | } /* ixgbe_rxeof */ |
2129 | | | 2129 | |
2130 | | | 2130 | |
2131 | /************************************************************************ | | 2131 | /************************************************************************ |
2132 | * ixgbe_rx_checksum | | 2132 | * ixgbe_rx_checksum |
2133 | * | | 2133 | * |
2134 | * Verify that the hardware indicated that the checksum is valid. | | 2134 | * Verify that the hardware indicated that the checksum is valid. |
2135 | * Inform the stack about the status of checksum so that stack | | 2135 | * Inform the stack about the status of checksum so that stack |
2136 | * doesn't spend time verifying the checksum. | | 2136 | * doesn't spend time verifying the checksum. |
2137 | ************************************************************************/ | | 2137 | ************************************************************************/ |
2138 | static void | | 2138 | static void |
2139 | ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype, | | 2139 | ixgbe_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype, |
2140 | struct ixgbe_hw_stats *stats) | | 2140 | struct ixgbe_hw_stats *stats) |
2141 | { | | 2141 | { |
2142 | u16 status = (u16)staterr; | | 2142 | u16 status = (u16)staterr; |
2143 | u8 errors = (u8)(staterr >> 24); | | 2143 | u8 errors = (u8)(staterr >> 24); |
2144 | #if 0 | | 2144 | #if 0 |
2145 | bool sctp = false; | | 2145 | bool sctp = false; |
2146 | | | 2146 | |
2147 | if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && | | 2147 | if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && |
2148 | (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) | | 2148 | (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) |
2149 | sctp = true; | | 2149 | sctp = true; |
2150 | #endif | | 2150 | #endif |
2151 | | | 2151 | |
2152 | /* IPv4 checksum */ | | 2152 | /* IPv4 checksum */ |
2153 | if (status & IXGBE_RXD_STAT_IPCS) { | | 2153 | if (status & IXGBE_RXD_STAT_IPCS) { |
2154 | stats->ipcs.ev_count++; | | 2154 | stats->ipcs.ev_count++; |
2155 | if (!(errors & IXGBE_RXD_ERR_IPE)) { | | 2155 | if (!(errors & IXGBE_RXD_ERR_IPE)) { |
2156 | /* IP Checksum Good */ | | 2156 | /* IP Checksum Good */ |
2157 | mp->m_pkthdr.csum_flags = M_CSUM_IPv4; | | 2157 | mp->m_pkthdr.csum_flags = M_CSUM_IPv4; |
2158 | } else { | | 2158 | } else { |
2159 | stats->ipcs_bad.ev_count++; | | 2159 | stats->ipcs_bad.ev_count++; |
2160 | mp->m_pkthdr.csum_flags = M_CSUM_IPv4|M_CSUM_IPv4_BAD; | | 2160 | mp->m_pkthdr.csum_flags = M_CSUM_IPv4|M_CSUM_IPv4_BAD; |
2161 | } | | 2161 | } |
2162 | } | | 2162 | } |
2163 | /* TCP/UDP/SCTP checksum */ | | 2163 | /* TCP/UDP/SCTP checksum */ |
2164 | if (status & IXGBE_RXD_STAT_L4CS) { | | 2164 | if (status & IXGBE_RXD_STAT_L4CS) { |
2165 | stats->l4cs.ev_count++; | | 2165 | stats->l4cs.ev_count++; |
2166 | int type = M_CSUM_TCPv4|M_CSUM_TCPv6|M_CSUM_UDPv4|M_CSUM_UDPv6; | | 2166 | int type = M_CSUM_TCPv4|M_CSUM_TCPv6|M_CSUM_UDPv4|M_CSUM_UDPv6; |
2167 | if (!(errors & IXGBE_RXD_ERR_TCPE)) { | | 2167 | if (!(errors & IXGBE_RXD_ERR_TCPE)) { |
2168 | mp->m_pkthdr.csum_flags |= type; | | 2168 | mp->m_pkthdr.csum_flags |= type; |
2169 | } else { | | 2169 | } else { |
2170 | stats->l4cs_bad.ev_count++; | | 2170 | stats->l4cs_bad.ev_count++; |
2171 | mp->m_pkthdr.csum_flags |= type | M_CSUM_TCP_UDP_BAD; | | 2171 | mp->m_pkthdr.csum_flags |= type | M_CSUM_TCP_UDP_BAD; |
2172 | } | | 2172 | } |
2173 | } | | 2173 | } |
2174 | } /* ixgbe_rx_checksum */ | | 2174 | } /* ixgbe_rx_checksum */ |
2175 | | | 2175 | |
2176 | /************************************************************************ | | 2176 | /************************************************************************ |
2177 | * ixgbe_dma_malloc | | 2177 | * ixgbe_dma_malloc |
2178 | ************************************************************************/ | | 2178 | ************************************************************************/ |
2179 | int | | 2179 | int |
2180 | ixgbe_dma_malloc(struct adapter *adapter, const bus_size_t size, | | 2180 | ixgbe_dma_malloc(struct adapter *adapter, const bus_size_t size, |
2181 | struct ixgbe_dma_alloc *dma, const int mapflags) | | 2181 | struct ixgbe_dma_alloc *dma, const int mapflags) |
2182 | { | | 2182 | { |
2183 | device_t dev = adapter->dev; | | 2183 | device_t dev = adapter->dev; |
2184 | int r, rsegs; | | 2184 | int r, rsegs; |
2185 | | | 2185 | |
2186 | r = ixgbe_dma_tag_create( | | 2186 | r = ixgbe_dma_tag_create( |
2187 | /* parent */ adapter->osdep.dmat, | | 2187 | /* parent */ adapter->osdep.dmat, |
2188 | /* alignment */ DBA_ALIGN, | | 2188 | /* alignment */ DBA_ALIGN, |
2189 | /* bounds */ 0, | | 2189 | /* bounds */ 0, |
2190 | /* maxsize */ size, | | 2190 | /* maxsize */ size, |
2191 | /* nsegments */ 1, | | 2191 | /* nsegments */ 1, |
2192 | /* maxsegsize */ size, | | 2192 | /* maxsegsize */ size, |
2193 | /* flags */ BUS_DMA_ALLOCNOW, | | 2193 | /* flags */ BUS_DMA_ALLOCNOW, |
2194 | &dma->dma_tag); | | 2194 | &dma->dma_tag); |
2195 | if (r != 0) { | | 2195 | if (r != 0) { |
2196 | aprint_error_dev(dev, | | 2196 | aprint_error_dev(dev, |
2197 | "%s: ixgbe_dma_tag_create failed; error %d\n", __func__, | | 2197 | "%s: ixgbe_dma_tag_create failed; error %d\n", __func__, |
2198 | r); | | 2198 | r); |
2199 | goto fail_0; | | 2199 | goto fail_0; |
2200 | } | | 2200 | } |
2201 | | | 2201 | |
2202 | r = bus_dmamem_alloc(dma->dma_tag->dt_dmat, size, | | 2202 | r = bus_dmamem_alloc(dma->dma_tag->dt_dmat, size, |
2203 | dma->dma_tag->dt_alignment, dma->dma_tag->dt_boundary, | | 2203 | dma->dma_tag->dt_alignment, dma->dma_tag->dt_boundary, |
2204 | &dma->dma_seg, 1, &rsegs, BUS_DMA_NOWAIT); | | 2204 | &dma->dma_seg, 1, &rsegs, BUS_DMA_NOWAIT); |
2205 | if (r != 0) { | | 2205 | if (r != 0) { |
2206 | aprint_error_dev(dev, | | 2206 | aprint_error_dev(dev, |
2207 | "%s: bus_dmamem_alloc failed; error %d\n", __func__, r); | | 2207 | "%s: bus_dmamem_alloc failed; error %d\n", __func__, r); |
2208 | goto fail_1; | | 2208 | goto fail_1; |
2209 | } | | 2209 | } |
2210 | | | 2210 | |
2211 | r = bus_dmamem_map(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs, | | 2211 | r = bus_dmamem_map(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs, |
2212 | size, &dma->dma_vaddr, BUS_DMA_NOWAIT | BUS_DMA_COHERENT); | | 2212 | size, &dma->dma_vaddr, BUS_DMA_NOWAIT | BUS_DMA_COHERENT); |
2213 | if (r != 0) { | | 2213 | if (r != 0) { |
2214 | aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", | | 2214 | aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", |
2215 | __func__, r); | | 2215 | __func__, r); |
2216 | goto fail_2; | | 2216 | goto fail_2; |
2217 | } | | 2217 | } |
2218 | | | 2218 | |
2219 | r = ixgbe_dmamap_create(dma->dma_tag, 0, &dma->dma_map); | | 2219 | r = ixgbe_dmamap_create(dma->dma_tag, 0, &dma->dma_map); |
2220 | if (r != 0) { | | 2220 | if (r != 0) { |
2221 | aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", | | 2221 | aprint_error_dev(dev, "%s: bus_dmamem_map failed; error %d\n", |
2222 | __func__, r); | | 2222 | __func__, r); |
2223 | goto fail_3; | | 2223 | goto fail_3; |
2224 | } | | 2224 | } |
2225 | | | 2225 | |
2226 | r = bus_dmamap_load(dma->dma_tag->dt_dmat, dma->dma_map, | | 2226 | r = bus_dmamap_load(dma->dma_tag->dt_dmat, dma->dma_map, |
2227 | dma->dma_vaddr, size, NULL, mapflags | BUS_DMA_NOWAIT); | | 2227 | dma->dma_vaddr, size, NULL, mapflags | BUS_DMA_NOWAIT); |
2228 | if (r != 0) { | | 2228 | if (r != 0) { |
2229 | aprint_error_dev(dev, "%s: bus_dmamap_load failed; error %d\n", | | 2229 | aprint_error_dev(dev, "%s: bus_dmamap_load failed; error %d\n", |
2230 | __func__, r); | | 2230 | __func__, r); |
2231 | goto fail_4; | | 2231 | goto fail_4; |
2232 | } | | 2232 | } |
2233 | dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; | | 2233 | dma->dma_paddr = dma->dma_map->dm_segs[0].ds_addr; |
2234 | dma->dma_size = size; | | 2234 | dma->dma_size = size; |
2235 | return 0; | | 2235 | return 0; |
2236 | fail_4: | | 2236 | fail_4: |
2237 | ixgbe_dmamap_destroy(dma->dma_tag, dma->dma_map); | | 2237 | ixgbe_dmamap_destroy(dma->dma_tag, dma->dma_map); |
2238 | fail_3: | | 2238 | fail_3: |
2239 | bus_dmamem_unmap(dma->dma_tag->dt_dmat, dma->dma_vaddr, size); | | 2239 | bus_dmamem_unmap(dma->dma_tag->dt_dmat, dma->dma_vaddr, size); |
2240 | fail_2: | | 2240 | fail_2: |
2241 | bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs); | | 2241 | bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, rsegs); |
2242 | fail_1: | | 2242 | fail_1: |
2243 | ixgbe_dma_tag_destroy(dma->dma_tag); | | 2243 | ixgbe_dma_tag_destroy(dma->dma_tag); |
2244 | fail_0: | | 2244 | fail_0: |
2245 | | | 2245 | |
2246 | return (r); | | 2246 | return (r); |
2247 | } /* ixgbe_dma_malloc */ | | 2247 | } /* ixgbe_dma_malloc */ |
2248 | | | 2248 | |
2249 | /************************************************************************ | | 2249 | /************************************************************************ |
2250 | * ixgbe_dma_free | | 2250 | * ixgbe_dma_free |
2251 | ************************************************************************/ | | 2251 | ************************************************************************/ |
2252 | void | | 2252 | void |
2253 | ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma) | | 2253 | ixgbe_dma_free(struct adapter *adapter, struct ixgbe_dma_alloc *dma) |
2254 | { | | 2254 | { |
2255 | bus_dmamap_sync(dma->dma_tag->dt_dmat, dma->dma_map, 0, dma->dma_size, | | 2255 | bus_dmamap_sync(dma->dma_tag->dt_dmat, dma->dma_map, 0, dma->dma_size, |
2256 | BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); | | 2256 | BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); |
2257 | ixgbe_dmamap_unload(dma->dma_tag, dma->dma_map); | | 2257 | ixgbe_dmamap_unload(dma->dma_tag, dma->dma_map); |
2258 | bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, 1); | | 2258 | bus_dmamem_free(dma->dma_tag->dt_dmat, &dma->dma_seg, 1); |
2259 | ixgbe_dma_tag_destroy(dma->dma_tag); | | 2259 | ixgbe_dma_tag_destroy(dma->dma_tag); |
2260 | } /* ixgbe_dma_free */ | | 2260 | } /* ixgbe_dma_free */ |
2261 | | | 2261 | |
2262 | | | 2262 | |
2263 | /************************************************************************ | | 2263 | /************************************************************************ |
2264 | * ixgbe_allocate_queues | | 2264 | * ixgbe_allocate_queues |
2265 | * | | 2265 | * |
2266 | * Allocate memory for the transmit and receive rings, and then | | 2266 | * Allocate memory for the transmit and receive rings, and then |
2267 | * the descriptors associated with each, called only once at attach. | | 2267 | * the descriptors associated with each, called only once at attach. |
2268 | ************************************************************************/ | | 2268 | ************************************************************************/ |
2269 | int | | 2269 | int |
2270 | ixgbe_allocate_queues(struct adapter *adapter) | | 2270 | ixgbe_allocate_queues(struct adapter *adapter) |
2271 | { | | 2271 | { |
2272 | device_t dev = adapter->dev; | | 2272 | device_t dev = adapter->dev; |
2273 | struct ix_queue *que; | | 2273 | struct ix_queue *que; |
2274 | struct tx_ring *txr; | | 2274 | struct tx_ring *txr; |
2275 | struct rx_ring *rxr; | | 2275 | struct rx_ring *rxr; |
2276 | int rsize, tsize, error = IXGBE_SUCCESS; | | 2276 | int rsize, tsize, error = IXGBE_SUCCESS; |
2277 | int txconf = 0, rxconf = 0; | | 2277 | int txconf = 0, rxconf = 0; |
2278 | | | 2278 | |
2279 | /* First, allocate the top level queue structs */ | | 2279 | /* First, allocate the top level queue structs */ |
2280 | adapter->queues = (struct ix_queue *)malloc(sizeof(struct ix_queue) * | | 2280 | adapter->queues = (struct ix_queue *)malloc(sizeof(struct ix_queue) * |
2281 | adapter->num_queues, M_DEVBUF, M_WAITOK | M_ZERO); | | 2281 | adapter->num_queues, M_DEVBUF, M_WAITOK | M_ZERO); |
2282 | | | 2282 | |
2283 | /* Second, allocate the TX ring struct memory */ | | 2283 | /* Second, allocate the TX ring struct memory */ |
2284 | adapter->tx_rings = malloc(sizeof(struct tx_ring) * | | 2284 | adapter->tx_rings = malloc(sizeof(struct tx_ring) * |
2285 | adapter->num_queues, M_DEVBUF, M_WAITOK | M_ZERO); | | 2285 | adapter->num_queues, M_DEVBUF, M_WAITOK | M_ZERO); |
2286 | | | 2286 | |
2287 | /* Third, allocate the RX ring */ | | 2287 | /* Third, allocate the RX ring */ |
2288 | adapter->rx_rings = (struct rx_ring *)malloc(sizeof(struct rx_ring) * | | 2288 | adapter->rx_rings = (struct rx_ring *)malloc(sizeof(struct rx_ring) * |
2289 | adapter->num_queues, M_DEVBUF, M_WAITOK | M_ZERO); | | 2289 | adapter->num_queues, M_DEVBUF, M_WAITOK | M_ZERO); |
2290 | | | 2290 | |
2291 | /* For the ring itself */ | | 2291 | /* For the ring itself */ |
2292 | tsize = roundup2(adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc), | | 2292 | tsize = roundup2(adapter->num_tx_desc * sizeof(union ixgbe_adv_tx_desc), |
2293 | DBA_ALIGN); | | 2293 | DBA_ALIGN); |
2294 | | | 2294 | |
2295 | /* | | 2295 | /* |
2296 | * Now set up the TX queues, txconf is needed to handle the | | 2296 | * Now set up the TX queues, txconf is needed to handle the |
2297 | * possibility that things fail midcourse and we need to | | 2297 | * possibility that things fail midcourse and we need to |
2298 | * undo memory gracefully | | 2298 | * undo memory gracefully |
2299 | */ | | 2299 | */ |
2300 | for (int i = 0; i < adapter->num_queues; i++, txconf++) { | | 2300 | for (int i = 0; i < adapter->num_queues; i++, txconf++) { |
2301 | /* Set up some basics */ | | 2301 | /* Set up some basics */ |
2302 | txr = &adapter->tx_rings[i]; | | 2302 | txr = &adapter->tx_rings[i]; |
2303 | txr->adapter = adapter; | | 2303 | txr->adapter = adapter; |
2304 | txr->txr_interq = NULL; | | 2304 | txr->txr_interq = NULL; |
2305 | /* In case SR-IOV is enabled, align the index properly */ | | 2305 | /* In case SR-IOV is enabled, align the index properly */ |
2306 | #ifdef PCI_IOV | | 2306 | #ifdef PCI_IOV |
2307 | txr->me = ixgbe_vf_que_index(adapter->iov_mode, adapter->pool, | | 2307 | txr->me = ixgbe_vf_que_index(adapter->iov_mode, adapter->pool, |
2308 | i); | | 2308 | i); |
2309 | #else | | 2309 | #else |
2310 | txr->me = i; | | 2310 | txr->me = i; |
2311 | #endif | | 2311 | #endif |
2312 | txr->num_desc = adapter->num_tx_desc; | | 2312 | txr->num_desc = adapter->num_tx_desc; |
2313 | | | 2313 | |
2314 | /* Initialize the TX side lock */ | | 2314 | /* Initialize the TX side lock */ |
2315 | mutex_init(&txr->tx_mtx, MUTEX_DEFAULT, IPL_NET); | | 2315 | mutex_init(&txr->tx_mtx, MUTEX_DEFAULT, IPL_NET); |
2316 | | | 2316 | |
2317 | if (ixgbe_dma_malloc(adapter, tsize, &txr->txdma, | | 2317 | if (ixgbe_dma_malloc(adapter, tsize, &txr->txdma, |
2318 | BUS_DMA_NOWAIT)) { | | 2318 | BUS_DMA_NOWAIT)) { |
2319 | aprint_error_dev(dev, | | 2319 | aprint_error_dev(dev, |
2320 | "Unable to allocate TX Descriptor memory\n"); | | 2320 | "Unable to allocate TX Descriptor memory\n"); |
2321 | error = ENOMEM; | | 2321 | error = ENOMEM; |
2322 | goto err_tx_desc; | | 2322 | goto err_tx_desc; |
2323 | } | | 2323 | } |
2324 | txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; | | 2324 | txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; |
2325 | bzero((void *)txr->tx_base, tsize); | | 2325 | bzero((void *)txr->tx_base, tsize); |
2326 | | | 2326 | |
2327 | /* Now allocate transmit buffers for the ring */ | | 2327 | /* Now allocate transmit buffers for the ring */ |
2328 | if (ixgbe_allocate_transmit_buffers(txr)) { | | 2328 | if (ixgbe_allocate_transmit_buffers(txr)) { |
2329 | aprint_error_dev(dev, | | 2329 | aprint_error_dev(dev, |
2330 | "Critical Failure setting up transmit buffers\n"); | | 2330 | "Critical Failure setting up transmit buffers\n"); |
2331 | error = ENOMEM; | | 2331 | error = ENOMEM; |
2332 | goto err_tx_desc; | | 2332 | goto err_tx_desc; |
2333 | } | | 2333 | } |
2334 | if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { | | 2334 | if (!(adapter->feat_en & IXGBE_FEATURE_LEGACY_TX)) { |
2335 | /* Allocate a buf ring */ | | 2335 | /* Allocate a buf ring */ |
2336 | txr->txr_interq = pcq_create(IXGBE_BR_SIZE, KM_SLEEP); | | 2336 | txr->txr_interq = pcq_create(IXGBE_BR_SIZE, KM_SLEEP); |
2337 | if (txr->txr_interq == NULL) { | | 2337 | if (txr->txr_interq == NULL) { |
2338 | aprint_error_dev(dev, | | 2338 | aprint_error_dev(dev, |
2339 | "Critical Failure setting up buf ring\n"); | | 2339 | "Critical Failure setting up buf ring\n"); |
2340 | error = ENOMEM; | | 2340 | error = ENOMEM; |
2341 | goto err_tx_desc; | | 2341 | goto err_tx_desc; |
2342 | } | | 2342 | } |
2343 | } | | 2343 | } |
2344 | } | | 2344 | } |
2345 | | | 2345 | |
2346 | /* | | 2346 | /* |
2347 | * Next the RX queues... | | 2347 | * Next the RX queues... |
2348 | */ | | 2348 | */ |
2349 | rsize = roundup2(adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc), | | 2349 | rsize = roundup2(adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc), |
2350 | DBA_ALIGN); | | 2350 | DBA_ALIGN); |
2351 | for (int i = 0; i < adapter->num_queues; i++, rxconf++) { | | 2351 | for (int i = 0; i < adapter->num_queues; i++, rxconf++) { |
2352 | rxr = &adapter->rx_rings[i]; | | 2352 | rxr = &adapter->rx_rings[i]; |
2353 | /* Set up some basics */ | | 2353 | /* Set up some basics */ |
2354 | rxr->adapter = adapter; | | 2354 | rxr->adapter = adapter; |
2355 | #ifdef PCI_IOV | | 2355 | #ifdef PCI_IOV |
2356 | /* In case SR-IOV is enabled, align the index properly */ | | 2356 | /* In case SR-IOV is enabled, align the index properly */ |
2357 | rxr->me = ixgbe_vf_que_index(adapter->iov_mode, adapter->pool, | | 2357 | rxr->me = ixgbe_vf_que_index(adapter->iov_mode, adapter->pool, |
2358 | i); | | 2358 | i); |
2359 | #else | | 2359 | #else |
2360 | rxr->me = i; | | 2360 | rxr->me = i; |
2361 | #endif | | 2361 | #endif |
2362 | rxr->num_desc = adapter->num_rx_desc; | | 2362 | rxr->num_desc = adapter->num_rx_desc; |
2363 | | | 2363 | |
2364 | /* Initialize the RX side lock */ | | 2364 | /* Initialize the RX side lock */ |
2365 | mutex_init(&rxr->rx_mtx, MUTEX_DEFAULT, IPL_NET); | | 2365 | mutex_init(&rxr->rx_mtx, MUTEX_DEFAULT, IPL_NET); |
2366 | | | 2366 | |
2367 | if (ixgbe_dma_malloc(adapter, rsize, &rxr->rxdma, | | 2367 | if (ixgbe_dma_malloc(adapter, rsize, &rxr->rxdma, |
2368 | BUS_DMA_NOWAIT)) { | | 2368 | BUS_DMA_NOWAIT)) { |
2369 | aprint_error_dev(dev, | | 2369 | aprint_error_dev(dev, |
2370 | "Unable to allocate RxDescriptor memory\n"); | | 2370 | "Unable to allocate RxDescriptor memory\n"); |
2371 | error = ENOMEM; | | 2371 | error = ENOMEM; |
2372 | goto err_rx_desc; | | 2372 | goto err_rx_desc; |
2373 | } | | 2373 | } |
2374 | rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; | | 2374 | rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; |
2375 | bzero((void *)rxr->rx_base, rsize); | | 2375 | bzero((void *)rxr->rx_base, rsize); |
2376 | | | 2376 | |
2377 | /* Allocate receive buffers for the ring */ | | 2377 | /* Allocate receive buffers for the ring */ |
2378 | if (ixgbe_allocate_receive_buffers(rxr)) { | | 2378 | if (ixgbe_allocate_receive_buffers(rxr)) { |
2379 | aprint_error_dev(dev, | | 2379 | aprint_error_dev(dev, |
2380 | "Critical Failure setting up receive buffers\n"); | | 2380 | "Critical Failure setting up receive buffers\n"); |
2381 | error = ENOMEM; | | 2381 | error = ENOMEM; |
2382 | goto err_rx_desc; | | 2382 | goto err_rx_desc; |
2383 | } | | 2383 | } |
2384 | } | | 2384 | } |
2385 | | | 2385 | |
2386 | /* | | 2386 | /* |
2387 | * Finally set up the queue holding structs | | 2387 | * Finally set up the queue holding structs |
2388 | */ | | 2388 | */ |
2389 | for (int i = 0; i < adapter->num_queues; i++) { | | 2389 | for (int i = 0; i < adapter->num_queues; i++) { |
2390 | que = &adapter->queues[i]; | | 2390 | que = &adapter->queues[i]; |
2391 | que->adapter = adapter; | | 2391 | que->adapter = adapter; |
2392 | que->me = i; | | 2392 | que->me = i; |
2393 | que->txr = &adapter->tx_rings[i]; | | 2393 | que->txr = &adapter->tx_rings[i]; |
2394 | que->rxr = &adapter->rx_rings[i]; | | 2394 | que->rxr = &adapter->rx_rings[i]; |
2395 | | | 2395 | |
2396 | mutex_init(&que->dc_mtx, MUTEX_DEFAULT, IPL_NET); | | 2396 | mutex_init(&que->dc_mtx, MUTEX_DEFAULT, IPL_NET); |
2397 | que->disabled_count = 0; | | 2397 | que->disabled_count = 0; |
2398 | } | | 2398 | } |
2399 | | | 2399 | |
2400 | return (0); | | 2400 | return (0); |
2401 | | | 2401 | |
2402 | err_rx_desc: | | 2402 | err_rx_desc: |
2403 | for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) | | 2403 | for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) |
2404 | ixgbe_dma_free(adapter, &rxr->rxdma); | | 2404 | ixgbe_dma_free(adapter, &rxr->rxdma); |
2405 | err_tx_desc: | | 2405 | err_tx_desc: |
2406 | for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) | | 2406 | for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) |
2407 | ixgbe_dma_free(adapter, &txr->txdma); | | 2407 | ixgbe_dma_free(adapter, &txr->txdma); |
2408 | free(adapter->rx_rings, M_DEVBUF); | | 2408 | free(adapter->rx_rings, M_DEVBUF); |
2409 | free(adapter->tx_rings, M_DEVBUF); | | 2409 | free(adapter->tx_rings, M_DEVBUF); |
2410 | free(adapter->queues, M_DEVBUF); | | 2410 | free(adapter->queues, M_DEVBUF); |
2411 | return (error); | | 2411 | return (error); |
2412 | } /* ixgbe_allocate_queues */ | | 2412 | } /* ixgbe_allocate_queues */ |
2413 | | | 2413 | |
2414 | /************************************************************************ | | 2414 | /************************************************************************ |
2415 | * ixgbe_free_queues | | 2415 | * ixgbe_free_queues |
2416 | * | | 2416 | * |
2417 | * Free descriptors for the transmit and receive rings, and then | | 2417 | * Free descriptors for the transmit and receive rings, and then |
2418 | * the memory associated with each. | | 2418 | * the memory associated with each. |
2419 | ************************************************************************/ | | 2419 | ************************************************************************/ |
2420 | void | | 2420 | void |
2421 | ixgbe_free_queues(struct adapter *adapter) | | 2421 | ixgbe_free_queues(struct adapter *adapter) |
2422 | { | | 2422 | { |
2423 | struct ix_queue *que; | | 2423 | struct ix_queue *que; |
2424 | int i; | | 2424 | int i; |
2425 | | | 2425 | |
2426 | ixgbe_free_transmit_structures(adapter); | | 2426 | ixgbe_free_transmit_structures(adapter); |
2427 | ixgbe_free_receive_structures(adapter); | | 2427 | ixgbe_free_receive_structures(adapter); |
2428 | for (i = 0; i < adapter->num_queues; i++) { | | 2428 | for (i = 0; i < adapter->num_queues; i++) { |
2429 | que = &adapter->queues[i]; | | 2429 | que = &adapter->queues[i]; |
2430 | mutex_destroy(&que->dc_mtx); | | 2430 | mutex_destroy(&que->dc_mtx); |
2431 | } | | 2431 | } |
2432 | free(adapter->queues, M_DEVBUF); | | 2432 | free(adapter->queues, M_DEVBUF); |
2433 | } /* ixgbe_free_queues */ | | 2433 | } /* ixgbe_free_queues */ |