Thu May 27 03:23:29 2021 UTC ()
Move the send queue checking to a new function, and also call this
in the rx interrupt path.  Measureable improvement on a NFS "create
small files" test.


(simonb)
diff -r1.25 -r1.26 src/sys/arch/mips/cavium/dev/if_cnmac.c
cvs diff -r1.25 -r1.26 src/sys/arch/mips/cavium/dev/if_cnmac.c (switch to unified diff)

--- src/sys/arch/mips/cavium/dev/if_cnmac.c 2021/05/27 01:43:32 1.25
+++ src/sys/arch/mips/cavium/dev/if_cnmac.c 2021/05/27 03:23:29 1.26
@@ -1,1371 +1,1381 @@ @@ -1,1371 +1,1381 @@
1/* $NetBSD: if_cnmac.c,v 1.25 2021/05/27 01:43:32 simonb Exp $ */ 1/* $NetBSD: if_cnmac.c,v 1.26 2021/05/27 03:23:29 simonb Exp $ */
2 2
3/* 3/*
4 * Copyright (c) 2007 Internet Initiative Japan, Inc. 4 * Copyright (c) 2007 Internet Initiative Japan, Inc.
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * Redistribution and use in source and binary forms, with or without 7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions 8 * modification, are permitted provided that the following conditions
9 * are met: 9 * are met:
10 * 1. Redistributions of source code must retain the above copyright 10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer. 11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright 12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the 13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution. 14 * documentation and/or other materials provided with the distribution.
15 * 15 *
16 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 16 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE. 26 * SUCH DAMAGE.
27 */ 27 */
28 28
29#include <sys/cdefs.h> 29#include <sys/cdefs.h>
30__KERNEL_RCSID(0, "$NetBSD: if_cnmac.c,v 1.25 2021/05/27 01:43:32 simonb Exp $"); 30__KERNEL_RCSID(0, "$NetBSD: if_cnmac.c,v 1.26 2021/05/27 03:23:29 simonb Exp $");
31 31
32/* 32/*
33 * If no free send buffer is available, free all the sent buffers and bail out. 33 * If no free send buffer is available, free all the sent buffers and bail out.
34 */ 34 */
35#define CNMAC_SEND_QUEUE_CHECK 35#define CNMAC_SEND_QUEUE_CHECK
36 36
37/* XXX XXX XXX XXX XXX XXX */ 37/* XXX XXX XXX XXX XXX XXX */
38 38
39#include <sys/param.h> 39#include <sys/param.h>
40#include <sys/systm.h> 40#include <sys/systm.h>
41#include <sys/pool.h> 41#include <sys/pool.h>
42#include <sys/mbuf.h> 42#include <sys/mbuf.h>
43#include <sys/malloc.h> 43#include <sys/malloc.h>
44#include <sys/kernel.h> 44#include <sys/kernel.h>
45#include <sys/socket.h> 45#include <sys/socket.h>
46#include <sys/ioctl.h> 46#include <sys/ioctl.h>
47#include <sys/errno.h> 47#include <sys/errno.h>
48#include <sys/device.h> 48#include <sys/device.h>
49#include <sys/queue.h> 49#include <sys/queue.h>
50#include <sys/conf.h> 50#include <sys/conf.h>
51#include <sys/sysctl.h> 51#include <sys/sysctl.h>
52#include <sys/syslog.h> 52#include <sys/syslog.h>
53 53
54#include <net/if.h> 54#include <net/if.h>
55#include <net/if_media.h> 55#include <net/if_media.h>
56#include <net/if_ether.h> 56#include <net/if_ether.h>
57#include <net/route.h> 57#include <net/route.h>
58#include <net/bpf.h> 58#include <net/bpf.h>
59 59
60#include <netinet/in.h> 60#include <netinet/in.h>
61#include <netinet/in_systm.h> 61#include <netinet/in_systm.h>
62#include <netinet/in_var.h> 62#include <netinet/in_var.h>
63#include <netinet/ip.h> 63#include <netinet/ip.h>
64 64
65#include <sys/bus.h> 65#include <sys/bus.h>
66#include <machine/intr.h> 66#include <machine/intr.h>
67#include <machine/endian.h> 67#include <machine/endian.h>
68#include <machine/locore.h> 68#include <machine/locore.h>
69 69
70#include <dev/mii/mii.h> 70#include <dev/mii/mii.h>
71#include <dev/mii/miivar.h> 71#include <dev/mii/miivar.h>
72 72
73#include <mips/cpuregs.h> 73#include <mips/cpuregs.h>
74 74
75#include <mips/cavium/octeonreg.h> 75#include <mips/cavium/octeonreg.h>
76#include <mips/cavium/octeonvar.h> 76#include <mips/cavium/octeonvar.h>
77#include <mips/cavium/include/iobusvar.h> 77#include <mips/cavium/include/iobusvar.h>
78 78
79#include <mips/cavium/dev/octeon_ciureg.h> 79#include <mips/cavium/dev/octeon_ciureg.h>
80#include <mips/cavium/dev/octeon_faureg.h> 80#include <mips/cavium/dev/octeon_faureg.h>
81#include <mips/cavium/dev/octeon_fpareg.h> 81#include <mips/cavium/dev/octeon_fpareg.h>
82#include <mips/cavium/dev/octeon_gmxreg.h> 82#include <mips/cavium/dev/octeon_gmxreg.h>
83#include <mips/cavium/dev/octeon_pipreg.h> 83#include <mips/cavium/dev/octeon_pipreg.h>
84#include <mips/cavium/dev/octeon_powreg.h> 84#include <mips/cavium/dev/octeon_powreg.h>
85#include <mips/cavium/dev/octeon_fauvar.h> 85#include <mips/cavium/dev/octeon_fauvar.h>
86#include <mips/cavium/dev/octeon_fpavar.h> 86#include <mips/cavium/dev/octeon_fpavar.h>
87#include <mips/cavium/dev/octeon_gmxvar.h> 87#include <mips/cavium/dev/octeon_gmxvar.h>
88#include <mips/cavium/dev/octeon_ipdvar.h> 88#include <mips/cavium/dev/octeon_ipdvar.h>
89#include <mips/cavium/dev/octeon_pipvar.h> 89#include <mips/cavium/dev/octeon_pipvar.h>
90#include <mips/cavium/dev/octeon_pkovar.h> 90#include <mips/cavium/dev/octeon_pkovar.h>
91#include <mips/cavium/dev/octeon_powvar.h> 91#include <mips/cavium/dev/octeon_powvar.h>
92#include <mips/cavium/dev/octeon_smivar.h> 92#include <mips/cavium/dev/octeon_smivar.h>
93 93
94#include <mips/cavium/dev/if_cnmacvar.h> 94#include <mips/cavium/dev/if_cnmacvar.h>
95 95
96/* 96/*
97 * Set the PKO to think command buffers are an odd length. This makes it so we 97 * Set the PKO to think command buffers are an odd length. This makes it so we
98 * never have to divide a comamnd across two buffers. 98 * never have to divide a comamnd across two buffers.
99 */ 99 */
100#define OCTEON_POOL_NWORDS_CMD \ 100#define OCTEON_POOL_NWORDS_CMD \
101 (((uint32_t)OCTEON_POOL_SIZE_CMD / sizeof(uint64_t)) - 1) 101 (((uint32_t)OCTEON_POOL_SIZE_CMD / sizeof(uint64_t)) - 1)
102#define FPA_COMMAND_BUFFER_POOL_NWORDS OCTEON_POOL_NWORDS_CMD /* XXX */ 102#define FPA_COMMAND_BUFFER_POOL_NWORDS OCTEON_POOL_NWORDS_CMD /* XXX */
103 103
104static void cnmac_buf_init(struct cnmac_softc *); 104static void cnmac_buf_init(struct cnmac_softc *);
105 105
106static int cnmac_match(device_t, struct cfdata *, void *); 106static int cnmac_match(device_t, struct cfdata *, void *);
107static void cnmac_attach(device_t, device_t, void *); 107static void cnmac_attach(device_t, device_t, void *);
108static void cnmac_pip_init(struct cnmac_softc *); 108static void cnmac_pip_init(struct cnmac_softc *);
109static void cnmac_ipd_init(struct cnmac_softc *); 109static void cnmac_ipd_init(struct cnmac_softc *);
110static void cnmac_pko_init(struct cnmac_softc *); 110static void cnmac_pko_init(struct cnmac_softc *);
111 111
112static void cnmac_board_mac_addr(uint8_t *, size_t, struct cnmac_softc *); 112static void cnmac_board_mac_addr(uint8_t *, size_t, struct cnmac_softc *);
113 113
114static int cnmac_mii_readreg(device_t, int, int, uint16_t *); 114static int cnmac_mii_readreg(device_t, int, int, uint16_t *);
115static int cnmac_mii_writereg(device_t, int, int, uint16_t); 115static int cnmac_mii_writereg(device_t, int, int, uint16_t);
116static void cnmac_mii_statchg(struct ifnet *); 116static void cnmac_mii_statchg(struct ifnet *);
117 117
118static int cnmac_mediainit(struct cnmac_softc *); 118static int cnmac_mediainit(struct cnmac_softc *);
119static void cnmac_mediastatus(struct ifnet *, struct ifmediareq *); 119static void cnmac_mediastatus(struct ifnet *, struct ifmediareq *);
120 120
121static inline void cnmac_send_queue_flush_prefetch(struct cnmac_softc *); 121static inline void cnmac_send_queue_flush_prefetch(struct cnmac_softc *);
122static inline void cnmac_send_queue_flush_fetch(struct cnmac_softc *); 122static inline void cnmac_send_queue_flush_fetch(struct cnmac_softc *);
123static inline void cnmac_send_queue_flush(struct cnmac_softc *); 123static inline void cnmac_send_queue_flush(struct cnmac_softc *);
124static inline void cnmac_send_queue_flush_sync(struct cnmac_softc *); 124static inline void cnmac_send_queue_flush_sync(struct cnmac_softc *);
 125static void cnmac_send_queue_check_and_flush(struct cnmac_softc *);
125static inline int cnmac_send_queue_is_full(struct cnmac_softc *); 126static inline int cnmac_send_queue_is_full(struct cnmac_softc *);
126static inline void cnmac_send_queue_add(struct cnmac_softc *, struct mbuf *, 127static inline void cnmac_send_queue_add(struct cnmac_softc *, struct mbuf *,
127 uint64_t *); 128 uint64_t *);
128static inline void cnmac_send_queue_del(struct cnmac_softc *, struct mbuf **, 129static inline void cnmac_send_queue_del(struct cnmac_softc *, struct mbuf **,
129 uint64_t **); 130 uint64_t **);
130static inline int cnmac_buf_free_work(struct cnmac_softc *, uint64_t *); 131static inline int cnmac_buf_free_work(struct cnmac_softc *, uint64_t *);
131static inline void cnmac_buf_ext_free(struct mbuf *, void *, size_t, void *); 132static inline void cnmac_buf_ext_free(struct mbuf *, void *, size_t, void *);
132 133
133static int cnmac_ioctl(struct ifnet *, u_long, void *); 134static int cnmac_ioctl(struct ifnet *, u_long, void *);
134static void cnmac_watchdog(struct ifnet *); 135static void cnmac_watchdog(struct ifnet *);
135static int cnmac_init(struct ifnet *); 136static int cnmac_init(struct ifnet *);
136static void cnmac_stop(struct ifnet *, int); 137static void cnmac_stop(struct ifnet *, int);
137static void cnmac_start(struct ifnet *); 138static void cnmac_start(struct ifnet *);
138 139
139static inline int cnmac_send_cmd(struct cnmac_softc *, uint64_t, uint64_t, 140static inline int cnmac_send_cmd(struct cnmac_softc *, uint64_t, uint64_t,
140 int *); 141 int *);
141static inline uint64_t cnmac_send_makecmd_w1(int, paddr_t); 142static inline uint64_t cnmac_send_makecmd_w1(int, paddr_t);
142static inline uint64_t cnmac_send_makecmd_w0(uint64_t, uint64_t, size_t, int, 143static inline uint64_t cnmac_send_makecmd_w0(uint64_t, uint64_t, size_t, int,
143 int); 144 int);
144static inline int cnmac_send_makecmd_gbuf(struct cnmac_softc *, struct mbuf *, 145static inline int cnmac_send_makecmd_gbuf(struct cnmac_softc *, struct mbuf *,
145 uint64_t *, int *); 146 uint64_t *, int *);
146static inline int cnmac_send_makecmd(struct cnmac_softc *, struct mbuf *, 147static inline int cnmac_send_makecmd(struct cnmac_softc *, struct mbuf *,
147 uint64_t *, uint64_t *, uint64_t *); 148 uint64_t *, uint64_t *, uint64_t *);
148static inline int cnmac_send_buf(struct cnmac_softc *, struct mbuf *, 149static inline int cnmac_send_buf(struct cnmac_softc *, struct mbuf *,
149 uint64_t *, int *); 150 uint64_t *, int *);
150static inline int cnmac_send(struct cnmac_softc *, struct mbuf *, int *); 151static inline int cnmac_send(struct cnmac_softc *, struct mbuf *, int *);
151 152
152static int cnmac_reset(struct cnmac_softc *); 153static int cnmac_reset(struct cnmac_softc *);
153static int cnmac_configure(struct cnmac_softc *); 154static int cnmac_configure(struct cnmac_softc *);
154static int cnmac_configure_common(struct cnmac_softc *); 155static int cnmac_configure_common(struct cnmac_softc *);
155 156
156static void cnmac_tick_free(void *); 157static void cnmac_tick_free(void *);
157static void cnmac_tick_misc(void *); 158static void cnmac_tick_misc(void *);
158 159
159static inline int cnmac_recv_mbuf(struct cnmac_softc *, uint64_t *, 160static inline int cnmac_recv_mbuf(struct cnmac_softc *, uint64_t *,
160 struct mbuf **); 161 struct mbuf **);
161static inline int cnmac_recv_check(struct cnmac_softc *, uint64_t); 162static inline int cnmac_recv_check(struct cnmac_softc *, uint64_t);
162static inline int cnmac_recv(struct cnmac_softc *, uint64_t *); 163static inline int cnmac_recv(struct cnmac_softc *, uint64_t *);
163static int cnmac_intr(void *); 164static int cnmac_intr(void *);
164 165
165/* device parameters */ 166/* device parameters */
166int cnmac_param_pko_cmd_w0_n2 = 1; 167int cnmac_param_pko_cmd_w0_n2 = 1;
167 168
168CFATTACH_DECL_NEW(cnmac, sizeof(struct cnmac_softc), 169CFATTACH_DECL_NEW(cnmac, sizeof(struct cnmac_softc),
169 cnmac_match, cnmac_attach, NULL, NULL); 170 cnmac_match, cnmac_attach, NULL, NULL);
170 171
171/* ---- buffer management */ 172/* ---- buffer management */
172 173
173static const struct cnmac_pool_param { 174static const struct cnmac_pool_param {
174 int poolno; 175 int poolno;
175 size_t size; 176 size_t size;
176 size_t nelems; 177 size_t nelems;
177} cnmac_pool_params[] = { 178} cnmac_pool_params[] = {
178#define _ENTRY(x) { OCTEON_POOL_NO_##x, OCTEON_POOL_SIZE_##x, OCTEON_POOL_NELEMS_##x } 179#define _ENTRY(x) { OCTEON_POOL_NO_##x, OCTEON_POOL_SIZE_##x, OCTEON_POOL_NELEMS_##x }
179 _ENTRY(PKT), 180 _ENTRY(PKT),
180 _ENTRY(WQE), 181 _ENTRY(WQE),
181 _ENTRY(CMD), 182 _ENTRY(CMD),
182 _ENTRY(SG) 183 _ENTRY(SG)
183#undef _ENTRY 184#undef _ENTRY
184}; 185};
185struct octfpa_buf *cnmac_pools[FPA_NPOOLS]; 186struct octfpa_buf *cnmac_pools[FPA_NPOOLS];
186#define cnmac_fb_pkt cnmac_pools[OCTEON_POOL_NO_PKT] 187#define cnmac_fb_pkt cnmac_pools[OCTEON_POOL_NO_PKT]
187#define cnmac_fb_wqe cnmac_pools[OCTEON_POOL_NO_WQE] 188#define cnmac_fb_wqe cnmac_pools[OCTEON_POOL_NO_WQE]
188#define cnmac_fb_cmd cnmac_pools[OCTEON_POOL_NO_CMD] 189#define cnmac_fb_cmd cnmac_pools[OCTEON_POOL_NO_CMD]
189#define cnmac_fb_sg cnmac_pools[OCTEON_POOL_NO_SG] 190#define cnmac_fb_sg cnmac_pools[OCTEON_POOL_NO_SG]
190 191
191static int cnmac_npowgroups = 0; 192static int cnmac_npowgroups = 0;
192 193
193static void 194static void
194cnmac_buf_init(struct cnmac_softc *sc) 195cnmac_buf_init(struct cnmac_softc *sc)
195{ 196{
196 static int once; 197 static int once;
197 int i; 198 int i;
198 const struct cnmac_pool_param *pp; 199 const struct cnmac_pool_param *pp;
199 struct octfpa_buf *fb; 200 struct octfpa_buf *fb;
200 201
201 if (once == 1) 202 if (once == 1)
202 return; 203 return;
203 once = 1; 204 once = 1;
204 205
205 for (i = 0; i < (int)__arraycount(cnmac_pool_params); i++) { 206 for (i = 0; i < (int)__arraycount(cnmac_pool_params); i++) {
206 pp = &cnmac_pool_params[i]; 207 pp = &cnmac_pool_params[i];
207 octfpa_buf_init(pp->poolno, pp->size, pp->nelems, &fb); 208 octfpa_buf_init(pp->poolno, pp->size, pp->nelems, &fb);
208 cnmac_pools[i] = fb; 209 cnmac_pools[i] = fb;
209 } 210 }
210} 211}
211 212
212/* ---- autoconf */ 213/* ---- autoconf */
213 214
214static int 215static int
215cnmac_match(device_t parent, struct cfdata *match, void *aux) 216cnmac_match(device_t parent, struct cfdata *match, void *aux)
216{ 217{
217 struct octgmx_attach_args *ga = aux; 218 struct octgmx_attach_args *ga = aux;
218 219
219 if (strcmp(match->cf_name, ga->ga_name) != 0) { 220 if (strcmp(match->cf_name, ga->ga_name) != 0) {
220 return 0; 221 return 0;
221 } 222 }
222 return 1; 223 return 1;
223} 224}
224 225
225static void 226static void
226cnmac_attach(device_t parent, device_t self, void *aux) 227cnmac_attach(device_t parent, device_t self, void *aux)
227{ 228{
228 struct cnmac_softc *sc = device_private(self); 229 struct cnmac_softc *sc = device_private(self);
229 struct octgmx_attach_args *ga = aux; 230 struct octgmx_attach_args *ga = aux;
230 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 231 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
231 prop_dictionary_t dict; 232 prop_dictionary_t dict;
232 prop_object_t clk; 233 prop_object_t clk;
233 uint8_t enaddr[ETHER_ADDR_LEN]; 234 uint8_t enaddr[ETHER_ADDR_LEN];
234 235
235 if (cnmac_npowgroups >= OCTEON_POW_GROUP_MAX) { 236 if (cnmac_npowgroups >= OCTEON_POW_GROUP_MAX) {
236 printf(": out of POW groups\n"); 237 printf(": out of POW groups\n");
237 } 238 }
238 239
239 sc->sc_dev = self; 240 sc->sc_dev = self;
240 sc->sc_regt = ga->ga_regt; 241 sc->sc_regt = ga->ga_regt;
241 sc->sc_port = ga->ga_portno; 242 sc->sc_port = ga->ga_portno;
242 sc->sc_port_type = ga->ga_port_type; 243 sc->sc_port_type = ga->ga_port_type;
243 sc->sc_gmx = ga->ga_gmx; 244 sc->sc_gmx = ga->ga_gmx;
244 sc->sc_gmx_port = ga->ga_gmx_port; 245 sc->sc_gmx_port = ga->ga_gmx_port;
245 sc->sc_smi = ga->ga_smi; 246 sc->sc_smi = ga->ga_smi;
246 sc->sc_powgroup = cnmac_npowgroups++; 247 sc->sc_powgroup = cnmac_npowgroups++;
247 248
248 if (sc->sc_port >= CVMSEG_LM_ETHER_COUNT) { 249 if (sc->sc_port >= CVMSEG_LM_ETHER_COUNT) {
249 /* 250 /*
250 * If we got here, increase CVMSEG_LM_ETHER_COUNT 251 * If we got here, increase CVMSEG_LM_ETHER_COUNT
251 * in octeonvar.h . 252 * in octeonvar.h .
252 */ 253 */
253 printf("%s: ERROR out of CVMSEG LM buffers\n", 254 printf("%s: ERROR out of CVMSEG LM buffers\n",
254 device_xname(self)); 255 device_xname(self));
255 return; 256 return;
256 } 257 }
257 258
258 sc->sc_init_flag = 0; 259 sc->sc_init_flag = 0;
259 /* 260 /*
260 * XXXUEBAYASI 261 * XXXUEBAYASI
261 * Setting PIP_IP_OFFSET[OFFSET] to 8 causes panic ... why??? 262 * Setting PIP_IP_OFFSET[OFFSET] to 8 causes panic ... why???
262 */ 263 */
263 sc->sc_ip_offset = 0/* XXX */; 264 sc->sc_ip_offset = 0/* XXX */;
264 265
265 if (MIPS_PRID_IMPL(mips_options.mips_cpu_id) <= MIPS_CN30XX) { 266 if (MIPS_PRID_IMPL(mips_options.mips_cpu_id) <= MIPS_CN30XX) {
266 SET(sc->sc_quirks, CNMAC_QUIRKS_NO_PRE_ALIGN); 267 SET(sc->sc_quirks, CNMAC_QUIRKS_NO_PRE_ALIGN);
267 SET(sc->sc_quirks, CNMAC_QUIRKS_NO_RX_INBND); 268 SET(sc->sc_quirks, CNMAC_QUIRKS_NO_RX_INBND);
268 } 269 }
269 270
270 cnmac_board_mac_addr(enaddr, sizeof(enaddr), sc); 271 cnmac_board_mac_addr(enaddr, sizeof(enaddr), sc);
271 printf("%s: Ethernet address %s\n", device_xname(self), 272 printf("%s: Ethernet address %s\n", device_xname(self),
272 ether_sprintf(enaddr)); 273 ether_sprintf(enaddr));
273 274
274 SIMPLEQ_INIT(&sc->sc_sendq); 275 SIMPLEQ_INIT(&sc->sc_sendq);
275 sc->sc_soft_req_thresh = 15/* XXX */; 276 sc->sc_soft_req_thresh = 15/* XXX */;
276 sc->sc_ext_callback_cnt = 0; 277 sc->sc_ext_callback_cnt = 0;
277 278
278 octgmx_stats_init(sc->sc_gmx_port); 279 octgmx_stats_init(sc->sc_gmx_port);
279 280
280 callout_init(&sc->sc_tick_misc_ch, 0); 281 callout_init(&sc->sc_tick_misc_ch, 0);
281 callout_setfunc(&sc->sc_tick_misc_ch, cnmac_tick_misc, sc); 282 callout_setfunc(&sc->sc_tick_misc_ch, cnmac_tick_misc, sc);
282 283
283 callout_init(&sc->sc_tick_free_ch, 0); 284 callout_init(&sc->sc_tick_free_ch, 0);
284 callout_setfunc(&sc->sc_tick_free_ch, cnmac_tick_free, sc); 285 callout_setfunc(&sc->sc_tick_free_ch, cnmac_tick_free, sc);
285 286
286 const int dv_unit = device_unit(self); 287 const int dv_unit = device_unit(self);
287 octfau_op_init(&sc->sc_fau_done, 288 octfau_op_init(&sc->sc_fau_done,
288 OCTEON_CVMSEG_ETHER_OFFSET(dv_unit, csm_ether_fau_done), 289 OCTEON_CVMSEG_ETHER_OFFSET(dv_unit, csm_ether_fau_done),
289 OCT_FAU_REG_ADDR_END - (8 * (dv_unit + 1))/* XXX */); 290 OCT_FAU_REG_ADDR_END - (8 * (dv_unit + 1))/* XXX */);
290 octfau_op_set_8(&sc->sc_fau_done, 0); 291 octfau_op_set_8(&sc->sc_fau_done, 0);
291 292
292 cnmac_pip_init(sc); 293 cnmac_pip_init(sc);
293 cnmac_ipd_init(sc); 294 cnmac_ipd_init(sc);
294 cnmac_pko_init(sc); 295 cnmac_pko_init(sc);
295 296
296 cnmac_configure_common(sc); 297 cnmac_configure_common(sc);
297 298
298 sc->sc_gmx_port->sc_ipd = sc->sc_ipd; 299 sc->sc_gmx_port->sc_ipd = sc->sc_ipd;
299 sc->sc_gmx_port->sc_port_mii = &sc->sc_mii; 300 sc->sc_gmx_port->sc_port_mii = &sc->sc_mii;
300 sc->sc_gmx_port->sc_port_ec = &sc->sc_ethercom; 301 sc->sc_gmx_port->sc_port_ec = &sc->sc_ethercom;
301 /* XXX */ 302 /* XXX */
302 sc->sc_gmx_port->sc_quirks = sc->sc_quirks; 303 sc->sc_gmx_port->sc_quirks = sc->sc_quirks;
303 304
304 /* XXX */ 305 /* XXX */
305 sc->sc_pow = &octpow_softc; 306 sc->sc_pow = &octpow_softc;
306 307
307 cnmac_mediainit(sc); 308 cnmac_mediainit(sc);
308 309
309 strncpy(ifp->if_xname, device_xname(self), sizeof(ifp->if_xname)); 310 strncpy(ifp->if_xname, device_xname(self), sizeof(ifp->if_xname));
310 ifp->if_softc = sc; 311 ifp->if_softc = sc;
311 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 312 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
312 ifp->if_ioctl = cnmac_ioctl; 313 ifp->if_ioctl = cnmac_ioctl;
313 ifp->if_start = cnmac_start; 314 ifp->if_start = cnmac_start;
314 ifp->if_watchdog = cnmac_watchdog; 315 ifp->if_watchdog = cnmac_watchdog;
315 ifp->if_init = cnmac_init; 316 ifp->if_init = cnmac_init;
316 ifp->if_stop = cnmac_stop; 317 ifp->if_stop = cnmac_stop;
317 IFQ_SET_MAXLEN(&ifp->if_snd, uimax(GATHER_QUEUE_SIZE, IFQ_MAXLEN)); 318 IFQ_SET_MAXLEN(&ifp->if_snd, uimax(GATHER_QUEUE_SIZE, IFQ_MAXLEN));
318 IFQ_SET_READY(&ifp->if_snd); 319 IFQ_SET_READY(&ifp->if_snd);
319 320
320  321
321 ifp->if_capabilities = 322 ifp->if_capabilities =
322#if 0 /* XXX: no tx checksum yet */ 323#if 0 /* XXX: no tx checksum yet */
323 IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx | 324 IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx |
324 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 325 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx |
325 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | 326 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx |
326 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | 327 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx |
327 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; 328 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx;
328#else 329#else
329 IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 330 IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
330 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 331 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx;
331#endif 332#endif
332 333
333 /* 802.1Q VLAN-sized frames are supported */ 334 /* 802.1Q VLAN-sized frames are supported */
334 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; 335 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
335 336
336 octgmx_set_mac_addr(sc->sc_gmx_port, enaddr); 337 octgmx_set_mac_addr(sc->sc_gmx_port, enaddr);
337 338
338 if_attach(ifp); 339 if_attach(ifp);
339 ether_ifattach(ifp, enaddr); 340 ether_ifattach(ifp, enaddr);
340 octgmx_set_filter(sc->sc_gmx_port); 341 octgmx_set_filter(sc->sc_gmx_port);
341 342
342#if 1 343#if 1
343 cnmac_buf_init(sc); 344 cnmac_buf_init(sc);
344#endif 345#endif
345 346
346 sc->sc_ih = octeon_intr_establish(POW_WORKQ_IRQ(sc->sc_powgroup), 347 sc->sc_ih = octeon_intr_establish(POW_WORKQ_IRQ(sc->sc_powgroup),
347 IPL_NET, cnmac_intr, sc); 348 IPL_NET, cnmac_intr, sc);
348 if (sc->sc_ih == NULL) 349 if (sc->sc_ih == NULL)
349 panic("%s: could not set up interrupt", device_xname(self)); 350 panic("%s: could not set up interrupt", device_xname(self));
350 351
351 dict = device_properties(sc->sc_gmx->sc_dev); 352 dict = device_properties(sc->sc_gmx->sc_dev);
352 353
353 clk = prop_dictionary_get(dict, "rgmii-tx"); 354 clk = prop_dictionary_get(dict, "rgmii-tx");
354 if (clk) 355 if (clk)
355 sc->sc_gmx_port->sc_clk_tx_setting = 356 sc->sc_gmx_port->sc_clk_tx_setting =
356 prop_number_signed_value(clk); 357 prop_number_signed_value(clk);
357 clk = prop_dictionary_get(dict, "rgmii-rx"); 358 clk = prop_dictionary_get(dict, "rgmii-rx");
358 if (clk) 359 if (clk)
359 sc->sc_gmx_port->sc_clk_rx_setting = 360 sc->sc_gmx_port->sc_clk_rx_setting =
360 prop_number_signed_value(clk); 361 prop_number_signed_value(clk);
361} 362}
362 363
363/* ---- submodules */ 364/* ---- submodules */
364 365
365/* XXX */ 366/* XXX */
366static void 367static void
367cnmac_pip_init(struct cnmac_softc *sc) 368cnmac_pip_init(struct cnmac_softc *sc)
368{ 369{
369 struct octpip_attach_args pip_aa; 370 struct octpip_attach_args pip_aa;
370 371
371 pip_aa.aa_port = sc->sc_port; 372 pip_aa.aa_port = sc->sc_port;
372 pip_aa.aa_regt = sc->sc_regt; 373 pip_aa.aa_regt = sc->sc_regt;
373 pip_aa.aa_tag_type = POW_TAG_TYPE_ORDERED/* XXX */; 374 pip_aa.aa_tag_type = POW_TAG_TYPE_ORDERED/* XXX */;
374 pip_aa.aa_receive_group = sc->sc_powgroup; 375 pip_aa.aa_receive_group = sc->sc_powgroup;
375 pip_aa.aa_ip_offset = sc->sc_ip_offset; 376 pip_aa.aa_ip_offset = sc->sc_ip_offset;
376 octpip_init(&pip_aa, &sc->sc_pip); 377 octpip_init(&pip_aa, &sc->sc_pip);
377 octpip_port_config(sc->sc_pip); 378 octpip_port_config(sc->sc_pip);
378} 379}
379 380
380/* XXX */ 381/* XXX */
381static void 382static void
382cnmac_ipd_init(struct cnmac_softc *sc) 383cnmac_ipd_init(struct cnmac_softc *sc)
383{ 384{
384 struct octipd_attach_args ipd_aa; 385 struct octipd_attach_args ipd_aa;
385 386
386 ipd_aa.aa_port = sc->sc_port; 387 ipd_aa.aa_port = sc->sc_port;
387 ipd_aa.aa_regt = sc->sc_regt; 388 ipd_aa.aa_regt = sc->sc_regt;
388 ipd_aa.aa_first_mbuff_skip = 184/* XXX */; 389 ipd_aa.aa_first_mbuff_skip = 184/* XXX */;
389 ipd_aa.aa_not_first_mbuff_skip = 0/* XXX */; 390 ipd_aa.aa_not_first_mbuff_skip = 0/* XXX */;
390 octipd_init(&ipd_aa, &sc->sc_ipd); 391 octipd_init(&ipd_aa, &sc->sc_ipd);
391} 392}
392 393
393/* XXX */ 394/* XXX */
394static void 395static void
395cnmac_pko_init(struct cnmac_softc *sc) 396cnmac_pko_init(struct cnmac_softc *sc)
396{ 397{
397 struct octpko_attach_args pko_aa; 398 struct octpko_attach_args pko_aa;
398 399
399 pko_aa.aa_port = sc->sc_port; 400 pko_aa.aa_port = sc->sc_port;
400 pko_aa.aa_regt = sc->sc_regt; 401 pko_aa.aa_regt = sc->sc_regt;
401 pko_aa.aa_cmdptr = &sc->sc_cmdptr; 402 pko_aa.aa_cmdptr = &sc->sc_cmdptr;
402 pko_aa.aa_cmd_buf_pool = OCTEON_POOL_NO_CMD; 403 pko_aa.aa_cmd_buf_pool = OCTEON_POOL_NO_CMD;
403 pko_aa.aa_cmd_buf_size = OCTEON_POOL_NWORDS_CMD; 404 pko_aa.aa_cmd_buf_size = OCTEON_POOL_NWORDS_CMD;
404 octpko_init(&pko_aa, &sc->sc_pko); 405 octpko_init(&pko_aa, &sc->sc_pko);
405} 406}
406 407
407/* ---- XXX */ 408/* ---- XXX */
408 409
409#define ADDR2UINT64(u, a) \ 410#define ADDR2UINT64(u, a) \
410 do { \ 411 do { \
411 u = \ 412 u = \
412 (((uint64_t)a[0] << 40) | ((uint64_t)a[1] << 32) | \ 413 (((uint64_t)a[0] << 40) | ((uint64_t)a[1] << 32) | \
413 ((uint64_t)a[2] << 24) | ((uint64_t)a[3] << 16) | \ 414 ((uint64_t)a[2] << 24) | ((uint64_t)a[3] << 16) | \
414 ((uint64_t)a[4] << 8) | ((uint64_t)a[5] << 0)); \ 415 ((uint64_t)a[4] << 8) | ((uint64_t)a[5] << 0)); \
415 } while (0) 416 } while (0)
416#define UINT642ADDR(a, u) \ 417#define UINT642ADDR(a, u) \
417 do { \ 418 do { \
418 a[0] = (uint8_t)((u) >> 40); a[1] = (uint8_t)((u) >> 32); \ 419 a[0] = (uint8_t)((u) >> 40); a[1] = (uint8_t)((u) >> 32); \
419 a[2] = (uint8_t)((u) >> 24); a[3] = (uint8_t)((u) >> 16); \ 420 a[2] = (uint8_t)((u) >> 24); a[3] = (uint8_t)((u) >> 16); \
420 a[4] = (uint8_t)((u) >> 8); a[5] = (uint8_t)((u) >> 0); \ 421 a[4] = (uint8_t)((u) >> 8); a[5] = (uint8_t)((u) >> 0); \
421 } while (0) 422 } while (0)
422 423
423static void 424static void
424cnmac_board_mac_addr(uint8_t *enaddr, size_t size, struct cnmac_softc *sc) 425cnmac_board_mac_addr(uint8_t *enaddr, size_t size, struct cnmac_softc *sc)
425{ 426{
426 prop_dictionary_t dict; 427 prop_dictionary_t dict;
427 prop_data_t ea; 428 prop_data_t ea;
428 429
429 dict = device_properties(sc->sc_dev); 430 dict = device_properties(sc->sc_dev);
430 KASSERT(dict != NULL); 431 KASSERT(dict != NULL);
431 ea = prop_dictionary_get(dict, "mac-address"); 432 ea = prop_dictionary_get(dict, "mac-address");
432 KASSERT(ea != NULL); 433 KASSERT(ea != NULL);
433 memcpy(enaddr, prop_data_value(ea), size); 434 memcpy(enaddr, prop_data_value(ea), size);
434} 435}
435 436
436/* ---- media */ 437/* ---- media */
437 438
438static int 439static int
439cnmac_mii_readreg(device_t self, int phy_addr, int reg, uint16_t *val) 440cnmac_mii_readreg(device_t self, int phy_addr, int reg, uint16_t *val)
440{ 441{
441 struct cnmac_softc *sc = device_private(self); 442 struct cnmac_softc *sc = device_private(self);
442 443
443 return octsmi_read(sc->sc_smi, phy_addr, reg, val); 444 return octsmi_read(sc->sc_smi, phy_addr, reg, val);
444} 445}
445 446
446static int 447static int
447cnmac_mii_writereg(device_t self, int phy_addr, int reg, uint16_t val) 448cnmac_mii_writereg(device_t self, int phy_addr, int reg, uint16_t val)
448{ 449{
449 struct cnmac_softc *sc = device_private(self); 450 struct cnmac_softc *sc = device_private(self);
450 451
451 return octsmi_write(sc->sc_smi, phy_addr, reg, val); 452 return octsmi_write(sc->sc_smi, phy_addr, reg, val);
452} 453}
453 454
454static void 455static void
455cnmac_mii_statchg(struct ifnet *ifp) 456cnmac_mii_statchg(struct ifnet *ifp)
456{ 457{
457 struct cnmac_softc *sc = ifp->if_softc; 458 struct cnmac_softc *sc = ifp->if_softc;
458 459
459 octpko_port_enable(sc->sc_pko, 0); 460 octpko_port_enable(sc->sc_pko, 0);
460 octgmx_port_enable(sc->sc_gmx_port, 0); 461 octgmx_port_enable(sc->sc_gmx_port, 0);
461 462
462 cnmac_reset(sc); 463 cnmac_reset(sc);
463 464
464 if (ISSET(ifp->if_flags, IFF_RUNNING)) 465 if (ISSET(ifp->if_flags, IFF_RUNNING))
465 octgmx_set_filter(sc->sc_gmx_port); 466 octgmx_set_filter(sc->sc_gmx_port);
466 467
467 octpko_port_enable(sc->sc_pko, 1); 468 octpko_port_enable(sc->sc_pko, 1);
468 octgmx_port_enable(sc->sc_gmx_port, 1); 469 octgmx_port_enable(sc->sc_gmx_port, 1);
469} 470}
470 471
471static int 472static int
472cnmac_mediainit(struct cnmac_softc *sc) 473cnmac_mediainit(struct cnmac_softc *sc)
473{ 474{
474 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 475 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
475 struct mii_data *mii = &sc->sc_mii; 476 struct mii_data *mii = &sc->sc_mii;
476 prop_object_t phy; 477 prop_object_t phy;
477 478
478 mii->mii_ifp = ifp; 479 mii->mii_ifp = ifp;
479 mii->mii_readreg = cnmac_mii_readreg; 480 mii->mii_readreg = cnmac_mii_readreg;
480 mii->mii_writereg = cnmac_mii_writereg; 481 mii->mii_writereg = cnmac_mii_writereg;
481 mii->mii_statchg = cnmac_mii_statchg; 482 mii->mii_statchg = cnmac_mii_statchg;
482 sc->sc_ethercom.ec_mii = mii; 483 sc->sc_ethercom.ec_mii = mii;
483 484
484 /* Initialize ifmedia structures. */ 485 /* Initialize ifmedia structures. */
485 ifmedia_init(&mii->mii_media, 0, ether_mediachange, cnmac_mediastatus); 486 ifmedia_init(&mii->mii_media, 0, ether_mediachange, cnmac_mediastatus);
486 487
487 phy = prop_dictionary_get(device_properties(sc->sc_dev), "phy-addr"); 488 phy = prop_dictionary_get(device_properties(sc->sc_dev), "phy-addr");
488 KASSERT(phy != NULL); 489 KASSERT(phy != NULL);
489 490
490 mii_attach(sc->sc_dev, mii, 0xffffffff, prop_number_signed_value(phy), 491 mii_attach(sc->sc_dev, mii, 0xffffffff, prop_number_signed_value(phy),
491 MII_OFFSET_ANY, MIIF_DOPAUSE); 492 MII_OFFSET_ANY, MIIF_DOPAUSE);
492 493
493 /* XXX XXX XXX */ 494 /* XXX XXX XXX */
494 if (LIST_FIRST(&mii->mii_phys) != NULL) { 495 if (LIST_FIRST(&mii->mii_phys) != NULL) {
495 /* XXX XXX XXX */ 496 /* XXX XXX XXX */
496 ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_AUTO); 497 ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_AUTO);
497 /* XXX XXX XXX */ 498 /* XXX XXX XXX */
498 } else { 499 } else {
499 /* XXX XXX XXX */ 500 /* XXX XXX XXX */
500 ifmedia_add(&mii->mii_media, IFM_ETHER | IFM_NONE, 501 ifmedia_add(&mii->mii_media, IFM_ETHER | IFM_NONE,
501 MII_MEDIA_NONE, NULL); 502 MII_MEDIA_NONE, NULL);
502 /* XXX XXX XXX */ 503 /* XXX XXX XXX */
503 ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_NONE); 504 ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_NONE);
504 /* XXX XXX XXX */ 505 /* XXX XXX XXX */
505 } 506 }
506 /* XXX XXX XXX */ 507 /* XXX XXX XXX */
507 508
508 return 0; 509 return 0;
509} 510}
510 511
511static void 512static void
512cnmac_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr) 513cnmac_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
513{ 514{
514 struct cnmac_softc *sc = ifp->if_softc; 515 struct cnmac_softc *sc = ifp->if_softc;
515 516
516 mii_pollstat(&sc->sc_mii); 517 mii_pollstat(&sc->sc_mii);
517 518
518 ifmr->ifm_status = sc->sc_mii.mii_media_status; 519 ifmr->ifm_status = sc->sc_mii.mii_media_status;
519 ifmr->ifm_active = sc->sc_mii.mii_media_active; 520 ifmr->ifm_active = sc->sc_mii.mii_media_active;
520 ifmr->ifm_active = (sc->sc_mii.mii_media_active & ~IFM_ETH_FMASK) | 521 ifmr->ifm_active = (sc->sc_mii.mii_media_active & ~IFM_ETH_FMASK) |
521 sc->sc_gmx_port->sc_port_flowflags; 522 sc->sc_gmx_port->sc_port_flowflags;
522} 523}
523 524
524/* ---- send buffer garbage collection */ 525/* ---- send buffer garbage collection */
525 526
526static inline void 527static inline void
527cnmac_send_queue_flush_prefetch(struct cnmac_softc *sc) 528cnmac_send_queue_flush_prefetch(struct cnmac_softc *sc)
528{ 529{
529 530
530 KASSERT(sc->sc_prefetch == 0); 531 KASSERT(sc->sc_prefetch == 0);
531 octfau_op_inc_fetch_8(&sc->sc_fau_done, 0); 532 octfau_op_inc_fetch_8(&sc->sc_fau_done, 0);
532 sc->sc_prefetch = 1; 533 sc->sc_prefetch = 1;
533} 534}
534 535
535static inline void 536static inline void
536cnmac_send_queue_flush_fetch(struct cnmac_softc *sc) 537cnmac_send_queue_flush_fetch(struct cnmac_softc *sc)
537{ 538{
538 539
539 KASSERT(sc->sc_prefetch == 1); 540 KASSERT(sc->sc_prefetch == 1);
540 sc->sc_hard_done_cnt = octfau_op_inc_read_8(&sc->sc_fau_done); 541 sc->sc_hard_done_cnt = octfau_op_inc_read_8(&sc->sc_fau_done);
541 KASSERT(sc->sc_hard_done_cnt <= 0); 542 KASSERT(sc->sc_hard_done_cnt <= 0);
542 sc->sc_prefetch = 0; 543 sc->sc_prefetch = 0;
543} 544}
544 545
545static inline void 546static inline void
546cnmac_send_queue_flush(struct cnmac_softc *sc) 547cnmac_send_queue_flush(struct cnmac_softc *sc)
547{ 548{
548 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 549 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
549 const int64_t sent_count = sc->sc_hard_done_cnt; 550 const int64_t sent_count = sc->sc_hard_done_cnt;
550 int i; 551 int i;
551 552
552 KASSERT(sc->sc_flush == 0); 553 KASSERT(sc->sc_flush == 0);
553 KASSERT(sent_count <= 0); 554 KASSERT(sent_count <= 0);
554 555
555 for (i = 0; i < 0 - sent_count; i++) { 556 for (i = 0; i < 0 - sent_count; i++) {
556 struct mbuf *m; 557 struct mbuf *m;
557 uint64_t *gbuf; 558 uint64_t *gbuf;
558 559
559 cnmac_send_queue_del(sc, &m, &gbuf); 560 cnmac_send_queue_del(sc, &m, &gbuf);
560 561
561 octfpa_buf_put(cnmac_fb_sg, gbuf); 562 octfpa_buf_put(cnmac_fb_sg, gbuf);
562 563
563 m_freem(m); 564 m_freem(m);
564 565
565 CLR(ifp->if_flags, IFF_OACTIVE); 566 CLR(ifp->if_flags, IFF_OACTIVE);
566 } 567 }
567 568
568 octfau_op_inc_fetch_8(&sc->sc_fau_done, i); 569 octfau_op_inc_fetch_8(&sc->sc_fau_done, i);
569 sc->sc_flush = i; 570 sc->sc_flush = i;
570} 571}
571 572
572static inline void 573static inline void
573cnmac_send_queue_flush_sync(struct cnmac_softc *sc) 574cnmac_send_queue_flush_sync(struct cnmac_softc *sc)
574{ 575{
575 if (sc->sc_flush == 0) 576 if (sc->sc_flush == 0)
576 return; 577 return;
577 578
578 KASSERT(sc->sc_flush > 0); 579 KASSERT(sc->sc_flush > 0);
579 580
580 /* XXX XXX XXX */ 581 /* XXX XXX XXX */
581 octfau_op_inc_read_8(&sc->sc_fau_done); 582 octfau_op_inc_read_8(&sc->sc_fau_done);
582 sc->sc_soft_req_cnt -= sc->sc_flush; 583 sc->sc_soft_req_cnt -= sc->sc_flush;
583 KASSERT(sc->sc_soft_req_cnt >= 0); 584 KASSERT(sc->sc_soft_req_cnt >= 0);
584 /* XXX XXX XXX */ 585 /* XXX XXX XXX */
585 586
586 sc->sc_flush = 0; 587 sc->sc_flush = 0;
587} 588}
588 589
589static inline int 590static inline int
590cnmac_send_queue_is_full(struct cnmac_softc *sc) 591cnmac_send_queue_is_full(struct cnmac_softc *sc)
591{ 592{
592#ifdef CNMAC_SEND_QUEUE_CHECK 593#ifdef CNMAC_SEND_QUEUE_CHECK
593 int64_t nofree_cnt; 594 int64_t nofree_cnt;
594 595
595 nofree_cnt = sc->sc_soft_req_cnt + sc->sc_hard_done_cnt; 596 nofree_cnt = sc->sc_soft_req_cnt + sc->sc_hard_done_cnt;
596 597
597 if (__predict_false(nofree_cnt == GATHER_QUEUE_SIZE - 1)) { 598 if (__predict_false(nofree_cnt == GATHER_QUEUE_SIZE - 1)) {
598 cnmac_send_queue_flush(sc); 599 cnmac_send_queue_flush(sc);
599 cnmac_send_queue_flush_sync(sc); 600 cnmac_send_queue_flush_sync(sc);
600 return 1; 601 return 1;
601 } 602 }
602 603
603#endif 604#endif
604 return 0; 605 return 0;
605} 606}
606 607
 608static void
 609cnmac_send_queue_check_and_flush(struct cnmac_softc *sc)
 610{
 611 int s;
 612
 613 /* XXX XXX XXX */
 614 s = splnet();
 615 if (sc->sc_soft_req_cnt > 0) {
 616 cnmac_send_queue_flush_prefetch(sc);
 617 cnmac_send_queue_flush_fetch(sc);
 618 cnmac_send_queue_flush(sc);
 619 cnmac_send_queue_flush_sync(sc);
 620 }
 621 splx(s);
 622 /* XXX XXX XXX */
 623}
 624
607/* 625/*
608 * (Ab)use m_nextpkt and m_paddr to maintain mbuf chain and pointer to gather 626 * (Ab)use m_nextpkt and m_paddr to maintain mbuf chain and pointer to gather
609 * buffer. Other mbuf members may be used by m_freem(), so don't touch them! 627 * buffer. Other mbuf members may be used by m_freem(), so don't touch them!
610 */ 628 */
611 629
612struct _send_queue_entry { 630struct _send_queue_entry {
613 union { 631 union {
614 struct mbuf _sqe_s_mbuf; 632 struct mbuf _sqe_s_mbuf;
615 struct { 633 struct {
616 char _sqe_s_entry_pad[offsetof(struct mbuf, m_nextpkt)]; 634 char _sqe_s_entry_pad[offsetof(struct mbuf, m_nextpkt)];
617 SIMPLEQ_ENTRY(_send_queue_entry) _sqe_s_entry_entry; 635 SIMPLEQ_ENTRY(_send_queue_entry) _sqe_s_entry_entry;
618 } _sqe_s_entry; 636 } _sqe_s_entry;
619 struct { 637 struct {
620 char _sqe_s_gbuf_pad[offsetof(struct mbuf, m_paddr)]; 638 char _sqe_s_gbuf_pad[offsetof(struct mbuf, m_paddr)];
621 uint64_t *_sqe_s_gbuf_gbuf; 639 uint64_t *_sqe_s_gbuf_gbuf;
622 } _sqe_s_gbuf; 640 } _sqe_s_gbuf;
623 } _sqe_u; 641 } _sqe_u;
624#define _sqe_entry _sqe_u._sqe_s_entry._sqe_s_entry_entry 642#define _sqe_entry _sqe_u._sqe_s_entry._sqe_s_entry_entry
625#define _sqe_gbuf _sqe_u._sqe_s_gbuf._sqe_s_gbuf_gbuf 643#define _sqe_gbuf _sqe_u._sqe_s_gbuf._sqe_s_gbuf_gbuf
626}; 644};
627 645
628static inline void 646static inline void
629cnmac_send_queue_add(struct cnmac_softc *sc, struct mbuf *m, 647cnmac_send_queue_add(struct cnmac_softc *sc, struct mbuf *m,
630 uint64_t *gbuf) 648 uint64_t *gbuf)
631{ 649{
632 struct _send_queue_entry *sqe = (struct _send_queue_entry *)m; 650 struct _send_queue_entry *sqe = (struct _send_queue_entry *)m;
633 651
634 sqe->_sqe_gbuf = gbuf; 652 sqe->_sqe_gbuf = gbuf;
635 SIMPLEQ_INSERT_TAIL(&sc->sc_sendq, sqe, _sqe_entry); 653 SIMPLEQ_INSERT_TAIL(&sc->sc_sendq, sqe, _sqe_entry);
636 654
637 if ((m->m_flags & M_EXT) && m->m_ext.ext_free != NULL) 655 if ((m->m_flags & M_EXT) && m->m_ext.ext_free != NULL)
638 sc->sc_ext_callback_cnt++; 656 sc->sc_ext_callback_cnt++;
639} 657}
640 658
641static inline void 659static inline void
642cnmac_send_queue_del(struct cnmac_softc *sc, struct mbuf **rm, uint64_t **rgbuf) 660cnmac_send_queue_del(struct cnmac_softc *sc, struct mbuf **rm, uint64_t **rgbuf)
643{ 661{
644 struct _send_queue_entry *sqe; 662 struct _send_queue_entry *sqe;
645 663
646 sqe = SIMPLEQ_FIRST(&sc->sc_sendq); 664 sqe = SIMPLEQ_FIRST(&sc->sc_sendq);
647 KASSERT(sqe != NULL); 665 KASSERT(sqe != NULL);
648 SIMPLEQ_REMOVE_HEAD(&sc->sc_sendq, _sqe_entry); 666 SIMPLEQ_REMOVE_HEAD(&sc->sc_sendq, _sqe_entry);
649 667
650 *rm = (void *)sqe; 668 *rm = (void *)sqe;
651 *rgbuf = sqe->_sqe_gbuf; 669 *rgbuf = sqe->_sqe_gbuf;
652 670
653 if (((*rm)->m_flags & M_EXT) && (*rm)->m_ext.ext_free != NULL) { 671 if (((*rm)->m_flags & M_EXT) && (*rm)->m_ext.ext_free != NULL) {
654 sc->sc_ext_callback_cnt--; 672 sc->sc_ext_callback_cnt--;
655 KASSERT(sc->sc_ext_callback_cnt >= 0); 673 KASSERT(sc->sc_ext_callback_cnt >= 0);
656 } 674 }
657} 675}
658 676
659static inline int 677static inline int
660cnmac_buf_free_work(struct cnmac_softc *sc, uint64_t *work) 678cnmac_buf_free_work(struct cnmac_softc *sc, uint64_t *work)
661{ 679{
662 680
663 /* XXX when jumbo frame */ 681 /* XXX when jumbo frame */
664 if (ISSET(work[2], PIP_WQE_WORD2_IP_BUFS)) { 682 if (ISSET(work[2], PIP_WQE_WORD2_IP_BUFS)) {
665 paddr_t addr; 683 paddr_t addr;
666 paddr_t start_buffer; 684 paddr_t start_buffer;
667 685
668 addr = work[3] & PIP_WQE_WORD3_ADDR; 686 addr = work[3] & PIP_WQE_WORD3_ADDR;
669 start_buffer = addr & ~(2048 - 1); 687 start_buffer = addr & ~(2048 - 1);
670 688
671 octfpa_buf_put_paddr(cnmac_fb_pkt, start_buffer); 689 octfpa_buf_put_paddr(cnmac_fb_pkt, start_buffer);
672 } 690 }
673 691
674 octfpa_buf_put(cnmac_fb_wqe, work); 692 octfpa_buf_put(cnmac_fb_wqe, work);
675 693
676 return 0; 694 return 0;
677} 695}
678 696
679static inline void 697static inline void
680cnmac_buf_ext_free(struct mbuf *m, void *buf, size_t size, void *arg) 698cnmac_buf_ext_free(struct mbuf *m, void *buf, size_t size, void *arg)
681{ 699{
682 octfpa_buf_put(cnmac_fb_pkt, buf); 700 octfpa_buf_put(cnmac_fb_pkt, buf);
683 701
684 KASSERT(m != NULL); 702 KASSERT(m != NULL);
685 703
686 pool_cache_put(mb_cache, m); 704 pool_cache_put(mb_cache, m);
687} 705}
688 706
689/* ---- ifnet interfaces */ 707/* ---- ifnet interfaces */
690 708
691static int 709static int
692cnmac_ioctl(struct ifnet *ifp, u_long cmd, void *data) 710cnmac_ioctl(struct ifnet *ifp, u_long cmd, void *data)
693{ 711{
694 struct cnmac_softc *sc = ifp->if_softc; 712 struct cnmac_softc *sc = ifp->if_softc;
695 struct ifreq *ifr = (struct ifreq *)data; 713 struct ifreq *ifr = (struct ifreq *)data;
696 int s, error; 714 int s, error;
697 715
698 s = splnet(); 716 s = splnet();
699 switch (cmd) { 717 switch (cmd) {
700 case SIOCSIFMEDIA: 718 case SIOCSIFMEDIA:
701 /* Flow control requires full-duplex mode. */ 719 /* Flow control requires full-duplex mode. */
702 if (IFM_SUBTYPE(ifr->ifr_media) == IFM_AUTO || 720 if (IFM_SUBTYPE(ifr->ifr_media) == IFM_AUTO ||
703 (ifr->ifr_media & IFM_FDX) == 0) { 721 (ifr->ifr_media & IFM_FDX) == 0) {
704 ifr->ifr_media &= ~IFM_ETH_FMASK; 722 ifr->ifr_media &= ~IFM_ETH_FMASK;
705 } 723 }
706 if (IFM_SUBTYPE(ifr->ifr_media) != IFM_AUTO) { 724 if (IFM_SUBTYPE(ifr->ifr_media) != IFM_AUTO) {
707 if ((ifr->ifr_media & IFM_ETH_FMASK) == IFM_FLOW) { 725 if ((ifr->ifr_media & IFM_ETH_FMASK) == IFM_FLOW) {
708 ifr->ifr_media |= 726 ifr->ifr_media |=
709 IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE; 727 IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE;
710 } 728 }
711 sc->sc_gmx_port->sc_port_flowflags = 729 sc->sc_gmx_port->sc_port_flowflags =
712 ifr->ifr_media & IFM_ETH_FMASK; 730 ifr->ifr_media & IFM_ETH_FMASK;
713 } 731 }
714 error = ifmedia_ioctl(ifp, ifr, &sc->sc_mii.mii_media, cmd); 732 error = ifmedia_ioctl(ifp, ifr, &sc->sc_mii.mii_media, cmd);
715 break; 733 break;
716 default: 734 default:
717 error = ether_ioctl(ifp, cmd, data); 735 error = ether_ioctl(ifp, cmd, data);
718 break; 736 break;
719 } 737 }
720 738
721 if (error == ENETRESET) { 739 if (error == ENETRESET) {
722 if (ISSET(ifp->if_flags, IFF_RUNNING)) 740 if (ISSET(ifp->if_flags, IFF_RUNNING))
723 octgmx_set_filter(sc->sc_gmx_port); 741 octgmx_set_filter(sc->sc_gmx_port);
724 error = 0; 742 error = 0;
725 } 743 }
726 744
727 cnmac_start(ifp); 745 cnmac_start(ifp);
728 746
729 splx(s); 747 splx(s);
730 748
731 return error; 749 return error;
732} 750}
733 751
734/* ---- send (output) */ 752/* ---- send (output) */
735 753
736static inline uint64_t 754static inline uint64_t
737cnmac_send_makecmd_w0(uint64_t fau0, uint64_t fau1, size_t len, int segs, 755cnmac_send_makecmd_w0(uint64_t fau0, uint64_t fau1, size_t len, int segs,
738 int ipoffp1) 756 int ipoffp1)
739{ 757{
740 758
741 return octpko_cmd_word0( 759 return octpko_cmd_word0(
742 OCT_FAU_OP_SIZE_64, /* sz1 */ 760 OCT_FAU_OP_SIZE_64, /* sz1 */
743 OCT_FAU_OP_SIZE_64, /* sz0 */ 761 OCT_FAU_OP_SIZE_64, /* sz0 */
744 1, fau1, 1, fau0, /* s1, reg1, s0, reg0 */ 762 1, fau1, 1, fau0, /* s1, reg1, s0, reg0 */
745 0, /* le */ 763 0, /* le */
746 cnmac_param_pko_cmd_w0_n2, /* n2 */ 764 cnmac_param_pko_cmd_w0_n2, /* n2 */
747 1, 0, /* q, r */ 765 1, 0, /* q, r */
748 (segs == 1) ? 0 : 1, /* g */ 766 (segs == 1) ? 0 : 1, /* g */
749 0, 0, 1, /* ipoffp1, ii, df */ 767 0, 0, 1, /* ipoffp1, ii, df */
750 segs, (int)len); /* segs, totalbytes */ 768 segs, (int)len); /* segs, totalbytes */
751} 769}
752 770
753static inline uint64_t 771static inline uint64_t
754cnmac_send_makecmd_w1(int size, paddr_t addr) 772cnmac_send_makecmd_w1(int size, paddr_t addr)
755{ 773{
756 774
757 return octpko_cmd_word1( 775 return octpko_cmd_word1(
758 0, 0, /* i, back */ 776 0, 0, /* i, back */
759 OCTEON_POOL_NO_SG, /* pool */ 777 OCTEON_POOL_NO_SG, /* pool */
760 size, addr); /* size, addr */ 778 size, addr); /* size, addr */
761} 779}
762 780
763static inline int 781static inline int
764cnmac_send_makecmd_gbuf(struct cnmac_softc *sc, struct mbuf *m0, uint64_t *gbuf, 782cnmac_send_makecmd_gbuf(struct cnmac_softc *sc, struct mbuf *m0, uint64_t *gbuf,
765 int *rsegs) 783 int *rsegs)
766{ 784{
767 struct mbuf *m; 785 struct mbuf *m;
768 int segs = 0; 786 int segs = 0;
769 uintptr_t laddr, rlen, nlen; 787 uintptr_t laddr, rlen, nlen;
770 788
771 for (m = m0; m != NULL; m = m->m_next) { 789 for (m = m0; m != NULL; m = m->m_next) {
772 790
773 if (__predict_false(m->m_len == 0)) 791 if (__predict_false(m->m_len == 0))
774 continue; 792 continue;
775 793
776 /* Aligned 4k */ 794 /* Aligned 4k */
777 laddr = (uintptr_t)m->m_data & (PAGE_SIZE - 1); 795 laddr = (uintptr_t)m->m_data & (PAGE_SIZE - 1);
778 796
779 if (laddr + m->m_len > PAGE_SIZE) { 797 if (laddr + m->m_len > PAGE_SIZE) {
780 /* XXX XXX XXX */ 798 /* XXX XXX XXX */
781 rlen = PAGE_SIZE - laddr; 799 rlen = PAGE_SIZE - laddr;
782 nlen = m->m_len - rlen; 800 nlen = m->m_len - rlen;
783 *(gbuf + segs) = cnmac_send_makecmd_w1(rlen, 801 *(gbuf + segs) = cnmac_send_makecmd_w1(rlen,
784 kvtophys((vaddr_t)m->m_data)); 802 kvtophys((vaddr_t)m->m_data));
785 segs++; 803 segs++;
786 if (segs > 63) { 804 if (segs > 63) {
787 return 1; 805 return 1;
788 } 806 }
789 /* XXX XXX XXX */ 807 /* XXX XXX XXX */
790 } else { 808 } else {
791 rlen = 0; 809 rlen = 0;
792 nlen = m->m_len; 810 nlen = m->m_len;
793 } 811 }
794 812
795 *(gbuf + segs) = cnmac_send_makecmd_w1(nlen, 813 *(gbuf + segs) = cnmac_send_makecmd_w1(nlen,
796 kvtophys((vaddr_t)(m->m_data + rlen))); 814 kvtophys((vaddr_t)(m->m_data + rlen)));
797 segs++; 815 segs++;
798 if (segs > 63) { 816 if (segs > 63) {
799 return 1; 817 return 1;
800 } 818 }
801 } 819 }
802 820
803 KASSERT(m == NULL); 821 KASSERT(m == NULL);
804 822
805 *rsegs = segs; 823 *rsegs = segs;
806 824
807 return 0; 825 return 0;
808} 826}
809 827
810static inline int 828static inline int
811cnmac_send_makecmd(struct cnmac_softc *sc, struct mbuf *m, 829cnmac_send_makecmd(struct cnmac_softc *sc, struct mbuf *m,
812 uint64_t *gbuf, uint64_t *rpko_cmd_w0, uint64_t *rpko_cmd_w1) 830 uint64_t *gbuf, uint64_t *rpko_cmd_w0, uint64_t *rpko_cmd_w1)
813{ 831{
814 uint64_t pko_cmd_w0, pko_cmd_w1; 832 uint64_t pko_cmd_w0, pko_cmd_w1;
815 int ipoffp1; 833 int ipoffp1;
816 int segs; 834 int segs;
817 int result = 0; 835 int result = 0;
818 836
819 if (cnmac_send_makecmd_gbuf(sc, m, gbuf, &segs)) { 837 if (cnmac_send_makecmd_gbuf(sc, m, gbuf, &segs)) {
820 log(LOG_WARNING, "%s: there are a lot of number of segments" 838 log(LOG_WARNING, "%s: there are a lot of number of segments"
821 " of transmission data", device_xname(sc->sc_dev)); 839 " of transmission data", device_xname(sc->sc_dev));
822 result = 1; 840 result = 1;
823 goto done; 841 goto done;
824 } 842 }
825 843
826 /* Get the IP packet offset for TCP/UDP checksum offloading. */ 844 /* Get the IP packet offset for TCP/UDP checksum offloading. */
827 ipoffp1 = (m->m_pkthdr.csum_flags & (M_CSUM_TCPv4 | M_CSUM_UDPv4)) 845 ipoffp1 = (m->m_pkthdr.csum_flags & (M_CSUM_TCPv4 | M_CSUM_UDPv4))
828 ? (ETHER_HDR_LEN + 1) : 0; 846 ? (ETHER_HDR_LEN + 1) : 0;
829 847
830 /* 848 /*
831 * segs == 1 -> link mode (single continuous buffer) 849 * segs == 1 -> link mode (single continuous buffer)
832 * WORD1[size] is number of bytes pointed by segment 850 * WORD1[size] is number of bytes pointed by segment
833 * 851 *
834 * segs > 1 -> gather mode (scatter-gather buffer) 852 * segs > 1 -> gather mode (scatter-gather buffer)
835 * WORD1[size] is number of segments 853 * WORD1[size] is number of segments
836 */ 854 */
837 pko_cmd_w0 = cnmac_send_makecmd_w0(sc->sc_fau_done.fd_regno, 855 pko_cmd_w0 = cnmac_send_makecmd_w0(sc->sc_fau_done.fd_regno,
838 0, m->m_pkthdr.len, segs, ipoffp1); 856 0, m->m_pkthdr.len, segs, ipoffp1);
839 if (segs == 1) { 857 if (segs == 1) {
840 pko_cmd_w1 = cnmac_send_makecmd_w1( 858 pko_cmd_w1 = cnmac_send_makecmd_w1(
841 m->m_pkthdr.len, kvtophys((vaddr_t)m->m_data)); 859 m->m_pkthdr.len, kvtophys((vaddr_t)m->m_data));
842 } else { 860 } else {
843#ifdef __mips_n32 861#ifdef __mips_n32
844 KASSERT(MIPS_KSEG0_P(gbuf)); 862 KASSERT(MIPS_KSEG0_P(gbuf));
845 pko_cmd_w1 = cnmac_send_makecmd_w1(segs, 863 pko_cmd_w1 = cnmac_send_makecmd_w1(segs,
846 MIPS_KSEG0_TO_PHYS(gbuf)); 864 MIPS_KSEG0_TO_PHYS(gbuf));
847#else 865#else
848 pko_cmd_w1 = cnmac_send_makecmd_w1(segs, 866 pko_cmd_w1 = cnmac_send_makecmd_w1(segs,
849 MIPS_XKPHYS_TO_PHYS(gbuf)); 867 MIPS_XKPHYS_TO_PHYS(gbuf));
850#endif 868#endif
851 } 869 }
852 870
853 *rpko_cmd_w0 = pko_cmd_w0; 871 *rpko_cmd_w0 = pko_cmd_w0;
854 *rpko_cmd_w1 = pko_cmd_w1; 872 *rpko_cmd_w1 = pko_cmd_w1;
855 873
856done: 874done:
857 return result; 875 return result;
858} 876}
859 877
860static inline int 878static inline int
861cnmac_send_cmd(struct cnmac_softc *sc, uint64_t pko_cmd_w0, 879cnmac_send_cmd(struct cnmac_softc *sc, uint64_t pko_cmd_w0,
862 uint64_t pko_cmd_w1, int *pwdc) 880 uint64_t pko_cmd_w1, int *pwdc)
863{ 881{
864 uint64_t *cmdptr; 882 uint64_t *cmdptr;
865 int result = 0; 883 int result = 0;
866 884
867#ifdef __mips_n32 885#ifdef __mips_n32
868 KASSERT((sc->sc_cmdptr.cmdptr & ~MIPS_PHYS_MASK) == 0); 886 KASSERT((sc->sc_cmdptr.cmdptr & ~MIPS_PHYS_MASK) == 0);
869 cmdptr = (uint64_t *)MIPS_PHYS_TO_KSEG0(sc->sc_cmdptr.cmdptr); 887 cmdptr = (uint64_t *)MIPS_PHYS_TO_KSEG0(sc->sc_cmdptr.cmdptr);
870#else 888#else
871 cmdptr = (uint64_t *)MIPS_PHYS_TO_XKPHYS_CACHED(sc->sc_cmdptr.cmdptr); 889 cmdptr = (uint64_t *)MIPS_PHYS_TO_XKPHYS_CACHED(sc->sc_cmdptr.cmdptr);
872#endif 890#endif
873 cmdptr += sc->sc_cmdptr.cmdptr_idx; 891 cmdptr += sc->sc_cmdptr.cmdptr_idx;
874 892
875 KASSERT(cmdptr != NULL); 893 KASSERT(cmdptr != NULL);
876 894
877 *cmdptr++ = pko_cmd_w0; 895 *cmdptr++ = pko_cmd_w0;
878 *cmdptr++ = pko_cmd_w1; 896 *cmdptr++ = pko_cmd_w1;
879 897
880 KASSERT(sc->sc_cmdptr.cmdptr_idx + 2 <= FPA_COMMAND_BUFFER_POOL_NWORDS - 1); 898 KASSERT(sc->sc_cmdptr.cmdptr_idx + 2 <= FPA_COMMAND_BUFFER_POOL_NWORDS - 1);
881 899
882 if (sc->sc_cmdptr.cmdptr_idx + 2 == FPA_COMMAND_BUFFER_POOL_NWORDS - 1) { 900 if (sc->sc_cmdptr.cmdptr_idx + 2 == FPA_COMMAND_BUFFER_POOL_NWORDS - 1) {
883 paddr_t buf; 901 paddr_t buf;
884 902
885 buf = octfpa_buf_get_paddr(cnmac_fb_cmd); 903 buf = octfpa_buf_get_paddr(cnmac_fb_cmd);
886 if (buf == 0) { 904 if (buf == 0) {
887 log(LOG_WARNING, 905 log(LOG_WARNING,
888 "%s: can not allocate command buffer from free pool allocator\n", 906 "%s: can not allocate command buffer from free pool allocator\n",
889 device_xname(sc->sc_dev)); 907 device_xname(sc->sc_dev));
890 result = 1; 908 result = 1;
891 goto done; 909 goto done;
892 } 910 }
893 *cmdptr++ = buf; 911 *cmdptr++ = buf;
894 sc->sc_cmdptr.cmdptr = (uint64_t)buf; 912 sc->sc_cmdptr.cmdptr = (uint64_t)buf;
895 sc->sc_cmdptr.cmdptr_idx = 0; 913 sc->sc_cmdptr.cmdptr_idx = 0;
896 } else { 914 } else {
897 sc->sc_cmdptr.cmdptr_idx += 2; 915 sc->sc_cmdptr.cmdptr_idx += 2;
898 } 916 }
899 917
900 *pwdc += 2; 918 *pwdc += 2;
901 919
902done: 920done:
903 return result; 921 return result;
904} 922}
905 923
906static inline int 924static inline int
907cnmac_send_buf(struct cnmac_softc *sc, struct mbuf *m, uint64_t *gbuf, 925cnmac_send_buf(struct cnmac_softc *sc, struct mbuf *m, uint64_t *gbuf,
908 int *pwdc) 926 int *pwdc)
909{ 927{
910 int result = 0, error; 928 int result = 0, error;
911 uint64_t pko_cmd_w0, pko_cmd_w1; 929 uint64_t pko_cmd_w0, pko_cmd_w1;
912 930
913 error = cnmac_send_makecmd(sc, m, gbuf, &pko_cmd_w0, &pko_cmd_w1); 931 error = cnmac_send_makecmd(sc, m, gbuf, &pko_cmd_w0, &pko_cmd_w1);
914 if (error != 0) { 932 if (error != 0) {
915 /* Already logging */ 933 /* Already logging */
916 result = error; 934 result = error;
917 goto done; 935 goto done;
918 } 936 }
919 937
920 error = cnmac_send_cmd(sc, pko_cmd_w0, pko_cmd_w1, pwdc); 938 error = cnmac_send_cmd(sc, pko_cmd_w0, pko_cmd_w1, pwdc);
921 if (error != 0) { 939 if (error != 0) {
922 /* Already logging */ 940 /* Already logging */
923 result = error; 941 result = error;
924 } 942 }
925 943
926done: 944done:
927 return result; 945 return result;
928} 946}
929 947
930static inline int 948static inline int
931cnmac_send(struct cnmac_softc *sc, struct mbuf *m, int *pwdc) 949cnmac_send(struct cnmac_softc *sc, struct mbuf *m, int *pwdc)
932{ 950{
933 paddr_t gaddr = 0; 951 paddr_t gaddr = 0;
934 uint64_t *gbuf = NULL; 952 uint64_t *gbuf = NULL;
935 int result = 0, error; 953 int result = 0, error;
936 954
937 gaddr = octfpa_buf_get_paddr(cnmac_fb_sg); 955 gaddr = octfpa_buf_get_paddr(cnmac_fb_sg);
938 if (gaddr == 0) { 956 if (gaddr == 0) {
939 log(LOG_WARNING, "%s: can not allocate gather buffer from " 957 log(LOG_WARNING, "%s: can not allocate gather buffer from "
940 "free pool allocator\n", device_xname(sc->sc_dev)); 958 "free pool allocator\n", device_xname(sc->sc_dev));
941 result = 1; 959 result = 1;
942 goto done; 960 goto done;
943 } 961 }
944 962
945#ifdef __mips_n32 963#ifdef __mips_n32
946 KASSERT((gaddr & ~MIPS_PHYS_MASK) == 0); 964 KASSERT((gaddr & ~MIPS_PHYS_MASK) == 0);
947 gbuf = (uint64_t *)(uintptr_t)MIPS_PHYS_TO_KSEG0(gaddr); 965 gbuf = (uint64_t *)(uintptr_t)MIPS_PHYS_TO_KSEG0(gaddr);
948#else 966#else
949 gbuf = (uint64_t *)(uintptr_t)MIPS_PHYS_TO_XKPHYS_CACHED(gaddr); 967 gbuf = (uint64_t *)(uintptr_t)MIPS_PHYS_TO_XKPHYS_CACHED(gaddr);
950#endif 968#endif
951 969
952 KASSERT(gbuf != NULL); 970 KASSERT(gbuf != NULL);
953 971
954 error = cnmac_send_buf(sc, m, gbuf, pwdc); 972 error = cnmac_send_buf(sc, m, gbuf, pwdc);
955 if (error != 0) { 973 if (error != 0) {
956 /* Already logging */ 974 /* Already logging */
957 octfpa_buf_put_paddr(cnmac_fb_sg, gaddr); 975 octfpa_buf_put_paddr(cnmac_fb_sg, gaddr);
958 result = error; 976 result = error;
959 goto done; 977 goto done;
960 } 978 }
961 979
962 cnmac_send_queue_add(sc, m, gbuf); 980 cnmac_send_queue_add(sc, m, gbuf);
963 981
964done: 982done:
965 return result; 983 return result;
966} 984}
967 985
968static void 986static void
969cnmac_start(struct ifnet *ifp) 987cnmac_start(struct ifnet *ifp)
970{ 988{
971 struct cnmac_softc *sc = ifp->if_softc; 989 struct cnmac_softc *sc = ifp->if_softc;
972 struct mbuf *m; 990 struct mbuf *m;
973 int wdc = 0; 991 int wdc = 0;
974 992
975 /* 993 /*
976 * Performance tuning 994 * Performance tuning
977 * pre-send iobdma request 995 * pre-send iobdma request
978 */ 996 */
979 cnmac_send_queue_flush_prefetch(sc); 997 cnmac_send_queue_flush_prefetch(sc);
980 998
981 if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING) 999 if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING)
982 goto last; 1000 goto last;
983 1001
984 if (__predict_false(!octgmx_link_status(sc->sc_gmx_port))) 1002 if (__predict_false(!octgmx_link_status(sc->sc_gmx_port)))
985 goto last; 1003 goto last;
986 1004
987 for (;;) { 1005 for (;;) {
988 IFQ_POLL(&ifp->if_snd, m); 1006 IFQ_POLL(&ifp->if_snd, m);
989 if (__predict_false(m == NULL)) 1007 if (__predict_false(m == NULL))
990 break; 1008 break;
991 1009
992 /* XXX XXX XXX */ 1010 /* XXX XXX XXX */
993 cnmac_send_queue_flush_fetch(sc); 1011 cnmac_send_queue_flush_fetch(sc);
994 1012
995 /* 1013 /*
996 * If no free send buffer is available, free all the sent 1014 * If no free send buffer is available, free all the sent
997 * buffers and bail out. 1015 * buffers and bail out.
998 */ 1016 */
999 if (cnmac_send_queue_is_full(sc)) { 1017 if (cnmac_send_queue_is_full(sc)) {
1000 SET(ifp->if_flags, IFF_OACTIVE); 1018 SET(ifp->if_flags, IFF_OACTIVE);
1001 if (wdc > 0) 1019 if (wdc > 0)
1002 octpko_op_doorbell_write(sc->sc_port, 1020 octpko_op_doorbell_write(sc->sc_port,
1003 sc->sc_port, wdc); 1021 sc->sc_port, wdc);
1004 callout_schedule(&sc->sc_tick_free_ch, 1); 1022 callout_schedule(&sc->sc_tick_free_ch, 1);
1005 return; 1023 return;
1006 } 1024 }
1007 /* XXX XXX XXX */ 1025 /* XXX XXX XXX */
1008 1026
1009 IFQ_DEQUEUE(&ifp->if_snd, m); 1027 IFQ_DEQUEUE(&ifp->if_snd, m);
1010 1028
1011 bpf_mtap(ifp, m, BPF_D_OUT); 1029 bpf_mtap(ifp, m, BPF_D_OUT);
1012 1030
1013 /* XXX XXX XXX */ 1031 /* XXX XXX XXX */
1014 if (sc->sc_soft_req_cnt > sc->sc_soft_req_thresh) 1032 if (sc->sc_soft_req_cnt > sc->sc_soft_req_thresh)
1015 cnmac_send_queue_flush(sc); 1033 cnmac_send_queue_flush(sc);
1016 if (cnmac_send(sc, m, &wdc)) { 1034 if (cnmac_send(sc, m, &wdc)) {
1017 IF_DROP(&ifp->if_snd); 1035 IF_DROP(&ifp->if_snd);
1018 m_freem(m); 1036 m_freem(m);
1019 log(LOG_WARNING, 1037 log(LOG_WARNING,
1020 "%s: failed in the transmission of the packet\n", 1038 "%s: failed in the transmission of the packet\n",
1021 device_xname(sc->sc_dev)); 1039 device_xname(sc->sc_dev));
1022 } else 1040 } else
1023 sc->sc_soft_req_cnt++; 1041 sc->sc_soft_req_cnt++;
1024 1042
1025 if (sc->sc_flush) 1043 if (sc->sc_flush)
1026 cnmac_send_queue_flush_sync(sc); 1044 cnmac_send_queue_flush_sync(sc);
1027 /* XXX XXX XXX */ 1045 /* XXX XXX XXX */
1028 1046
1029 /* Send next iobdma request */ 1047 /* Send next iobdma request */
1030 cnmac_send_queue_flush_prefetch(sc); 1048 cnmac_send_queue_flush_prefetch(sc);
1031 } 1049 }
1032 1050
1033 if (wdc > 0) 1051 if (wdc > 0)
1034 octpko_op_doorbell_write(sc->sc_port, sc->sc_port, wdc); 1052 octpko_op_doorbell_write(sc->sc_port, sc->sc_port, wdc);
1035 1053
1036last: 1054last:
1037 cnmac_send_queue_flush_fetch(sc); 1055 cnmac_send_queue_flush_fetch(sc);
1038 callout_schedule(&sc->sc_tick_free_ch, 1); 1056 callout_schedule(&sc->sc_tick_free_ch, 1);
1039} 1057}
1040 1058
1041static void 1059static void
1042cnmac_watchdog(struct ifnet *ifp) 1060cnmac_watchdog(struct ifnet *ifp)
1043{ 1061{
1044 struct cnmac_softc *sc = ifp->if_softc; 1062 struct cnmac_softc *sc = ifp->if_softc;
1045 1063
1046 printf("%s: device timeout\n", device_xname(sc->sc_dev)); 1064 printf("%s: device timeout\n", device_xname(sc->sc_dev));
1047 1065
1048 cnmac_configure(sc); 1066 cnmac_configure(sc);
1049 1067
1050 SET(ifp->if_flags, IFF_RUNNING); 1068 SET(ifp->if_flags, IFF_RUNNING);
1051 CLR(ifp->if_flags, IFF_OACTIVE); 1069 CLR(ifp->if_flags, IFF_OACTIVE);
1052 ifp->if_timer = 0; 1070 ifp->if_timer = 0;
1053 1071
1054 cnmac_start(ifp); 1072 cnmac_start(ifp);
1055} 1073}
1056 1074
1057static int 1075static int
1058cnmac_init(struct ifnet *ifp) 1076cnmac_init(struct ifnet *ifp)
1059{ 1077{
1060 struct cnmac_softc *sc = ifp->if_softc; 1078 struct cnmac_softc *sc = ifp->if_softc;
1061 1079
1062 /* XXX don't disable commonly used parts!!! XXX */ 1080 /* XXX don't disable commonly used parts!!! XXX */
1063 if (sc->sc_init_flag == 0) { 1081 if (sc->sc_init_flag == 0) {
1064 /* Cancel any pending I/O. */ 1082 /* Cancel any pending I/O. */
1065 cnmac_stop(ifp, 0); 1083 cnmac_stop(ifp, 0);
1066 1084
1067 /* Initialize the device */ 1085 /* Initialize the device */
1068 cnmac_configure(sc); 1086 cnmac_configure(sc);
1069 1087
1070 octpko_enable(sc->sc_pko); 1088 octpko_enable(sc->sc_pko);
1071 octipd_enable(sc->sc_ipd); 1089 octipd_enable(sc->sc_ipd);
1072 1090
1073 sc->sc_init_flag = 1; 1091 sc->sc_init_flag = 1;
1074 } else { 1092 } else {
1075 octgmx_port_enable(sc->sc_gmx_port, 1); 1093 octgmx_port_enable(sc->sc_gmx_port, 1);
1076 } 1094 }
1077 mii_ifmedia_change(&sc->sc_mii); 1095 mii_ifmedia_change(&sc->sc_mii);
1078 1096
1079 octgmx_set_filter(sc->sc_gmx_port); 1097 octgmx_set_filter(sc->sc_gmx_port);
1080 1098
1081 callout_schedule(&sc->sc_tick_misc_ch, hz); 1099 callout_schedule(&sc->sc_tick_misc_ch, hz);
1082 callout_schedule(&sc->sc_tick_free_ch, hz); 1100 callout_schedule(&sc->sc_tick_free_ch, hz);
1083 1101
1084 SET(ifp->if_flags, IFF_RUNNING); 1102 SET(ifp->if_flags, IFF_RUNNING);
1085 CLR(ifp->if_flags, IFF_OACTIVE); 1103 CLR(ifp->if_flags, IFF_OACTIVE);
1086 1104
1087 return 0; 1105 return 0;
1088} 1106}
1089 1107
1090static void 1108static void
1091cnmac_stop(struct ifnet *ifp, int disable) 1109cnmac_stop(struct ifnet *ifp, int disable)
1092{ 1110{
1093 struct cnmac_softc *sc = ifp->if_softc; 1111 struct cnmac_softc *sc = ifp->if_softc;
1094 1112
1095 callout_stop(&sc->sc_tick_misc_ch); 1113 callout_stop(&sc->sc_tick_misc_ch);
1096 callout_stop(&sc->sc_tick_free_ch); 1114 callout_stop(&sc->sc_tick_free_ch);
1097 1115
1098 mii_down(&sc->sc_mii); 1116 mii_down(&sc->sc_mii);
1099 1117
1100 octgmx_port_enable(sc->sc_gmx_port, 0); 1118 octgmx_port_enable(sc->sc_gmx_port, 0);
1101 1119
1102 /* Mark the interface as down and cancel the watchdog timer. */ 1120 /* Mark the interface as down and cancel the watchdog timer. */
1103 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 1121 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE);
1104 ifp->if_timer = 0; 1122 ifp->if_timer = 0;
1105} 1123}
1106 1124
1107/* ---- misc */ 1125/* ---- misc */
1108 1126
1109static int 1127static int
1110cnmac_reset(struct cnmac_softc *sc) 1128cnmac_reset(struct cnmac_softc *sc)
1111{ 1129{
1112 octgmx_reset_speed(sc->sc_gmx_port); 1130 octgmx_reset_speed(sc->sc_gmx_port);
1113 octgmx_reset_flowctl(sc->sc_gmx_port); 1131 octgmx_reset_flowctl(sc->sc_gmx_port);
1114 octgmx_reset_timing(sc->sc_gmx_port); 1132 octgmx_reset_timing(sc->sc_gmx_port);
1115 1133
1116 return 0; 1134 return 0;
1117} 1135}
1118 1136
1119static int 1137static int
1120cnmac_configure(struct cnmac_softc *sc) 1138cnmac_configure(struct cnmac_softc *sc)
1121{ 1139{
1122 octgmx_port_enable(sc->sc_gmx_port, 0); 1140 octgmx_port_enable(sc->sc_gmx_port, 0);
1123 1141
1124 cnmac_reset(sc); 1142 cnmac_reset(sc);
1125 1143
1126 cnmac_configure_common(sc); 1144 cnmac_configure_common(sc);
1127 1145
1128 octpko_port_config(sc->sc_pko); 1146 octpko_port_config(sc->sc_pko);
1129 octpko_port_enable(sc->sc_pko, 1); 1147 octpko_port_enable(sc->sc_pko, 1);
1130 octpow_config(sc->sc_pow, sc->sc_powgroup); 1148 octpow_config(sc->sc_pow, sc->sc_powgroup);
1131 1149
1132 octgmx_tx_stats_rd_clr(sc->sc_gmx_port, 1); 1150 octgmx_tx_stats_rd_clr(sc->sc_gmx_port, 1);
1133 octgmx_rx_stats_rd_clr(sc->sc_gmx_port, 1); 1151 octgmx_rx_stats_rd_clr(sc->sc_gmx_port, 1);
1134 1152
1135 octgmx_port_enable(sc->sc_gmx_port, 1); 1153 octgmx_port_enable(sc->sc_gmx_port, 1);
1136 1154
1137 return 0; 1155 return 0;
1138} 1156}
1139 1157
1140static int 1158static int
1141cnmac_configure_common(struct cnmac_softc *sc) 1159cnmac_configure_common(struct cnmac_softc *sc)
1142{ 1160{
1143 static int once; 1161 static int once;
1144 1162
1145 if (once == 1) 1163 if (once == 1)
1146 return 0; 1164 return 0;
1147 once = 1; 1165 once = 1;
1148 1166
1149 octipd_config(sc->sc_ipd); 1167 octipd_config(sc->sc_ipd);
1150 octpko_config(sc->sc_pko); 1168 octpko_config(sc->sc_pko);
1151 1169
1152 return 0; 1170 return 0;
1153} 1171}
1154 1172
1155/* ---- receive (input) */ 1173/* ---- receive (input) */
1156 1174
1157static inline int 1175static inline int
1158cnmac_recv_mbuf(struct cnmac_softc *sc, uint64_t *work, struct mbuf **rm) 1176cnmac_recv_mbuf(struct cnmac_softc *sc, uint64_t *work, struct mbuf **rm)
1159{ 1177{
1160 struct mbuf *m; 1178 struct mbuf *m;
1161 vaddr_t addr; 1179 vaddr_t addr;
1162 vaddr_t ext_buf; 1180 vaddr_t ext_buf;
1163 size_t ext_size; 1181 size_t ext_size;
1164 uint64_t word1 = work[1]; 1182 uint64_t word1 = work[1];
1165 uint64_t word2 = work[2]; 1183 uint64_t word2 = work[2];
1166 uint64_t word3 = work[3]; 1184 uint64_t word3 = work[3];
1167 1185
1168 MGETHDR(m, M_NOWAIT, MT_DATA); 1186 MGETHDR(m, M_NOWAIT, MT_DATA);
1169 if (m == NULL) 1187 if (m == NULL)
1170 return 1; 1188 return 1;
1171 1189
1172 octfpa_buf_put(cnmac_fb_wqe, work); 1190 octfpa_buf_put(cnmac_fb_wqe, work);
1173 1191
1174 if (__SHIFTOUT(word2, PIP_WQE_WORD2_IP_BUFS) != 1) 1192 if (__SHIFTOUT(word2, PIP_WQE_WORD2_IP_BUFS) != 1)
1175 panic("%s: expected one buffer, got %" PRId64, __func__, 1193 panic("%s: expected one buffer, got %" PRId64, __func__,
1176 __SHIFTOUT(word2, PIP_WQE_WORD2_IP_BUFS)); 1194 __SHIFTOUT(word2, PIP_WQE_WORD2_IP_BUFS));
1177 1195
1178 1196
1179#ifdef __mips_n32 1197#ifdef __mips_n32
1180 KASSERT((word3 & ~MIPS_PHYS_MASK) == 0); 1198 KASSERT((word3 & ~MIPS_PHYS_MASK) == 0);
1181 addr = MIPS_PHYS_TO_KSEG0(word3 & PIP_WQE_WORD3_ADDR); 1199 addr = MIPS_PHYS_TO_KSEG0(word3 & PIP_WQE_WORD3_ADDR);
1182#else 1200#else
1183 addr = MIPS_PHYS_TO_XKPHYS_CACHED(word3 & PIP_WQE_WORD3_ADDR); 1201 addr = MIPS_PHYS_TO_XKPHYS_CACHED(word3 & PIP_WQE_WORD3_ADDR);
1184#endif 1202#endif
1185 1203
1186 ext_size = OCTEON_POOL_SIZE_PKT; 1204 ext_size = OCTEON_POOL_SIZE_PKT;
1187 ext_buf = addr & ~(ext_size - 1); 1205 ext_buf = addr & ~(ext_size - 1);
1188 MEXTADD(m, ext_buf, ext_size, 0, cnmac_buf_ext_free, NULL); 1206 MEXTADD(m, ext_buf, ext_size, 0, cnmac_buf_ext_free, NULL);
1189 1207
1190 m->m_data = (void *)addr; 1208 m->m_data = (void *)addr;
1191 m->m_len = m->m_pkthdr.len = (word1 & PIP_WQE_WORD1_LEN) >> 48; 1209 m->m_len = m->m_pkthdr.len = (word1 & PIP_WQE_WORD1_LEN) >> 48;
1192 m_set_rcvif(m, &sc->sc_ethercom.ec_if); 1210 m_set_rcvif(m, &sc->sc_ethercom.ec_if);
1193 1211
1194 /* Not readonly buffer */ 1212 /* Not readonly buffer */
1195 m->m_flags |= M_EXT_RW; 1213 m->m_flags |= M_EXT_RW;
1196 1214
1197 *rm = m; 1215 *rm = m;
1198 1216
1199 KASSERT(*rm != NULL); 1217 KASSERT(*rm != NULL);
1200 1218
1201 return 0; 1219 return 0;
1202} 1220}
1203 1221
1204static inline int 1222static inline int
1205cnmac_recv_check(struct cnmac_softc *sc, uint64_t word2) 1223cnmac_recv_check(struct cnmac_softc *sc, uint64_t word2)
1206{ 1224{
1207 static struct timeval rxerr_log_interval = { 0, 2500000 }; 1225 static struct timeval rxerr_log_interval = { 0, 2500000 };
1208 uint64_t opecode; 1226 uint64_t opecode;
1209 1227
1210 if (__predict_true(!ISSET(word2, PIP_WQE_WORD2_NOIP_RE))) 1228 if (__predict_true(!ISSET(word2, PIP_WQE_WORD2_NOIP_RE)))
1211 return 0; 1229 return 0;
1212 1230
1213 opecode = word2 & PIP_WQE_WORD2_NOIP_OPECODE; 1231 opecode = word2 & PIP_WQE_WORD2_NOIP_OPECODE;
1214 if ((sc->sc_ethercom.ec_if.if_flags & IFF_DEBUG) && 1232 if ((sc->sc_ethercom.ec_if.if_flags & IFF_DEBUG) &&
1215 ratecheck(&sc->sc_rxerr_log_last, &rxerr_log_interval)) 1233 ratecheck(&sc->sc_rxerr_log_last, &rxerr_log_interval))
1216 log(LOG_DEBUG, "%s: rx error (%"PRId64")\n", 1234 log(LOG_DEBUG, "%s: rx error (%"PRId64")\n",
1217 device_xname(sc->sc_dev), opecode); 1235 device_xname(sc->sc_dev), opecode);
1218 1236
1219 /* This error is harmless */ 1237 /* This error is harmless */
1220 if (opecode == PIP_WQE_WORD2_RE_OPCODE_OVRRUN) 1238 if (opecode == PIP_WQE_WORD2_RE_OPCODE_OVRRUN)
1221 return 0; 1239 return 0;
1222 1240
1223 return 1; 1241 return 1;
1224} 1242}
1225 1243
1226static inline int 1244static inline int
1227cnmac_recv(struct cnmac_softc *sc, uint64_t *work) 1245cnmac_recv(struct cnmac_softc *sc, uint64_t *work)
1228{ 1246{
1229 struct ifnet *ifp; 1247 struct ifnet *ifp;
1230 struct mbuf *m; 1248 struct mbuf *m;
1231 uint64_t word2; 1249 uint64_t word2;
1232 1250
1233 KASSERT(sc != NULL); 1251 KASSERT(sc != NULL);
1234 KASSERT(work != NULL); 1252 KASSERT(work != NULL);
1235 1253
1236 word2 = work[2]; 1254 word2 = work[2];
1237 ifp = &sc->sc_ethercom.ec_if; 1255 ifp = &sc->sc_ethercom.ec_if;
1238 1256
1239 KASSERT(ifp != NULL); 1257 KASSERT(ifp != NULL);
1240 1258
1241 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 1259 if (!ISSET(ifp->if_flags, IFF_RUNNING))
1242 goto drop; 1260 goto drop;
1243 1261
1244 if (__predict_false(cnmac_recv_check(sc, word2) != 0)) { 1262 if (__predict_false(cnmac_recv_check(sc, word2) != 0)) {
1245 if_statinc(ifp, if_ierrors); 1263 if_statinc(ifp, if_ierrors);
1246 goto drop; 1264 goto drop;
1247 } 1265 }
1248 1266
1249 if (__predict_false(cnmac_recv_mbuf(sc, work, &m) != 0)) { 1267 if (__predict_false(cnmac_recv_mbuf(sc, work, &m) != 0)) {
1250 if_statinc(ifp, if_ierrors); 1268 if_statinc(ifp, if_ierrors);
1251 goto drop; 1269 goto drop;
1252 } 1270 }
1253 1271
1254 /* work[0] .. work[3] may not be valid any more */ 1272 /* work[0] .. work[3] may not be valid any more */
1255 1273
1256 KASSERT(m != NULL); 1274 KASSERT(m != NULL);
1257 1275
1258 octipd_offload(word2, m->m_data, &m->m_pkthdr.csum_flags); 1276 octipd_offload(word2, m->m_data, &m->m_pkthdr.csum_flags);
1259 1277
1260 if_percpuq_enqueue(ifp->if_percpuq, m); 1278 if_percpuq_enqueue(ifp->if_percpuq, m);
1261 1279
1262 return 0; 1280 return 0;
1263 1281
1264drop: 1282drop:
1265 cnmac_buf_free_work(sc, work); 1283 cnmac_buf_free_work(sc, work);
1266 return 1; 1284 return 1;
1267} 1285}
1268 1286
1269static int 1287static int
1270cnmac_intr(void *arg) 1288cnmac_intr(void *arg)
1271{ 1289{
1272 struct cnmac_softc *sc = arg; 1290 struct cnmac_softc *sc = arg;
1273 uint64_t *work; 1291 uint64_t *work;
1274 uint64_t wqmask = __BIT(sc->sc_powgroup); 1292 uint64_t wqmask = __BIT(sc->sc_powgroup);
1275 uint32_t coreid = 0; /* XXX octeon_get_coreid() */ 1293 uint32_t coreid = 0; /* XXX octeon_get_coreid() */
1276 uint32_t port; 1294 uint32_t port;
1277 1295
1278 _POW_WR8(sc->sc_pow, POW_PP_GRP_MSK_OFFSET(coreid), wqmask); 1296 _POW_WR8(sc->sc_pow, POW_PP_GRP_MSK_OFFSET(coreid), wqmask);
1279 1297
1280 octpow_tag_sw_wait(); 1298 octpow_tag_sw_wait();
1281 octpow_work_request_async(OCTEON_CVMSEG_OFFSET(csm_pow_intr), 1299 octpow_work_request_async(OCTEON_CVMSEG_OFFSET(csm_pow_intr),
1282 POW_NO_WAIT); 1300 POW_NO_WAIT);
1283 1301
1284 for (;;) { 1302 for (;;) {
1285 work = (uint64_t *)octpow_work_response_async( 1303 work = (uint64_t *)octpow_work_response_async(
1286 OCTEON_CVMSEG_OFFSET(csm_pow_intr)); 1304 OCTEON_CVMSEG_OFFSET(csm_pow_intr));
1287 if (work == NULL) 1305 if (work == NULL)
1288 break; 1306 break;
1289 1307
1290 octpow_tag_sw_wait(); 1308 octpow_tag_sw_wait();
1291 octpow_work_request_async(OCTEON_CVMSEG_OFFSET(csm_pow_intr), 1309 octpow_work_request_async(OCTEON_CVMSEG_OFFSET(csm_pow_intr),
1292 POW_NO_WAIT); 1310 POW_NO_WAIT);
1293 1311
1294 port = __SHIFTOUT(work[1], PIP_WQE_WORD1_IPRT); 1312 port = __SHIFTOUT(work[1], PIP_WQE_WORD1_IPRT);
1295 if (port != sc->sc_port) { 1313 if (port != sc->sc_port) {
1296 printf("%s: unexpected wqe port %u, should be %u\n", 1314 printf("%s: unexpected wqe port %u, should be %u\n",
1297 device_xname(sc->sc_dev), port, sc->sc_port); 1315 device_xname(sc->sc_dev), port, sc->sc_port);
1298 goto wqe_error; 1316 goto wqe_error;
1299 } 1317 }
1300 1318
1301 (void)cnmac_recv(sc, work); 1319 (void)cnmac_recv(sc, work);
 1320
 1321 cnmac_send_queue_check_and_flush(sc);
1302 } 1322 }
1303 1323
1304 _POW_WR8(sc->sc_pow, POW_WQ_INT_OFFSET, wqmask); 1324 _POW_WR8(sc->sc_pow, POW_WQ_INT_OFFSET, wqmask);
1305 1325
1306 return 1; 1326 return 1;
1307 1327
1308wqe_error: 1328wqe_error:
1309 printf("word0: 0x%016" PRIx64 "\n", work[0]); 1329 printf("word0: 0x%016" PRIx64 "\n", work[0]);
1310 printf("word1: 0x%016" PRIx64 "\n", work[1]); 1330 printf("word1: 0x%016" PRIx64 "\n", work[1]);
1311 printf("word2: 0x%016" PRIx64 "\n", work[2]); 1331 printf("word2: 0x%016" PRIx64 "\n", work[2]);
1312 printf("word3: 0x%016" PRIx64 "\n", work[3]); 1332 printf("word3: 0x%016" PRIx64 "\n", work[3]);
1313 panic("wqe_error"); 1333 panic("wqe_error");
1314} 1334}
1315 1335
1316/* ---- tick */ 1336/* ---- tick */
1317 1337
1318/* 1338/*
1319 * cnmac_tick_free 1339 * cnmac_tick_free
1320 * 1340 *
1321 * => garbage collect send gather buffer / mbuf 1341 * => garbage collect send gather buffer / mbuf
1322 * => called at softclock 1342 * => called at softclock
1323 */ 1343 */
1324static void 1344static void
1325cnmac_tick_free(void *arg) 1345cnmac_tick_free(void *arg)
1326{ 1346{
1327 struct cnmac_softc *sc = arg; 1347 struct cnmac_softc *sc = arg;
1328 int timo; 1348 int timo;
1329 int s; 
1330 1349
1331 s = splnet(); 1350 cnmac_send_queue_check_and_flush(sc);
1332 /* XXX XXX XXX */ 
1333 if (sc->sc_soft_req_cnt > 0) { 
1334 cnmac_send_queue_flush_prefetch(sc); 
1335 cnmac_send_queue_flush_fetch(sc); 
1336 cnmac_send_queue_flush(sc); 
1337 cnmac_send_queue_flush_sync(sc); 
1338 } 
1339 /* XXX XXX XXX */ 
1340 1351
1341 timo = (sc->sc_ext_callback_cnt > 0) ? 1 : hz; 1352 timo = (sc->sc_ext_callback_cnt > 0) ? 1 : hz;
1342 callout_schedule(&sc->sc_tick_free_ch, timo); 1353 callout_schedule(&sc->sc_tick_free_ch, timo);
1343 splx(s); 
1344} 1354}
1345 1355
1346/* 1356/*
1347 * cnmac_tick_misc 1357 * cnmac_tick_misc
1348 * 1358 *
1349 * => collect statistics 1359 * => collect statistics
1350 * => check link status 1360 * => check link status
1351 * => called at softclock 1361 * => called at softclock
1352 */ 1362 */
1353static void 1363static void
1354cnmac_tick_misc(void *arg) 1364cnmac_tick_misc(void *arg)
1355{ 1365{
1356 struct cnmac_softc *sc = arg; 1366 struct cnmac_softc *sc = arg;
1357 struct ifnet *ifp; 1367 struct ifnet *ifp;
1358 int s; 1368 int s;
1359 1369
1360 s = splnet(); 1370 s = splnet();
1361 1371
1362 ifp = &sc->sc_ethercom.ec_if; 1372 ifp = &sc->sc_ethercom.ec_if;
1363 1373
1364 octgmx_stats(sc->sc_gmx_port); 1374 octgmx_stats(sc->sc_gmx_port);
1365 octpip_stats(sc->sc_pip, ifp, sc->sc_port); 1375 octpip_stats(sc->sc_pip, ifp, sc->sc_port);
1366 mii_tick(&sc->sc_mii); 1376 mii_tick(&sc->sc_mii);
1367 1377
1368 splx(s); 1378 splx(s);
1369 1379
1370 callout_schedule(&sc->sc_tick_misc_ch, hz); 1380 callout_schedule(&sc->sc_tick_misc_ch, hz);
1371} 1381}