Thu May 27 01:43:32 2021 UTC ()
Schedule the send cleanup function for next tick in cnmac_start().  In
the send cleanup function, schedule for the next tick instead of waiting
for HZ ticks if there are still send requests outstanding.  Greatly
increases NFS throughput, perhaps other types of network traffic.

Use callout_setfunc() after callout_init() instead of callout_reset().


(simonb)
diff -r1.24 -r1.25 src/sys/arch/mips/cavium/dev/if_cnmac.c
cvs diff -r1.24 -r1.25 src/sys/arch/mips/cavium/dev/if_cnmac.c (switch to unified diff)

--- src/sys/arch/mips/cavium/dev/if_cnmac.c 2020/06/23 05:17:13 1.24
+++ src/sys/arch/mips/cavium/dev/if_cnmac.c 2021/05/27 01:43:32 1.25
@@ -1,1372 +1,1371 @@ @@ -1,1372 +1,1371 @@
1/* $NetBSD: if_cnmac.c,v 1.24 2020/06/23 05:17:13 simonb Exp $ */ 1/* $NetBSD: if_cnmac.c,v 1.25 2021/05/27 01:43:32 simonb Exp $ */
2 2
3/* 3/*
4 * Copyright (c) 2007 Internet Initiative Japan, Inc. 4 * Copyright (c) 2007 Internet Initiative Japan, Inc.
5 * All rights reserved. 5 * All rights reserved.
6 * 6 *
7 * Redistribution and use in source and binary forms, with or without 7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions 8 * modification, are permitted provided that the following conditions
9 * are met: 9 * are met:
10 * 1. Redistributions of source code must retain the above copyright 10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer. 11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright 12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the 13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution. 14 * documentation and/or other materials provided with the distribution.
15 * 15 *
16 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 16 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE. 26 * SUCH DAMAGE.
27 */ 27 */
28 28
29#include <sys/cdefs.h> 29#include <sys/cdefs.h>
30__KERNEL_RCSID(0, "$NetBSD: if_cnmac.c,v 1.24 2020/06/23 05:17:13 simonb Exp $"); 30__KERNEL_RCSID(0, "$NetBSD: if_cnmac.c,v 1.25 2021/05/27 01:43:32 simonb Exp $");
31 31
32/* 32/*
33 * If no free send buffer is available, free all the sent buffers and bail out. 33 * If no free send buffer is available, free all the sent buffers and bail out.
34 */ 34 */
35#define CNMAC_SEND_QUEUE_CHECK 35#define CNMAC_SEND_QUEUE_CHECK
36 36
37/* XXX XXX XXX XXX XXX XXX */ 37/* XXX XXX XXX XXX XXX XXX */
38 38
39#include <sys/param.h> 39#include <sys/param.h>
40#include <sys/systm.h> 40#include <sys/systm.h>
41#include <sys/pool.h> 41#include <sys/pool.h>
42#include <sys/mbuf.h> 42#include <sys/mbuf.h>
43#include <sys/malloc.h> 43#include <sys/malloc.h>
44#include <sys/kernel.h> 44#include <sys/kernel.h>
45#include <sys/socket.h> 45#include <sys/socket.h>
46#include <sys/ioctl.h> 46#include <sys/ioctl.h>
47#include <sys/errno.h> 47#include <sys/errno.h>
48#include <sys/device.h> 48#include <sys/device.h>
49#include <sys/queue.h> 49#include <sys/queue.h>
50#include <sys/conf.h> 50#include <sys/conf.h>
51#include <sys/sysctl.h> 51#include <sys/sysctl.h>
52#include <sys/syslog.h> 52#include <sys/syslog.h>
53 53
54#include <net/if.h> 54#include <net/if.h>
55#include <net/if_media.h> 55#include <net/if_media.h>
56#include <net/if_ether.h> 56#include <net/if_ether.h>
57#include <net/route.h> 57#include <net/route.h>
58#include <net/bpf.h> 58#include <net/bpf.h>
59 59
60#include <netinet/in.h> 60#include <netinet/in.h>
61#include <netinet/in_systm.h> 61#include <netinet/in_systm.h>
62#include <netinet/in_var.h> 62#include <netinet/in_var.h>
63#include <netinet/ip.h> 63#include <netinet/ip.h>
64 64
65#include <sys/bus.h> 65#include <sys/bus.h>
66#include <machine/intr.h> 66#include <machine/intr.h>
67#include <machine/endian.h> 67#include <machine/endian.h>
68#include <machine/locore.h> 68#include <machine/locore.h>
69 69
70#include <dev/mii/mii.h> 70#include <dev/mii/mii.h>
71#include <dev/mii/miivar.h> 71#include <dev/mii/miivar.h>
72 72
73#include <mips/cpuregs.h> 73#include <mips/cpuregs.h>
74 74
75#include <mips/cavium/octeonreg.h> 75#include <mips/cavium/octeonreg.h>
76#include <mips/cavium/octeonvar.h> 76#include <mips/cavium/octeonvar.h>
77#include <mips/cavium/include/iobusvar.h> 77#include <mips/cavium/include/iobusvar.h>
78 78
79#include <mips/cavium/dev/octeon_ciureg.h> 79#include <mips/cavium/dev/octeon_ciureg.h>
80#include <mips/cavium/dev/octeon_faureg.h> 80#include <mips/cavium/dev/octeon_faureg.h>
81#include <mips/cavium/dev/octeon_fpareg.h> 81#include <mips/cavium/dev/octeon_fpareg.h>
82#include <mips/cavium/dev/octeon_gmxreg.h> 82#include <mips/cavium/dev/octeon_gmxreg.h>
83#include <mips/cavium/dev/octeon_pipreg.h> 83#include <mips/cavium/dev/octeon_pipreg.h>
84#include <mips/cavium/dev/octeon_powreg.h> 84#include <mips/cavium/dev/octeon_powreg.h>
85#include <mips/cavium/dev/octeon_fauvar.h> 85#include <mips/cavium/dev/octeon_fauvar.h>
86#include <mips/cavium/dev/octeon_fpavar.h> 86#include <mips/cavium/dev/octeon_fpavar.h>
87#include <mips/cavium/dev/octeon_gmxvar.h> 87#include <mips/cavium/dev/octeon_gmxvar.h>
88#include <mips/cavium/dev/octeon_ipdvar.h> 88#include <mips/cavium/dev/octeon_ipdvar.h>
89#include <mips/cavium/dev/octeon_pipvar.h> 89#include <mips/cavium/dev/octeon_pipvar.h>
90#include <mips/cavium/dev/octeon_pkovar.h> 90#include <mips/cavium/dev/octeon_pkovar.h>
91#include <mips/cavium/dev/octeon_powvar.h> 91#include <mips/cavium/dev/octeon_powvar.h>
92#include <mips/cavium/dev/octeon_smivar.h> 92#include <mips/cavium/dev/octeon_smivar.h>
93 93
94#include <mips/cavium/dev/if_cnmacvar.h> 94#include <mips/cavium/dev/if_cnmacvar.h>
95 95
96/* 96/*
97 * Set the PKO to think command buffers are an odd length. This makes it so we 97 * Set the PKO to think command buffers are an odd length. This makes it so we
98 * never have to divide a comamnd across two buffers. 98 * never have to divide a comamnd across two buffers.
99 */ 99 */
100#define OCTEON_POOL_NWORDS_CMD \ 100#define OCTEON_POOL_NWORDS_CMD \
101 (((uint32_t)OCTEON_POOL_SIZE_CMD / sizeof(uint64_t)) - 1) 101 (((uint32_t)OCTEON_POOL_SIZE_CMD / sizeof(uint64_t)) - 1)
102#define FPA_COMMAND_BUFFER_POOL_NWORDS OCTEON_POOL_NWORDS_CMD /* XXX */ 102#define FPA_COMMAND_BUFFER_POOL_NWORDS OCTEON_POOL_NWORDS_CMD /* XXX */
103 103
104static void cnmac_buf_init(struct cnmac_softc *); 104static void cnmac_buf_init(struct cnmac_softc *);
105 105
106static int cnmac_match(device_t, struct cfdata *, void *); 106static int cnmac_match(device_t, struct cfdata *, void *);
107static void cnmac_attach(device_t, device_t, void *); 107static void cnmac_attach(device_t, device_t, void *);
108static void cnmac_pip_init(struct cnmac_softc *); 108static void cnmac_pip_init(struct cnmac_softc *);
109static void cnmac_ipd_init(struct cnmac_softc *); 109static void cnmac_ipd_init(struct cnmac_softc *);
110static void cnmac_pko_init(struct cnmac_softc *); 110static void cnmac_pko_init(struct cnmac_softc *);
111 111
112static void cnmac_board_mac_addr(uint8_t *, size_t, struct cnmac_softc *); 112static void cnmac_board_mac_addr(uint8_t *, size_t, struct cnmac_softc *);
113 113
114static int cnmac_mii_readreg(device_t, int, int, uint16_t *); 114static int cnmac_mii_readreg(device_t, int, int, uint16_t *);
115static int cnmac_mii_writereg(device_t, int, int, uint16_t); 115static int cnmac_mii_writereg(device_t, int, int, uint16_t);
116static void cnmac_mii_statchg(struct ifnet *); 116static void cnmac_mii_statchg(struct ifnet *);
117 117
118static int cnmac_mediainit(struct cnmac_softc *); 118static int cnmac_mediainit(struct cnmac_softc *);
119static void cnmac_mediastatus(struct ifnet *, struct ifmediareq *); 119static void cnmac_mediastatus(struct ifnet *, struct ifmediareq *);
120 120
121static inline void cnmac_send_queue_flush_prefetch(struct cnmac_softc *); 121static inline void cnmac_send_queue_flush_prefetch(struct cnmac_softc *);
122static inline void cnmac_send_queue_flush_fetch(struct cnmac_softc *); 122static inline void cnmac_send_queue_flush_fetch(struct cnmac_softc *);
123static inline void cnmac_send_queue_flush(struct cnmac_softc *); 123static inline void cnmac_send_queue_flush(struct cnmac_softc *);
124static inline void cnmac_send_queue_flush_sync(struct cnmac_softc *); 124static inline void cnmac_send_queue_flush_sync(struct cnmac_softc *);
125static inline int cnmac_send_queue_is_full(struct cnmac_softc *); 125static inline int cnmac_send_queue_is_full(struct cnmac_softc *);
126static inline void cnmac_send_queue_add(struct cnmac_softc *, struct mbuf *, 126static inline void cnmac_send_queue_add(struct cnmac_softc *, struct mbuf *,
127 uint64_t *); 127 uint64_t *);
128static inline void cnmac_send_queue_del(struct cnmac_softc *, struct mbuf **, 128static inline void cnmac_send_queue_del(struct cnmac_softc *, struct mbuf **,
129 uint64_t **); 129 uint64_t **);
130static inline int cnmac_buf_free_work(struct cnmac_softc *, uint64_t *); 130static inline int cnmac_buf_free_work(struct cnmac_softc *, uint64_t *);
131static inline void cnmac_buf_ext_free(struct mbuf *, void *, size_t, void *); 131static inline void cnmac_buf_ext_free(struct mbuf *, void *, size_t, void *);
132 132
133static int cnmac_ioctl(struct ifnet *, u_long, void *); 133static int cnmac_ioctl(struct ifnet *, u_long, void *);
134static void cnmac_watchdog(struct ifnet *); 134static void cnmac_watchdog(struct ifnet *);
135static int cnmac_init(struct ifnet *); 135static int cnmac_init(struct ifnet *);
136static void cnmac_stop(struct ifnet *, int); 136static void cnmac_stop(struct ifnet *, int);
137static void cnmac_start(struct ifnet *); 137static void cnmac_start(struct ifnet *);
138 138
139static inline int cnmac_send_cmd(struct cnmac_softc *, uint64_t, uint64_t, 139static inline int cnmac_send_cmd(struct cnmac_softc *, uint64_t, uint64_t,
140 int *); 140 int *);
141static inline uint64_t cnmac_send_makecmd_w1(int, paddr_t); 141static inline uint64_t cnmac_send_makecmd_w1(int, paddr_t);
142static inline uint64_t cnmac_send_makecmd_w0(uint64_t, uint64_t, size_t, int, 142static inline uint64_t cnmac_send_makecmd_w0(uint64_t, uint64_t, size_t, int,
143 int); 143 int);
144static inline int cnmac_send_makecmd_gbuf(struct cnmac_softc *, struct mbuf *, 144static inline int cnmac_send_makecmd_gbuf(struct cnmac_softc *, struct mbuf *,
145 uint64_t *, int *); 145 uint64_t *, int *);
146static inline int cnmac_send_makecmd(struct cnmac_softc *, struct mbuf *, 146static inline int cnmac_send_makecmd(struct cnmac_softc *, struct mbuf *,
147 uint64_t *, uint64_t *, uint64_t *); 147 uint64_t *, uint64_t *, uint64_t *);
148static inline int cnmac_send_buf(struct cnmac_softc *, struct mbuf *, 148static inline int cnmac_send_buf(struct cnmac_softc *, struct mbuf *,
149 uint64_t *, int *); 149 uint64_t *, int *);
150static inline int cnmac_send(struct cnmac_softc *, struct mbuf *, int *); 150static inline int cnmac_send(struct cnmac_softc *, struct mbuf *, int *);
151 151
152static int cnmac_reset(struct cnmac_softc *); 152static int cnmac_reset(struct cnmac_softc *);
153static int cnmac_configure(struct cnmac_softc *); 153static int cnmac_configure(struct cnmac_softc *);
154static int cnmac_configure_common(struct cnmac_softc *); 154static int cnmac_configure_common(struct cnmac_softc *);
155 155
156static void cnmac_tick_free(void *); 156static void cnmac_tick_free(void *);
157static void cnmac_tick_misc(void *); 157static void cnmac_tick_misc(void *);
158 158
159static inline int cnmac_recv_mbuf(struct cnmac_softc *, uint64_t *, 159static inline int cnmac_recv_mbuf(struct cnmac_softc *, uint64_t *,
160 struct mbuf **); 160 struct mbuf **);
161static inline int cnmac_recv_check(struct cnmac_softc *, uint64_t); 161static inline int cnmac_recv_check(struct cnmac_softc *, uint64_t);
162static inline int cnmac_recv(struct cnmac_softc *, uint64_t *); 162static inline int cnmac_recv(struct cnmac_softc *, uint64_t *);
163static int cnmac_intr(void *); 163static int cnmac_intr(void *);
164 164
165/* device parameters */ 165/* device parameters */
166int cnmac_param_pko_cmd_w0_n2 = 1; 166int cnmac_param_pko_cmd_w0_n2 = 1;
167 167
168CFATTACH_DECL_NEW(cnmac, sizeof(struct cnmac_softc), 168CFATTACH_DECL_NEW(cnmac, sizeof(struct cnmac_softc),
169 cnmac_match, cnmac_attach, NULL, NULL); 169 cnmac_match, cnmac_attach, NULL, NULL);
170 170
171/* ---- buffer management */ 171/* ---- buffer management */
172 172
173static const struct cnmac_pool_param { 173static const struct cnmac_pool_param {
174 int poolno; 174 int poolno;
175 size_t size; 175 size_t size;
176 size_t nelems; 176 size_t nelems;
177} cnmac_pool_params[] = { 177} cnmac_pool_params[] = {
178#define _ENTRY(x) { OCTEON_POOL_NO_##x, OCTEON_POOL_SIZE_##x, OCTEON_POOL_NELEMS_##x } 178#define _ENTRY(x) { OCTEON_POOL_NO_##x, OCTEON_POOL_SIZE_##x, OCTEON_POOL_NELEMS_##x }
179 _ENTRY(PKT), 179 _ENTRY(PKT),
180 _ENTRY(WQE), 180 _ENTRY(WQE),
181 _ENTRY(CMD), 181 _ENTRY(CMD),
182 _ENTRY(SG) 182 _ENTRY(SG)
183#undef _ENTRY 183#undef _ENTRY
184}; 184};
185struct octfpa_buf *cnmac_pools[FPA_NPOOLS]; 185struct octfpa_buf *cnmac_pools[FPA_NPOOLS];
186#define cnmac_fb_pkt cnmac_pools[OCTEON_POOL_NO_PKT] 186#define cnmac_fb_pkt cnmac_pools[OCTEON_POOL_NO_PKT]
187#define cnmac_fb_wqe cnmac_pools[OCTEON_POOL_NO_WQE] 187#define cnmac_fb_wqe cnmac_pools[OCTEON_POOL_NO_WQE]
188#define cnmac_fb_cmd cnmac_pools[OCTEON_POOL_NO_CMD] 188#define cnmac_fb_cmd cnmac_pools[OCTEON_POOL_NO_CMD]
189#define cnmac_fb_sg cnmac_pools[OCTEON_POOL_NO_SG] 189#define cnmac_fb_sg cnmac_pools[OCTEON_POOL_NO_SG]
190 190
191static int cnmac_npowgroups = 0; 191static int cnmac_npowgroups = 0;
192 192
193static void 193static void
194cnmac_buf_init(struct cnmac_softc *sc) 194cnmac_buf_init(struct cnmac_softc *sc)
195{ 195{
196 static int once; 196 static int once;
197 int i; 197 int i;
198 const struct cnmac_pool_param *pp; 198 const struct cnmac_pool_param *pp;
199 struct octfpa_buf *fb; 199 struct octfpa_buf *fb;
200 200
201 if (once == 1) 201 if (once == 1)
202 return; 202 return;
203 once = 1; 203 once = 1;
204 204
205 for (i = 0; i < (int)__arraycount(cnmac_pool_params); i++) { 205 for (i = 0; i < (int)__arraycount(cnmac_pool_params); i++) {
206 pp = &cnmac_pool_params[i]; 206 pp = &cnmac_pool_params[i];
207 octfpa_buf_init(pp->poolno, pp->size, pp->nelems, &fb); 207 octfpa_buf_init(pp->poolno, pp->size, pp->nelems, &fb);
208 cnmac_pools[i] = fb; 208 cnmac_pools[i] = fb;
209 } 209 }
210} 210}
211 211
212/* ---- autoconf */ 212/* ---- autoconf */
213 213
214static int 214static int
215cnmac_match(device_t parent, struct cfdata *match, void *aux) 215cnmac_match(device_t parent, struct cfdata *match, void *aux)
216{ 216{
217 struct octgmx_attach_args *ga = aux; 217 struct octgmx_attach_args *ga = aux;
218 218
219 if (strcmp(match->cf_name, ga->ga_name) != 0) { 219 if (strcmp(match->cf_name, ga->ga_name) != 0) {
220 return 0; 220 return 0;
221 } 221 }
222 return 1; 222 return 1;
223} 223}
224 224
225static void 225static void
226cnmac_attach(device_t parent, device_t self, void *aux) 226cnmac_attach(device_t parent, device_t self, void *aux)
227{ 227{
228 struct cnmac_softc *sc = device_private(self); 228 struct cnmac_softc *sc = device_private(self);
229 struct octgmx_attach_args *ga = aux; 229 struct octgmx_attach_args *ga = aux;
230 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 230 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
231 prop_dictionary_t dict; 231 prop_dictionary_t dict;
232 prop_object_t clk; 232 prop_object_t clk;
233 uint8_t enaddr[ETHER_ADDR_LEN]; 233 uint8_t enaddr[ETHER_ADDR_LEN];
234 234
235 if (cnmac_npowgroups >= OCTEON_POW_GROUP_MAX) { 235 if (cnmac_npowgroups >= OCTEON_POW_GROUP_MAX) {
236 printf(": out of POW groups\n"); 236 printf(": out of POW groups\n");
237 } 237 }
238 238
239 sc->sc_dev = self; 239 sc->sc_dev = self;
240 sc->sc_regt = ga->ga_regt; 240 sc->sc_regt = ga->ga_regt;
241 sc->sc_port = ga->ga_portno; 241 sc->sc_port = ga->ga_portno;
242 sc->sc_port_type = ga->ga_port_type; 242 sc->sc_port_type = ga->ga_port_type;
243 sc->sc_gmx = ga->ga_gmx; 243 sc->sc_gmx = ga->ga_gmx;
244 sc->sc_gmx_port = ga->ga_gmx_port; 244 sc->sc_gmx_port = ga->ga_gmx_port;
245 sc->sc_smi = ga->ga_smi; 245 sc->sc_smi = ga->ga_smi;
246 sc->sc_powgroup = cnmac_npowgroups++; 246 sc->sc_powgroup = cnmac_npowgroups++;
247 247
248 if (sc->sc_port >= CVMSEG_LM_ETHER_COUNT) { 248 if (sc->sc_port >= CVMSEG_LM_ETHER_COUNT) {
249 /* 249 /*
250 * If we got here, increase CVMSEG_LM_ETHER_COUNT 250 * If we got here, increase CVMSEG_LM_ETHER_COUNT
251 * in octeonvar.h . 251 * in octeonvar.h .
252 */ 252 */
253 printf("%s: ERROR out of CVMSEG LM buffers\n", 253 printf("%s: ERROR out of CVMSEG LM buffers\n",
254 device_xname(self)); 254 device_xname(self));
255 return; 255 return;
256 } 256 }
257 257
258 sc->sc_init_flag = 0; 258 sc->sc_init_flag = 0;
259 /* 259 /*
260 * XXXUEBAYASI 260 * XXXUEBAYASI
261 * Setting PIP_IP_OFFSET[OFFSET] to 8 causes panic ... why??? 261 * Setting PIP_IP_OFFSET[OFFSET] to 8 causes panic ... why???
262 */ 262 */
263 sc->sc_ip_offset = 0/* XXX */; 263 sc->sc_ip_offset = 0/* XXX */;
264 264
265 if (MIPS_PRID_IMPL(mips_options.mips_cpu_id) <= MIPS_CN30XX) { 265 if (MIPS_PRID_IMPL(mips_options.mips_cpu_id) <= MIPS_CN30XX) {
266 SET(sc->sc_quirks, CNMAC_QUIRKS_NO_PRE_ALIGN); 266 SET(sc->sc_quirks, CNMAC_QUIRKS_NO_PRE_ALIGN);
267 SET(sc->sc_quirks, CNMAC_QUIRKS_NO_RX_INBND); 267 SET(sc->sc_quirks, CNMAC_QUIRKS_NO_RX_INBND);
268 } 268 }
269 269
270 cnmac_board_mac_addr(enaddr, sizeof(enaddr), sc); 270 cnmac_board_mac_addr(enaddr, sizeof(enaddr), sc);
271 printf("%s: Ethernet address %s\n", device_xname(self), 271 printf("%s: Ethernet address %s\n", device_xname(self),
272 ether_sprintf(enaddr)); 272 ether_sprintf(enaddr));
273 273
274 SIMPLEQ_INIT(&sc->sc_sendq); 274 SIMPLEQ_INIT(&sc->sc_sendq);
275 sc->sc_soft_req_thresh = 15/* XXX */; 275 sc->sc_soft_req_thresh = 15/* XXX */;
276 sc->sc_ext_callback_cnt = 0; 276 sc->sc_ext_callback_cnt = 0;
277 277
278 octgmx_stats_init(sc->sc_gmx_port); 278 octgmx_stats_init(sc->sc_gmx_port);
279 279
280 callout_init(&sc->sc_tick_misc_ch, 0); 280 callout_init(&sc->sc_tick_misc_ch, 0);
 281 callout_setfunc(&sc->sc_tick_misc_ch, cnmac_tick_misc, sc);
 282
281 callout_init(&sc->sc_tick_free_ch, 0); 283 callout_init(&sc->sc_tick_free_ch, 0);
 284 callout_setfunc(&sc->sc_tick_free_ch, cnmac_tick_free, sc);
282 285
283 const int dv_unit = device_unit(self); 286 const int dv_unit = device_unit(self);
284 octfau_op_init(&sc->sc_fau_done, 287 octfau_op_init(&sc->sc_fau_done,
285 OCTEON_CVMSEG_ETHER_OFFSET(dv_unit, csm_ether_fau_done), 288 OCTEON_CVMSEG_ETHER_OFFSET(dv_unit, csm_ether_fau_done),
286 OCT_FAU_REG_ADDR_END - (8 * (dv_unit + 1))/* XXX */); 289 OCT_FAU_REG_ADDR_END - (8 * (dv_unit + 1))/* XXX */);
287 octfau_op_set_8(&sc->sc_fau_done, 0); 290 octfau_op_set_8(&sc->sc_fau_done, 0);
288 291
289 cnmac_pip_init(sc); 292 cnmac_pip_init(sc);
290 cnmac_ipd_init(sc); 293 cnmac_ipd_init(sc);
291 cnmac_pko_init(sc); 294 cnmac_pko_init(sc);
292 295
293 cnmac_configure_common(sc); 296 cnmac_configure_common(sc);
294 297
295 sc->sc_gmx_port->sc_ipd = sc->sc_ipd; 298 sc->sc_gmx_port->sc_ipd = sc->sc_ipd;
296 sc->sc_gmx_port->sc_port_mii = &sc->sc_mii; 299 sc->sc_gmx_port->sc_port_mii = &sc->sc_mii;
297 sc->sc_gmx_port->sc_port_ec = &sc->sc_ethercom; 300 sc->sc_gmx_port->sc_port_ec = &sc->sc_ethercom;
298 /* XXX */ 301 /* XXX */
299 sc->sc_gmx_port->sc_quirks = sc->sc_quirks; 302 sc->sc_gmx_port->sc_quirks = sc->sc_quirks;
300 303
301 /* XXX */ 304 /* XXX */
302 sc->sc_pow = &octpow_softc; 305 sc->sc_pow = &octpow_softc;
303 306
304 cnmac_mediainit(sc); 307 cnmac_mediainit(sc);
305 308
306 strncpy(ifp->if_xname, device_xname(self), sizeof(ifp->if_xname)); 309 strncpy(ifp->if_xname, device_xname(self), sizeof(ifp->if_xname));
307 ifp->if_softc = sc; 310 ifp->if_softc = sc;
308 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 311 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
309 ifp->if_ioctl = cnmac_ioctl; 312 ifp->if_ioctl = cnmac_ioctl;
310 ifp->if_start = cnmac_start; 313 ifp->if_start = cnmac_start;
311 ifp->if_watchdog = cnmac_watchdog; 314 ifp->if_watchdog = cnmac_watchdog;
312 ifp->if_init = cnmac_init; 315 ifp->if_init = cnmac_init;
313 ifp->if_stop = cnmac_stop; 316 ifp->if_stop = cnmac_stop;
314 IFQ_SET_MAXLEN(&ifp->if_snd, uimax(GATHER_QUEUE_SIZE, IFQ_MAXLEN)); 317 IFQ_SET_MAXLEN(&ifp->if_snd, uimax(GATHER_QUEUE_SIZE, IFQ_MAXLEN));
315 IFQ_SET_READY(&ifp->if_snd); 318 IFQ_SET_READY(&ifp->if_snd);
316 319
317  320
318 ifp->if_capabilities = 321 ifp->if_capabilities =
319#if 0 /* XXX: no tx checksum yet */ 322#if 0 /* XXX: no tx checksum yet */
320 IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx | 323 IFCAP_CSUM_IPv4_Tx | IFCAP_CSUM_IPv4_Rx |
321 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx | 324 IFCAP_CSUM_TCPv4_Tx | IFCAP_CSUM_TCPv4_Rx |
322 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx | 325 IFCAP_CSUM_UDPv4_Tx | IFCAP_CSUM_UDPv4_Rx |
323 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx | 326 IFCAP_CSUM_TCPv6_Tx | IFCAP_CSUM_TCPv6_Rx |
324 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx; 327 IFCAP_CSUM_UDPv6_Tx | IFCAP_CSUM_UDPv6_Rx;
325#else 328#else
326 IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx | 329 IFCAP_CSUM_IPv4_Rx | IFCAP_CSUM_TCPv4_Rx | IFCAP_CSUM_UDPv4_Rx |
327 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx; 330 IFCAP_CSUM_TCPv6_Rx | IFCAP_CSUM_UDPv6_Rx;
328#endif 331#endif
329 332
330 /* 802.1Q VLAN-sized frames are supported */ 333 /* 802.1Q VLAN-sized frames are supported */
331 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU; 334 sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
332 335
333 octgmx_set_mac_addr(sc->sc_gmx_port, enaddr); 336 octgmx_set_mac_addr(sc->sc_gmx_port, enaddr);
334 337
335 if_attach(ifp); 338 if_attach(ifp);
336 ether_ifattach(ifp, enaddr); 339 ether_ifattach(ifp, enaddr);
337 octgmx_set_filter(sc->sc_gmx_port); 340 octgmx_set_filter(sc->sc_gmx_port);
338 341
339#if 1 342#if 1
340 cnmac_buf_init(sc); 343 cnmac_buf_init(sc);
341#endif 344#endif
342 345
343 sc->sc_ih = octeon_intr_establish(POW_WORKQ_IRQ(sc->sc_powgroup), 346 sc->sc_ih = octeon_intr_establish(POW_WORKQ_IRQ(sc->sc_powgroup),
344 IPL_NET, cnmac_intr, sc); 347 IPL_NET, cnmac_intr, sc);
345 if (sc->sc_ih == NULL) 348 if (sc->sc_ih == NULL)
346 panic("%s: could not set up interrupt", device_xname(self)); 349 panic("%s: could not set up interrupt", device_xname(self));
347 350
348 dict = device_properties(sc->sc_gmx->sc_dev); 351 dict = device_properties(sc->sc_gmx->sc_dev);
349 352
350 clk = prop_dictionary_get(dict, "rgmii-tx"); 353 clk = prop_dictionary_get(dict, "rgmii-tx");
351 if (clk) 354 if (clk)
352 sc->sc_gmx_port->sc_clk_tx_setting = 355 sc->sc_gmx_port->sc_clk_tx_setting =
353 prop_number_signed_value(clk); 356 prop_number_signed_value(clk);
354 clk = prop_dictionary_get(dict, "rgmii-rx"); 357 clk = prop_dictionary_get(dict, "rgmii-rx");
355 if (clk) 358 if (clk)
356 sc->sc_gmx_port->sc_clk_rx_setting = 359 sc->sc_gmx_port->sc_clk_rx_setting =
357 prop_number_signed_value(clk); 360 prop_number_signed_value(clk);
358} 361}
359 362
360/* ---- submodules */ 363/* ---- submodules */
361 364
362/* XXX */ 365/* XXX */
363static void 366static void
364cnmac_pip_init(struct cnmac_softc *sc) 367cnmac_pip_init(struct cnmac_softc *sc)
365{ 368{
366 struct octpip_attach_args pip_aa; 369 struct octpip_attach_args pip_aa;
367 370
368 pip_aa.aa_port = sc->sc_port; 371 pip_aa.aa_port = sc->sc_port;
369 pip_aa.aa_regt = sc->sc_regt; 372 pip_aa.aa_regt = sc->sc_regt;
370 pip_aa.aa_tag_type = POW_TAG_TYPE_ORDERED/* XXX */; 373 pip_aa.aa_tag_type = POW_TAG_TYPE_ORDERED/* XXX */;
371 pip_aa.aa_receive_group = sc->sc_powgroup; 374 pip_aa.aa_receive_group = sc->sc_powgroup;
372 pip_aa.aa_ip_offset = sc->sc_ip_offset; 375 pip_aa.aa_ip_offset = sc->sc_ip_offset;
373 octpip_init(&pip_aa, &sc->sc_pip); 376 octpip_init(&pip_aa, &sc->sc_pip);
374 octpip_port_config(sc->sc_pip); 377 octpip_port_config(sc->sc_pip);
375} 378}
376 379
377/* XXX */ 380/* XXX */
378static void 381static void
379cnmac_ipd_init(struct cnmac_softc *sc) 382cnmac_ipd_init(struct cnmac_softc *sc)
380{ 383{
381 struct octipd_attach_args ipd_aa; 384 struct octipd_attach_args ipd_aa;
382 385
383 ipd_aa.aa_port = sc->sc_port; 386 ipd_aa.aa_port = sc->sc_port;
384 ipd_aa.aa_regt = sc->sc_regt; 387 ipd_aa.aa_regt = sc->sc_regt;
385 ipd_aa.aa_first_mbuff_skip = 184/* XXX */; 388 ipd_aa.aa_first_mbuff_skip = 184/* XXX */;
386 ipd_aa.aa_not_first_mbuff_skip = 0/* XXX */; 389 ipd_aa.aa_not_first_mbuff_skip = 0/* XXX */;
387 octipd_init(&ipd_aa, &sc->sc_ipd); 390 octipd_init(&ipd_aa, &sc->sc_ipd);
388} 391}
389 392
390/* XXX */ 393/* XXX */
391static void 394static void
392cnmac_pko_init(struct cnmac_softc *sc) 395cnmac_pko_init(struct cnmac_softc *sc)
393{ 396{
394 struct octpko_attach_args pko_aa; 397 struct octpko_attach_args pko_aa;
395 398
396 pko_aa.aa_port = sc->sc_port; 399 pko_aa.aa_port = sc->sc_port;
397 pko_aa.aa_regt = sc->sc_regt; 400 pko_aa.aa_regt = sc->sc_regt;
398 pko_aa.aa_cmdptr = &sc->sc_cmdptr; 401 pko_aa.aa_cmdptr = &sc->sc_cmdptr;
399 pko_aa.aa_cmd_buf_pool = OCTEON_POOL_NO_CMD; 402 pko_aa.aa_cmd_buf_pool = OCTEON_POOL_NO_CMD;
400 pko_aa.aa_cmd_buf_size = OCTEON_POOL_NWORDS_CMD; 403 pko_aa.aa_cmd_buf_size = OCTEON_POOL_NWORDS_CMD;
401 octpko_init(&pko_aa, &sc->sc_pko); 404 octpko_init(&pko_aa, &sc->sc_pko);
402} 405}
403 406
404/* ---- XXX */ 407/* ---- XXX */
405 408
406#define ADDR2UINT64(u, a) \ 409#define ADDR2UINT64(u, a) \
407 do { \ 410 do { \
408 u = \ 411 u = \
409 (((uint64_t)a[0] << 40) | ((uint64_t)a[1] << 32) | \ 412 (((uint64_t)a[0] << 40) | ((uint64_t)a[1] << 32) | \
410 ((uint64_t)a[2] << 24) | ((uint64_t)a[3] << 16) | \ 413 ((uint64_t)a[2] << 24) | ((uint64_t)a[3] << 16) | \
411 ((uint64_t)a[4] << 8) | ((uint64_t)a[5] << 0)); \ 414 ((uint64_t)a[4] << 8) | ((uint64_t)a[5] << 0)); \
412 } while (0) 415 } while (0)
413#define UINT642ADDR(a, u) \ 416#define UINT642ADDR(a, u) \
414 do { \ 417 do { \
415 a[0] = (uint8_t)((u) >> 40); a[1] = (uint8_t)((u) >> 32); \ 418 a[0] = (uint8_t)((u) >> 40); a[1] = (uint8_t)((u) >> 32); \
416 a[2] = (uint8_t)((u) >> 24); a[3] = (uint8_t)((u) >> 16); \ 419 a[2] = (uint8_t)((u) >> 24); a[3] = (uint8_t)((u) >> 16); \
417 a[4] = (uint8_t)((u) >> 8); a[5] = (uint8_t)((u) >> 0); \ 420 a[4] = (uint8_t)((u) >> 8); a[5] = (uint8_t)((u) >> 0); \
418 } while (0) 421 } while (0)
419 422
420static void 423static void
421cnmac_board_mac_addr(uint8_t *enaddr, size_t size, struct cnmac_softc *sc) 424cnmac_board_mac_addr(uint8_t *enaddr, size_t size, struct cnmac_softc *sc)
422{ 425{
423 prop_dictionary_t dict; 426 prop_dictionary_t dict;
424 prop_data_t ea; 427 prop_data_t ea;
425 428
426 dict = device_properties(sc->sc_dev); 429 dict = device_properties(sc->sc_dev);
427 KASSERT(dict != NULL); 430 KASSERT(dict != NULL);
428 ea = prop_dictionary_get(dict, "mac-address"); 431 ea = prop_dictionary_get(dict, "mac-address");
429 KASSERT(ea != NULL); 432 KASSERT(ea != NULL);
430 memcpy(enaddr, prop_data_value(ea), size); 433 memcpy(enaddr, prop_data_value(ea), size);
431} 434}
432 435
433/* ---- media */ 436/* ---- media */
434 437
435static int 438static int
436cnmac_mii_readreg(device_t self, int phy_addr, int reg, uint16_t *val) 439cnmac_mii_readreg(device_t self, int phy_addr, int reg, uint16_t *val)
437{ 440{
438 struct cnmac_softc *sc = device_private(self); 441 struct cnmac_softc *sc = device_private(self);
439 442
440 return octsmi_read(sc->sc_smi, phy_addr, reg, val); 443 return octsmi_read(sc->sc_smi, phy_addr, reg, val);
441} 444}
442 445
443static int 446static int
444cnmac_mii_writereg(device_t self, int phy_addr, int reg, uint16_t val) 447cnmac_mii_writereg(device_t self, int phy_addr, int reg, uint16_t val)
445{ 448{
446 struct cnmac_softc *sc = device_private(self); 449 struct cnmac_softc *sc = device_private(self);
447 450
448 return octsmi_write(sc->sc_smi, phy_addr, reg, val); 451 return octsmi_write(sc->sc_smi, phy_addr, reg, val);
449} 452}
450 453
451static void 454static void
452cnmac_mii_statchg(struct ifnet *ifp) 455cnmac_mii_statchg(struct ifnet *ifp)
453{ 456{
454 struct cnmac_softc *sc = ifp->if_softc; 457 struct cnmac_softc *sc = ifp->if_softc;
455 458
456 octpko_port_enable(sc->sc_pko, 0); 459 octpko_port_enable(sc->sc_pko, 0);
457 octgmx_port_enable(sc->sc_gmx_port, 0); 460 octgmx_port_enable(sc->sc_gmx_port, 0);
458 461
459 cnmac_reset(sc); 462 cnmac_reset(sc);
460 463
461 if (ISSET(ifp->if_flags, IFF_RUNNING)) 464 if (ISSET(ifp->if_flags, IFF_RUNNING))
462 octgmx_set_filter(sc->sc_gmx_port); 465 octgmx_set_filter(sc->sc_gmx_port);
463 466
464 octpko_port_enable(sc->sc_pko, 1); 467 octpko_port_enable(sc->sc_pko, 1);
465 octgmx_port_enable(sc->sc_gmx_port, 1); 468 octgmx_port_enable(sc->sc_gmx_port, 1);
466} 469}
467 470
468static int 471static int
469cnmac_mediainit(struct cnmac_softc *sc) 472cnmac_mediainit(struct cnmac_softc *sc)
470{ 473{
471 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 474 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
472 struct mii_data *mii = &sc->sc_mii; 475 struct mii_data *mii = &sc->sc_mii;
473 prop_object_t phy; 476 prop_object_t phy;
474 477
475 mii->mii_ifp = ifp; 478 mii->mii_ifp = ifp;
476 mii->mii_readreg = cnmac_mii_readreg; 479 mii->mii_readreg = cnmac_mii_readreg;
477 mii->mii_writereg = cnmac_mii_writereg; 480 mii->mii_writereg = cnmac_mii_writereg;
478 mii->mii_statchg = cnmac_mii_statchg; 481 mii->mii_statchg = cnmac_mii_statchg;
479 sc->sc_ethercom.ec_mii = mii; 482 sc->sc_ethercom.ec_mii = mii;
480 483
481 /* Initialize ifmedia structures. */ 484 /* Initialize ifmedia structures. */
482 ifmedia_init(&mii->mii_media, 0, ether_mediachange, cnmac_mediastatus); 485 ifmedia_init(&mii->mii_media, 0, ether_mediachange, cnmac_mediastatus);
483 486
484 phy = prop_dictionary_get(device_properties(sc->sc_dev), "phy-addr"); 487 phy = prop_dictionary_get(device_properties(sc->sc_dev), "phy-addr");
485 KASSERT(phy != NULL); 488 KASSERT(phy != NULL);
486 489
487 mii_attach(sc->sc_dev, mii, 0xffffffff, prop_number_signed_value(phy), 490 mii_attach(sc->sc_dev, mii, 0xffffffff, prop_number_signed_value(phy),
488 MII_OFFSET_ANY, MIIF_DOPAUSE); 491 MII_OFFSET_ANY, MIIF_DOPAUSE);
489 492
490 /* XXX XXX XXX */ 493 /* XXX XXX XXX */
491 if (LIST_FIRST(&mii->mii_phys) != NULL) { 494 if (LIST_FIRST(&mii->mii_phys) != NULL) {
492 /* XXX XXX XXX */ 495 /* XXX XXX XXX */
493 ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_AUTO); 496 ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_AUTO);
494 /* XXX XXX XXX */ 497 /* XXX XXX XXX */
495 } else { 498 } else {
496 /* XXX XXX XXX */ 499 /* XXX XXX XXX */
497 ifmedia_add(&mii->mii_media, IFM_ETHER | IFM_NONE, 500 ifmedia_add(&mii->mii_media, IFM_ETHER | IFM_NONE,
498 MII_MEDIA_NONE, NULL); 501 MII_MEDIA_NONE, NULL);
499 /* XXX XXX XXX */ 502 /* XXX XXX XXX */
500 ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_NONE); 503 ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_NONE);
501 /* XXX XXX XXX */ 504 /* XXX XXX XXX */
502 } 505 }
503 /* XXX XXX XXX */ 506 /* XXX XXX XXX */
504 507
505 return 0; 508 return 0;
506} 509}
507 510
508static void 511static void
509cnmac_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr) 512cnmac_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
510{ 513{
511 struct cnmac_softc *sc = ifp->if_softc; 514 struct cnmac_softc *sc = ifp->if_softc;
512 515
513 mii_pollstat(&sc->sc_mii); 516 mii_pollstat(&sc->sc_mii);
514 517
515 ifmr->ifm_status = sc->sc_mii.mii_media_status; 518 ifmr->ifm_status = sc->sc_mii.mii_media_status;
516 ifmr->ifm_active = sc->sc_mii.mii_media_active; 519 ifmr->ifm_active = sc->sc_mii.mii_media_active;
517 ifmr->ifm_active = (sc->sc_mii.mii_media_active & ~IFM_ETH_FMASK) | 520 ifmr->ifm_active = (sc->sc_mii.mii_media_active & ~IFM_ETH_FMASK) |
518 sc->sc_gmx_port->sc_port_flowflags; 521 sc->sc_gmx_port->sc_port_flowflags;
519} 522}
520 523
521/* ---- send buffer garbage collection */ 524/* ---- send buffer garbage collection */
522 525
523static inline void 526static inline void
524cnmac_send_queue_flush_prefetch(struct cnmac_softc *sc) 527cnmac_send_queue_flush_prefetch(struct cnmac_softc *sc)
525{ 528{
526 529
527 KASSERT(sc->sc_prefetch == 0); 530 KASSERT(sc->sc_prefetch == 0);
528 octfau_op_inc_fetch_8(&sc->sc_fau_done, 0); 531 octfau_op_inc_fetch_8(&sc->sc_fau_done, 0);
529 sc->sc_prefetch = 1; 532 sc->sc_prefetch = 1;
530} 533}
531 534
532static inline void 535static inline void
533cnmac_send_queue_flush_fetch(struct cnmac_softc *sc) 536cnmac_send_queue_flush_fetch(struct cnmac_softc *sc)
534{ 537{
535 538
536 KASSERT(sc->sc_prefetch == 1); 539 KASSERT(sc->sc_prefetch == 1);
537 sc->sc_hard_done_cnt = octfau_op_inc_read_8(&sc->sc_fau_done); 540 sc->sc_hard_done_cnt = octfau_op_inc_read_8(&sc->sc_fau_done);
538 KASSERT(sc->sc_hard_done_cnt <= 0); 541 KASSERT(sc->sc_hard_done_cnt <= 0);
539 sc->sc_prefetch = 0; 542 sc->sc_prefetch = 0;
540} 543}
541 544
542static inline void 545static inline void
543cnmac_send_queue_flush(struct cnmac_softc *sc) 546cnmac_send_queue_flush(struct cnmac_softc *sc)
544{ 547{
545 struct ifnet *ifp = &sc->sc_ethercom.ec_if; 548 struct ifnet *ifp = &sc->sc_ethercom.ec_if;
546 const int64_t sent_count = sc->sc_hard_done_cnt; 549 const int64_t sent_count = sc->sc_hard_done_cnt;
547 int i; 550 int i;
548 551
549 KASSERT(sc->sc_flush == 0); 552 KASSERT(sc->sc_flush == 0);
550 KASSERT(sent_count <= 0); 553 KASSERT(sent_count <= 0);
551 554
552 for (i = 0; i < 0 - sent_count; i++) { 555 for (i = 0; i < 0 - sent_count; i++) {
553 struct mbuf *m; 556 struct mbuf *m;
554 uint64_t *gbuf; 557 uint64_t *gbuf;
555 558
556 cnmac_send_queue_del(sc, &m, &gbuf); 559 cnmac_send_queue_del(sc, &m, &gbuf);
557 560
558 octfpa_buf_put(cnmac_fb_sg, gbuf); 561 octfpa_buf_put(cnmac_fb_sg, gbuf);
559 562
560 m_freem(m); 563 m_freem(m);
561 564
562 CLR(ifp->if_flags, IFF_OACTIVE); 565 CLR(ifp->if_flags, IFF_OACTIVE);
563 } 566 }
564 567
565 octfau_op_inc_fetch_8(&sc->sc_fau_done, i); 568 octfau_op_inc_fetch_8(&sc->sc_fau_done, i);
566 sc->sc_flush = i; 569 sc->sc_flush = i;
567} 570}
568 571
569static inline void 572static inline void
570cnmac_send_queue_flush_sync(struct cnmac_softc *sc) 573cnmac_send_queue_flush_sync(struct cnmac_softc *sc)
571{ 574{
572 if (sc->sc_flush == 0) 575 if (sc->sc_flush == 0)
573 return; 576 return;
574 577
575 KASSERT(sc->sc_flush > 0); 578 KASSERT(sc->sc_flush > 0);
576 579
577 /* XXX XXX XXX */ 580 /* XXX XXX XXX */
578 octfau_op_inc_read_8(&sc->sc_fau_done); 581 octfau_op_inc_read_8(&sc->sc_fau_done);
579 sc->sc_soft_req_cnt -= sc->sc_flush; 582 sc->sc_soft_req_cnt -= sc->sc_flush;
580 KASSERT(sc->sc_soft_req_cnt >= 0); 583 KASSERT(sc->sc_soft_req_cnt >= 0);
581 /* XXX XXX XXX */ 584 /* XXX XXX XXX */
582 585
583 sc->sc_flush = 0; 586 sc->sc_flush = 0;
584} 587}
585 588
586static inline int 589static inline int
587cnmac_send_queue_is_full(struct cnmac_softc *sc) 590cnmac_send_queue_is_full(struct cnmac_softc *sc)
588{ 591{
589#ifdef CNMAC_SEND_QUEUE_CHECK 592#ifdef CNMAC_SEND_QUEUE_CHECK
590 int64_t nofree_cnt; 593 int64_t nofree_cnt;
591 594
592 nofree_cnt = sc->sc_soft_req_cnt + sc->sc_hard_done_cnt; 595 nofree_cnt = sc->sc_soft_req_cnt + sc->sc_hard_done_cnt;
593 596
594 if (__predict_false(nofree_cnt == GATHER_QUEUE_SIZE - 1)) { 597 if (__predict_false(nofree_cnt == GATHER_QUEUE_SIZE - 1)) {
595 cnmac_send_queue_flush(sc); 598 cnmac_send_queue_flush(sc);
596 cnmac_send_queue_flush_sync(sc); 599 cnmac_send_queue_flush_sync(sc);
597 return 1; 600 return 1;
598 } 601 }
599 602
600#endif 603#endif
601 return 0; 604 return 0;
602} 605}
603 606
604/* 607/*
605 * (Ab)use m_nextpkt and m_paddr to maintain mbuf chain and pointer to gather 608 * (Ab)use m_nextpkt and m_paddr to maintain mbuf chain and pointer to gather
606 * buffer. Other mbuf members may be used by m_freem(), so don't touch them! 609 * buffer. Other mbuf members may be used by m_freem(), so don't touch them!
607 */ 610 */
608 611
609struct _send_queue_entry { 612struct _send_queue_entry {
610 union { 613 union {
611 struct mbuf _sqe_s_mbuf; 614 struct mbuf _sqe_s_mbuf;
612 struct { 615 struct {
613 char _sqe_s_entry_pad[offsetof(struct mbuf, m_nextpkt)]; 616 char _sqe_s_entry_pad[offsetof(struct mbuf, m_nextpkt)];
614 SIMPLEQ_ENTRY(_send_queue_entry) _sqe_s_entry_entry; 617 SIMPLEQ_ENTRY(_send_queue_entry) _sqe_s_entry_entry;
615 } _sqe_s_entry; 618 } _sqe_s_entry;
616 struct { 619 struct {
617 char _sqe_s_gbuf_pad[offsetof(struct mbuf, m_paddr)]; 620 char _sqe_s_gbuf_pad[offsetof(struct mbuf, m_paddr)];
618 uint64_t *_sqe_s_gbuf_gbuf; 621 uint64_t *_sqe_s_gbuf_gbuf;
619 } _sqe_s_gbuf; 622 } _sqe_s_gbuf;
620 } _sqe_u; 623 } _sqe_u;
621#define _sqe_entry _sqe_u._sqe_s_entry._sqe_s_entry_entry 624#define _sqe_entry _sqe_u._sqe_s_entry._sqe_s_entry_entry
622#define _sqe_gbuf _sqe_u._sqe_s_gbuf._sqe_s_gbuf_gbuf 625#define _sqe_gbuf _sqe_u._sqe_s_gbuf._sqe_s_gbuf_gbuf
623}; 626};
624 627
625static inline void 628static inline void
626cnmac_send_queue_add(struct cnmac_softc *sc, struct mbuf *m, 629cnmac_send_queue_add(struct cnmac_softc *sc, struct mbuf *m,
627 uint64_t *gbuf) 630 uint64_t *gbuf)
628{ 631{
629 struct _send_queue_entry *sqe = (struct _send_queue_entry *)m; 632 struct _send_queue_entry *sqe = (struct _send_queue_entry *)m;
630 633
631 sqe->_sqe_gbuf = gbuf; 634 sqe->_sqe_gbuf = gbuf;
632 SIMPLEQ_INSERT_TAIL(&sc->sc_sendq, sqe, _sqe_entry); 635 SIMPLEQ_INSERT_TAIL(&sc->sc_sendq, sqe, _sqe_entry);
633 636
634 if ((m->m_flags & M_EXT) && m->m_ext.ext_free != NULL) 637 if ((m->m_flags & M_EXT) && m->m_ext.ext_free != NULL)
635 sc->sc_ext_callback_cnt++; 638 sc->sc_ext_callback_cnt++;
636} 639}
637 640
638static inline void 641static inline void
639cnmac_send_queue_del(struct cnmac_softc *sc, struct mbuf **rm, uint64_t **rgbuf) 642cnmac_send_queue_del(struct cnmac_softc *sc, struct mbuf **rm, uint64_t **rgbuf)
640{ 643{
641 struct _send_queue_entry *sqe; 644 struct _send_queue_entry *sqe;
642 645
643 sqe = SIMPLEQ_FIRST(&sc->sc_sendq); 646 sqe = SIMPLEQ_FIRST(&sc->sc_sendq);
644 KASSERT(sqe != NULL); 647 KASSERT(sqe != NULL);
645 SIMPLEQ_REMOVE_HEAD(&sc->sc_sendq, _sqe_entry); 648 SIMPLEQ_REMOVE_HEAD(&sc->sc_sendq, _sqe_entry);
646 649
647 *rm = (void *)sqe; 650 *rm = (void *)sqe;
648 *rgbuf = sqe->_sqe_gbuf; 651 *rgbuf = sqe->_sqe_gbuf;
649 652
650 if (((*rm)->m_flags & M_EXT) && (*rm)->m_ext.ext_free != NULL) { 653 if (((*rm)->m_flags & M_EXT) && (*rm)->m_ext.ext_free != NULL) {
651 sc->sc_ext_callback_cnt--; 654 sc->sc_ext_callback_cnt--;
652 KASSERT(sc->sc_ext_callback_cnt >= 0); 655 KASSERT(sc->sc_ext_callback_cnt >= 0);
653 } 656 }
654} 657}
655 658
656static inline int 659static inline int
657cnmac_buf_free_work(struct cnmac_softc *sc, uint64_t *work) 660cnmac_buf_free_work(struct cnmac_softc *sc, uint64_t *work)
658{ 661{
659 662
660 /* XXX when jumbo frame */ 663 /* XXX when jumbo frame */
661 if (ISSET(work[2], PIP_WQE_WORD2_IP_BUFS)) { 664 if (ISSET(work[2], PIP_WQE_WORD2_IP_BUFS)) {
662 paddr_t addr; 665 paddr_t addr;
663 paddr_t start_buffer; 666 paddr_t start_buffer;
664 667
665 addr = work[3] & PIP_WQE_WORD3_ADDR; 668 addr = work[3] & PIP_WQE_WORD3_ADDR;
666 start_buffer = addr & ~(2048 - 1); 669 start_buffer = addr & ~(2048 - 1);
667 670
668 octfpa_buf_put_paddr(cnmac_fb_pkt, start_buffer); 671 octfpa_buf_put_paddr(cnmac_fb_pkt, start_buffer);
669 } 672 }
670 673
671 octfpa_buf_put(cnmac_fb_wqe, work); 674 octfpa_buf_put(cnmac_fb_wqe, work);
672 675
673 return 0; 676 return 0;
674} 677}
675 678
676static inline void 679static inline void
677cnmac_buf_ext_free(struct mbuf *m, void *buf, size_t size, void *arg) 680cnmac_buf_ext_free(struct mbuf *m, void *buf, size_t size, void *arg)
678{ 681{
679 octfpa_buf_put(cnmac_fb_pkt, buf); 682 octfpa_buf_put(cnmac_fb_pkt, buf);
680 683
681 KASSERT(m != NULL); 684 KASSERT(m != NULL);
682 685
683 pool_cache_put(mb_cache, m); 686 pool_cache_put(mb_cache, m);
684} 687}
685 688
686/* ---- ifnet interfaces */ 689/* ---- ifnet interfaces */
687 690
688static int 691static int
689cnmac_ioctl(struct ifnet *ifp, u_long cmd, void *data) 692cnmac_ioctl(struct ifnet *ifp, u_long cmd, void *data)
690{ 693{
691 struct cnmac_softc *sc = ifp->if_softc; 694 struct cnmac_softc *sc = ifp->if_softc;
692 struct ifreq *ifr = (struct ifreq *)data; 695 struct ifreq *ifr = (struct ifreq *)data;
693 int s, error; 696 int s, error;
694 697
695 s = splnet(); 698 s = splnet();
696 switch (cmd) { 699 switch (cmd) {
697 case SIOCSIFMEDIA: 700 case SIOCSIFMEDIA:
698 /* Flow control requires full-duplex mode. */ 701 /* Flow control requires full-duplex mode. */
699 if (IFM_SUBTYPE(ifr->ifr_media) == IFM_AUTO || 702 if (IFM_SUBTYPE(ifr->ifr_media) == IFM_AUTO ||
700 (ifr->ifr_media & IFM_FDX) == 0) { 703 (ifr->ifr_media & IFM_FDX) == 0) {
701 ifr->ifr_media &= ~IFM_ETH_FMASK; 704 ifr->ifr_media &= ~IFM_ETH_FMASK;
702 } 705 }
703 if (IFM_SUBTYPE(ifr->ifr_media) != IFM_AUTO) { 706 if (IFM_SUBTYPE(ifr->ifr_media) != IFM_AUTO) {
704 if ((ifr->ifr_media & IFM_ETH_FMASK) == IFM_FLOW) { 707 if ((ifr->ifr_media & IFM_ETH_FMASK) == IFM_FLOW) {
705 ifr->ifr_media |= 708 ifr->ifr_media |=
706 IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE; 709 IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE;
707 } 710 }
708 sc->sc_gmx_port->sc_port_flowflags = 711 sc->sc_gmx_port->sc_port_flowflags =
709 ifr->ifr_media & IFM_ETH_FMASK; 712 ifr->ifr_media & IFM_ETH_FMASK;
710 } 713 }
711 error = ifmedia_ioctl(ifp, ifr, &sc->sc_mii.mii_media, cmd); 714 error = ifmedia_ioctl(ifp, ifr, &sc->sc_mii.mii_media, cmd);
712 break; 715 break;
713 default: 716 default:
714 error = ether_ioctl(ifp, cmd, data); 717 error = ether_ioctl(ifp, cmd, data);
715 break; 718 break;
716 } 719 }
717 720
718 if (error == ENETRESET) { 721 if (error == ENETRESET) {
719 if (ISSET(ifp->if_flags, IFF_RUNNING)) 722 if (ISSET(ifp->if_flags, IFF_RUNNING))
720 octgmx_set_filter(sc->sc_gmx_port); 723 octgmx_set_filter(sc->sc_gmx_port);
721 error = 0; 724 error = 0;
722 } 725 }
723 726
724 cnmac_start(ifp); 727 cnmac_start(ifp);
725 728
726 splx(s); 729 splx(s);
727 730
728 return error; 731 return error;
729} 732}
730 733
731/* ---- send (output) */ 734/* ---- send (output) */
732 735
733static inline uint64_t 736static inline uint64_t
734cnmac_send_makecmd_w0(uint64_t fau0, uint64_t fau1, size_t len, int segs, 737cnmac_send_makecmd_w0(uint64_t fau0, uint64_t fau1, size_t len, int segs,
735 int ipoffp1) 738 int ipoffp1)
736{ 739{
737 740
738 return octpko_cmd_word0( 741 return octpko_cmd_word0(
739 OCT_FAU_OP_SIZE_64, /* sz1 */ 742 OCT_FAU_OP_SIZE_64, /* sz1 */
740 OCT_FAU_OP_SIZE_64, /* sz0 */ 743 OCT_FAU_OP_SIZE_64, /* sz0 */
741 1, fau1, 1, fau0, /* s1, reg1, s0, reg0 */ 744 1, fau1, 1, fau0, /* s1, reg1, s0, reg0 */
742 0, /* le */ 745 0, /* le */
743 cnmac_param_pko_cmd_w0_n2, /* n2 */ 746 cnmac_param_pko_cmd_w0_n2, /* n2 */
744 1, 0, /* q, r */ 747 1, 0, /* q, r */
745 (segs == 1) ? 0 : 1, /* g */ 748 (segs == 1) ? 0 : 1, /* g */
746 0, 0, 1, /* ipoffp1, ii, df */ 749 0, 0, 1, /* ipoffp1, ii, df */
747 segs, (int)len); /* segs, totalbytes */ 750 segs, (int)len); /* segs, totalbytes */
748} 751}
749 752
750static inline uint64_t 753static inline uint64_t
751cnmac_send_makecmd_w1(int size, paddr_t addr) 754cnmac_send_makecmd_w1(int size, paddr_t addr)
752{ 755{
753 756
754 return octpko_cmd_word1( 757 return octpko_cmd_word1(
755 0, 0, /* i, back */ 758 0, 0, /* i, back */
756 OCTEON_POOL_NO_SG, /* pool */ 759 OCTEON_POOL_NO_SG, /* pool */
757 size, addr); /* size, addr */ 760 size, addr); /* size, addr */
758} 761}
759 762
760static inline int 763static inline int
761cnmac_send_makecmd_gbuf(struct cnmac_softc *sc, struct mbuf *m0, uint64_t *gbuf, 764cnmac_send_makecmd_gbuf(struct cnmac_softc *sc, struct mbuf *m0, uint64_t *gbuf,
762 int *rsegs) 765 int *rsegs)
763{ 766{
764 struct mbuf *m; 767 struct mbuf *m;
765 int segs = 0; 768 int segs = 0;
766 uintptr_t laddr, rlen, nlen; 769 uintptr_t laddr, rlen, nlen;
767 770
768 for (m = m0; m != NULL; m = m->m_next) { 771 for (m = m0; m != NULL; m = m->m_next) {
769 772
770 if (__predict_false(m->m_len == 0)) 773 if (__predict_false(m->m_len == 0))
771 continue; 774 continue;
772 775
773 /* Aligned 4k */ 776 /* Aligned 4k */
774 laddr = (uintptr_t)m->m_data & (PAGE_SIZE - 1); 777 laddr = (uintptr_t)m->m_data & (PAGE_SIZE - 1);
775 778
776 if (laddr + m->m_len > PAGE_SIZE) { 779 if (laddr + m->m_len > PAGE_SIZE) {
777 /* XXX XXX XXX */ 780 /* XXX XXX XXX */
778 rlen = PAGE_SIZE - laddr; 781 rlen = PAGE_SIZE - laddr;
779 nlen = m->m_len - rlen; 782 nlen = m->m_len - rlen;
780 *(gbuf + segs) = cnmac_send_makecmd_w1(rlen, 783 *(gbuf + segs) = cnmac_send_makecmd_w1(rlen,
781 kvtophys((vaddr_t)m->m_data)); 784 kvtophys((vaddr_t)m->m_data));
782 segs++; 785 segs++;
783 if (segs > 63) { 786 if (segs > 63) {
784 return 1; 787 return 1;
785 } 788 }
786 /* XXX XXX XXX */ 789 /* XXX XXX XXX */
787 } else { 790 } else {
788 rlen = 0; 791 rlen = 0;
789 nlen = m->m_len; 792 nlen = m->m_len;
790 } 793 }
791 794
792 *(gbuf + segs) = cnmac_send_makecmd_w1(nlen, 795 *(gbuf + segs) = cnmac_send_makecmd_w1(nlen,
793 kvtophys((vaddr_t)(m->m_data + rlen))); 796 kvtophys((vaddr_t)(m->m_data + rlen)));
794 segs++; 797 segs++;
795 if (segs > 63) { 798 if (segs > 63) {
796 return 1; 799 return 1;
797 } 800 }
798 } 801 }
799 802
800 KASSERT(m == NULL); 803 KASSERT(m == NULL);
801 804
802 *rsegs = segs; 805 *rsegs = segs;
803 806
804 return 0; 807 return 0;
805} 808}
806 809
807static inline int 810static inline int
808cnmac_send_makecmd(struct cnmac_softc *sc, struct mbuf *m, 811cnmac_send_makecmd(struct cnmac_softc *sc, struct mbuf *m,
809 uint64_t *gbuf, uint64_t *rpko_cmd_w0, uint64_t *rpko_cmd_w1) 812 uint64_t *gbuf, uint64_t *rpko_cmd_w0, uint64_t *rpko_cmd_w1)
810{ 813{
811 uint64_t pko_cmd_w0, pko_cmd_w1; 814 uint64_t pko_cmd_w0, pko_cmd_w1;
812 int ipoffp1; 815 int ipoffp1;
813 int segs; 816 int segs;
814 int result = 0; 817 int result = 0;
815 818
816 if (cnmac_send_makecmd_gbuf(sc, m, gbuf, &segs)) { 819 if (cnmac_send_makecmd_gbuf(sc, m, gbuf, &segs)) {
817 log(LOG_WARNING, "%s: there are a lot of number of segments" 820 log(LOG_WARNING, "%s: there are a lot of number of segments"
818 " of transmission data", device_xname(sc->sc_dev)); 821 " of transmission data", device_xname(sc->sc_dev));
819 result = 1; 822 result = 1;
820 goto done; 823 goto done;
821 } 824 }
822 825
823 /* Get the IP packet offset for TCP/UDP checksum offloading. */ 826 /* Get the IP packet offset for TCP/UDP checksum offloading. */
824 ipoffp1 = (m->m_pkthdr.csum_flags & (M_CSUM_TCPv4 | M_CSUM_UDPv4)) 827 ipoffp1 = (m->m_pkthdr.csum_flags & (M_CSUM_TCPv4 | M_CSUM_UDPv4))
825 ? (ETHER_HDR_LEN + 1) : 0; 828 ? (ETHER_HDR_LEN + 1) : 0;
826 829
827 /* 830 /*
828 * segs == 1 -> link mode (single continuous buffer) 831 * segs == 1 -> link mode (single continuous buffer)
829 * WORD1[size] is number of bytes pointed by segment 832 * WORD1[size] is number of bytes pointed by segment
830 * 833 *
831 * segs > 1 -> gather mode (scatter-gather buffer) 834 * segs > 1 -> gather mode (scatter-gather buffer)
832 * WORD1[size] is number of segments 835 * WORD1[size] is number of segments
833 */ 836 */
834 pko_cmd_w0 = cnmac_send_makecmd_w0(sc->sc_fau_done.fd_regno, 837 pko_cmd_w0 = cnmac_send_makecmd_w0(sc->sc_fau_done.fd_regno,
835 0, m->m_pkthdr.len, segs, ipoffp1); 838 0, m->m_pkthdr.len, segs, ipoffp1);
836 if (segs == 1) { 839 if (segs == 1) {
837 pko_cmd_w1 = cnmac_send_makecmd_w1( 840 pko_cmd_w1 = cnmac_send_makecmd_w1(
838 m->m_pkthdr.len, kvtophys((vaddr_t)m->m_data)); 841 m->m_pkthdr.len, kvtophys((vaddr_t)m->m_data));
839 } else { 842 } else {
840#ifdef __mips_n32 843#ifdef __mips_n32
841 KASSERT(MIPS_KSEG0_P(gbuf)); 844 KASSERT(MIPS_KSEG0_P(gbuf));
842 pko_cmd_w1 = cnmac_send_makecmd_w1(segs, 845 pko_cmd_w1 = cnmac_send_makecmd_w1(segs,
843 MIPS_KSEG0_TO_PHYS(gbuf)); 846 MIPS_KSEG0_TO_PHYS(gbuf));
844#else 847#else
845 pko_cmd_w1 = cnmac_send_makecmd_w1(segs, 848 pko_cmd_w1 = cnmac_send_makecmd_w1(segs,
846 MIPS_XKPHYS_TO_PHYS(gbuf)); 849 MIPS_XKPHYS_TO_PHYS(gbuf));
847#endif 850#endif
848 } 851 }
849 852
850 *rpko_cmd_w0 = pko_cmd_w0; 853 *rpko_cmd_w0 = pko_cmd_w0;
851 *rpko_cmd_w1 = pko_cmd_w1; 854 *rpko_cmd_w1 = pko_cmd_w1;
852 855
853done: 856done:
854 return result; 857 return result;
855} 858}
856 859
857static inline int 860static inline int
858cnmac_send_cmd(struct cnmac_softc *sc, uint64_t pko_cmd_w0, 861cnmac_send_cmd(struct cnmac_softc *sc, uint64_t pko_cmd_w0,
859 uint64_t pko_cmd_w1, int *pwdc) 862 uint64_t pko_cmd_w1, int *pwdc)
860{ 863{
861 uint64_t *cmdptr; 864 uint64_t *cmdptr;
862 int result = 0; 865 int result = 0;
863 866
864#ifdef __mips_n32 867#ifdef __mips_n32
865 KASSERT((sc->sc_cmdptr.cmdptr & ~MIPS_PHYS_MASK) == 0); 868 KASSERT((sc->sc_cmdptr.cmdptr & ~MIPS_PHYS_MASK) == 0);
866 cmdptr = (uint64_t *)MIPS_PHYS_TO_KSEG0(sc->sc_cmdptr.cmdptr); 869 cmdptr = (uint64_t *)MIPS_PHYS_TO_KSEG0(sc->sc_cmdptr.cmdptr);
867#else 870#else
868 cmdptr = (uint64_t *)MIPS_PHYS_TO_XKPHYS_CACHED(sc->sc_cmdptr.cmdptr); 871 cmdptr = (uint64_t *)MIPS_PHYS_TO_XKPHYS_CACHED(sc->sc_cmdptr.cmdptr);
869#endif 872#endif
870 cmdptr += sc->sc_cmdptr.cmdptr_idx; 873 cmdptr += sc->sc_cmdptr.cmdptr_idx;
871 874
872 KASSERT(cmdptr != NULL); 875 KASSERT(cmdptr != NULL);
873 876
874 *cmdptr++ = pko_cmd_w0; 877 *cmdptr++ = pko_cmd_w0;
875 *cmdptr++ = pko_cmd_w1; 878 *cmdptr++ = pko_cmd_w1;
876 879
877 KASSERT(sc->sc_cmdptr.cmdptr_idx + 2 <= FPA_COMMAND_BUFFER_POOL_NWORDS - 1); 880 KASSERT(sc->sc_cmdptr.cmdptr_idx + 2 <= FPA_COMMAND_BUFFER_POOL_NWORDS - 1);
878 881
879 if (sc->sc_cmdptr.cmdptr_idx + 2 == FPA_COMMAND_BUFFER_POOL_NWORDS - 1) { 882 if (sc->sc_cmdptr.cmdptr_idx + 2 == FPA_COMMAND_BUFFER_POOL_NWORDS - 1) {
880 paddr_t buf; 883 paddr_t buf;
881 884
882 buf = octfpa_buf_get_paddr(cnmac_fb_cmd); 885 buf = octfpa_buf_get_paddr(cnmac_fb_cmd);
883 if (buf == 0) { 886 if (buf == 0) {
884 log(LOG_WARNING, 887 log(LOG_WARNING,
885 "%s: can not allocate command buffer from free pool allocator\n", 888 "%s: can not allocate command buffer from free pool allocator\n",
886 device_xname(sc->sc_dev)); 889 device_xname(sc->sc_dev));
887 result = 1; 890 result = 1;
888 goto done; 891 goto done;
889 } 892 }
890 *cmdptr++ = buf; 893 *cmdptr++ = buf;
891 sc->sc_cmdptr.cmdptr = (uint64_t)buf; 894 sc->sc_cmdptr.cmdptr = (uint64_t)buf;
892 sc->sc_cmdptr.cmdptr_idx = 0; 895 sc->sc_cmdptr.cmdptr_idx = 0;
893 } else { 896 } else {
894 sc->sc_cmdptr.cmdptr_idx += 2; 897 sc->sc_cmdptr.cmdptr_idx += 2;
895 } 898 }
896 899
897 *pwdc += 2; 900 *pwdc += 2;
898 901
899done: 902done:
900 return result; 903 return result;
901} 904}
902 905
903static inline int 906static inline int
904cnmac_send_buf(struct cnmac_softc *sc, struct mbuf *m, uint64_t *gbuf, 907cnmac_send_buf(struct cnmac_softc *sc, struct mbuf *m, uint64_t *gbuf,
905 int *pwdc) 908 int *pwdc)
906{ 909{
907 int result = 0, error; 910 int result = 0, error;
908 uint64_t pko_cmd_w0, pko_cmd_w1; 911 uint64_t pko_cmd_w0, pko_cmd_w1;
909 912
910 error = cnmac_send_makecmd(sc, m, gbuf, &pko_cmd_w0, &pko_cmd_w1); 913 error = cnmac_send_makecmd(sc, m, gbuf, &pko_cmd_w0, &pko_cmd_w1);
911 if (error != 0) { 914 if (error != 0) {
912 /* Already logging */ 915 /* Already logging */
913 result = error; 916 result = error;
914 goto done; 917 goto done;
915 } 918 }
916 919
917 error = cnmac_send_cmd(sc, pko_cmd_w0, pko_cmd_w1, pwdc); 920 error = cnmac_send_cmd(sc, pko_cmd_w0, pko_cmd_w1, pwdc);
918 if (error != 0) { 921 if (error != 0) {
919 /* Already logging */ 922 /* Already logging */
920 result = error; 923 result = error;
921 } 924 }
922 925
923done: 926done:
924 return result; 927 return result;
925} 928}
926 929
927static inline int 930static inline int
928cnmac_send(struct cnmac_softc *sc, struct mbuf *m, int *pwdc) 931cnmac_send(struct cnmac_softc *sc, struct mbuf *m, int *pwdc)
929{ 932{
930 paddr_t gaddr = 0; 933 paddr_t gaddr = 0;
931 uint64_t *gbuf = NULL; 934 uint64_t *gbuf = NULL;
932 int result = 0, error; 935 int result = 0, error;
933 936
934 gaddr = octfpa_buf_get_paddr(cnmac_fb_sg); 937 gaddr = octfpa_buf_get_paddr(cnmac_fb_sg);
935 if (gaddr == 0) { 938 if (gaddr == 0) {
936 log(LOG_WARNING, "%s: can not allocate gather buffer from " 939 log(LOG_WARNING, "%s: can not allocate gather buffer from "
937 "free pool allocator\n", device_xname(sc->sc_dev)); 940 "free pool allocator\n", device_xname(sc->sc_dev));
938 result = 1; 941 result = 1;
939 goto done; 942 goto done;
940 } 943 }
941 944
942#ifdef __mips_n32 945#ifdef __mips_n32
943 KASSERT((gaddr & ~MIPS_PHYS_MASK) == 0); 946 KASSERT((gaddr & ~MIPS_PHYS_MASK) == 0);
944 gbuf = (uint64_t *)(uintptr_t)MIPS_PHYS_TO_KSEG0(gaddr); 947 gbuf = (uint64_t *)(uintptr_t)MIPS_PHYS_TO_KSEG0(gaddr);
945#else 948#else
946 gbuf = (uint64_t *)(uintptr_t)MIPS_PHYS_TO_XKPHYS_CACHED(gaddr); 949 gbuf = (uint64_t *)(uintptr_t)MIPS_PHYS_TO_XKPHYS_CACHED(gaddr);
947#endif 950#endif
948 951
949 KASSERT(gbuf != NULL); 952 KASSERT(gbuf != NULL);
950 953
951 error = cnmac_send_buf(sc, m, gbuf, pwdc); 954 error = cnmac_send_buf(sc, m, gbuf, pwdc);
952 if (error != 0) { 955 if (error != 0) {
953 /* Already logging */ 956 /* Already logging */
954 octfpa_buf_put_paddr(cnmac_fb_sg, gaddr); 957 octfpa_buf_put_paddr(cnmac_fb_sg, gaddr);
955 result = error; 958 result = error;
956 goto done; 959 goto done;
957 } 960 }
958 961
959 cnmac_send_queue_add(sc, m, gbuf); 962 cnmac_send_queue_add(sc, m, gbuf);
960 963
961done: 964done:
962 return result; 965 return result;
963} 966}
964 967
965static void 968static void
966cnmac_start(struct ifnet *ifp) 969cnmac_start(struct ifnet *ifp)
967{ 970{
968 struct cnmac_softc *sc = ifp->if_softc; 971 struct cnmac_softc *sc = ifp->if_softc;
969 struct mbuf *m; 972 struct mbuf *m;
970 int wdc = 0; 973 int wdc = 0;
971 974
972 /* 975 /*
973 * Performance tuning 976 * Performance tuning
974 * pre-send iobdma request 977 * pre-send iobdma request
975 */ 978 */
976 cnmac_send_queue_flush_prefetch(sc); 979 cnmac_send_queue_flush_prefetch(sc);
977 980
978 if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING) 981 if ((ifp->if_flags & (IFF_RUNNING | IFF_OACTIVE)) != IFF_RUNNING)
979 goto last; 982 goto last;
980 983
981 if (__predict_false(!octgmx_link_status(sc->sc_gmx_port))) 984 if (__predict_false(!octgmx_link_status(sc->sc_gmx_port)))
982 goto last; 985 goto last;
983 986
984 for (;;) { 987 for (;;) {
985 IFQ_POLL(&ifp->if_snd, m); 988 IFQ_POLL(&ifp->if_snd, m);
986 if (__predict_false(m == NULL)) 989 if (__predict_false(m == NULL))
987 break; 990 break;
988 991
989 /* XXX XXX XXX */ 992 /* XXX XXX XXX */
990 cnmac_send_queue_flush_fetch(sc); 993 cnmac_send_queue_flush_fetch(sc);
991 994
992 /* 995 /*
993 * If no free send buffer is available, free all the sent 996 * If no free send buffer is available, free all the sent
994 * buffers and bail out. 997 * buffers and bail out.
995 */ 998 */
996 if (cnmac_send_queue_is_full(sc)) { 999 if (cnmac_send_queue_is_full(sc)) {
997 SET(ifp->if_flags, IFF_OACTIVE); 1000 SET(ifp->if_flags, IFF_OACTIVE);
998 if (wdc > 0) 1001 if (wdc > 0)
999 octpko_op_doorbell_write(sc->sc_port, 1002 octpko_op_doorbell_write(sc->sc_port,
1000 sc->sc_port, wdc); 1003 sc->sc_port, wdc);
 1004 callout_schedule(&sc->sc_tick_free_ch, 1);
1001 return; 1005 return;
1002 } 1006 }
1003 /* XXX XXX XXX */ 1007 /* XXX XXX XXX */
1004 1008
1005 IFQ_DEQUEUE(&ifp->if_snd, m); 1009 IFQ_DEQUEUE(&ifp->if_snd, m);
1006 1010
1007 bpf_mtap(ifp, m, BPF_D_OUT); 1011 bpf_mtap(ifp, m, BPF_D_OUT);
1008 1012
1009 /* XXX XXX XXX */ 1013 /* XXX XXX XXX */
1010 if (sc->sc_soft_req_cnt > sc->sc_soft_req_thresh) 1014 if (sc->sc_soft_req_cnt > sc->sc_soft_req_thresh)
1011 cnmac_send_queue_flush(sc); 1015 cnmac_send_queue_flush(sc);
1012 if (cnmac_send(sc, m, &wdc)) { 1016 if (cnmac_send(sc, m, &wdc)) {
1013 IF_DROP(&ifp->if_snd); 1017 IF_DROP(&ifp->if_snd);
1014 m_freem(m); 1018 m_freem(m);
1015 log(LOG_WARNING, 1019 log(LOG_WARNING,
1016 "%s: failed in the transmission of the packet\n", 1020 "%s: failed in the transmission of the packet\n",
1017 device_xname(sc->sc_dev)); 1021 device_xname(sc->sc_dev));
1018 } else 1022 } else
1019 sc->sc_soft_req_cnt++; 1023 sc->sc_soft_req_cnt++;
1020 1024
1021 if (sc->sc_flush) 1025 if (sc->sc_flush)
1022 cnmac_send_queue_flush_sync(sc); 1026 cnmac_send_queue_flush_sync(sc);
1023 /* XXX XXX XXX */ 1027 /* XXX XXX XXX */
1024 1028
1025 /* Send next iobdma request */ 1029 /* Send next iobdma request */
1026 cnmac_send_queue_flush_prefetch(sc); 1030 cnmac_send_queue_flush_prefetch(sc);
1027 } 1031 }
1028 1032
1029 if (wdc > 0) 1033 if (wdc > 0)
1030 octpko_op_doorbell_write(sc->sc_port, sc->sc_port, wdc); 1034 octpko_op_doorbell_write(sc->sc_port, sc->sc_port, wdc);
1031 1035
1032last: 1036last:
1033 cnmac_send_queue_flush_fetch(sc); 1037 cnmac_send_queue_flush_fetch(sc);
 1038 callout_schedule(&sc->sc_tick_free_ch, 1);
1034} 1039}
1035 1040
1036static void 1041static void
1037cnmac_watchdog(struct ifnet *ifp) 1042cnmac_watchdog(struct ifnet *ifp)
1038{ 1043{
1039 struct cnmac_softc *sc = ifp->if_softc; 1044 struct cnmac_softc *sc = ifp->if_softc;
1040 1045
1041 printf("%s: device timeout\n", device_xname(sc->sc_dev)); 1046 printf("%s: device timeout\n", device_xname(sc->sc_dev));
1042 1047
1043 cnmac_configure(sc); 1048 cnmac_configure(sc);
1044 1049
1045 SET(ifp->if_flags, IFF_RUNNING); 1050 SET(ifp->if_flags, IFF_RUNNING);
1046 CLR(ifp->if_flags, IFF_OACTIVE); 1051 CLR(ifp->if_flags, IFF_OACTIVE);
1047 ifp->if_timer = 0; 1052 ifp->if_timer = 0;
1048 1053
1049 cnmac_start(ifp); 1054 cnmac_start(ifp);
1050} 1055}
1051 1056
1052static int 1057static int
1053cnmac_init(struct ifnet *ifp) 1058cnmac_init(struct ifnet *ifp)
1054{ 1059{
1055 struct cnmac_softc *sc = ifp->if_softc; 1060 struct cnmac_softc *sc = ifp->if_softc;
1056 1061
1057 /* XXX don't disable commonly used parts!!! XXX */ 1062 /* XXX don't disable commonly used parts!!! XXX */
1058 if (sc->sc_init_flag == 0) { 1063 if (sc->sc_init_flag == 0) {
1059 /* Cancel any pending I/O. */ 1064 /* Cancel any pending I/O. */
1060 cnmac_stop(ifp, 0); 1065 cnmac_stop(ifp, 0);
1061 1066
1062 /* Initialize the device */ 1067 /* Initialize the device */
1063 cnmac_configure(sc); 1068 cnmac_configure(sc);
1064 1069
1065 octpko_enable(sc->sc_pko); 1070 octpko_enable(sc->sc_pko);
1066 octipd_enable(sc->sc_ipd); 1071 octipd_enable(sc->sc_ipd);
1067 1072
1068 sc->sc_init_flag = 1; 1073 sc->sc_init_flag = 1;
1069 } else { 1074 } else {
1070 octgmx_port_enable(sc->sc_gmx_port, 1); 1075 octgmx_port_enable(sc->sc_gmx_port, 1);
1071 } 1076 }
1072 mii_ifmedia_change(&sc->sc_mii); 1077 mii_ifmedia_change(&sc->sc_mii);
1073 1078
1074 octgmx_set_filter(sc->sc_gmx_port); 1079 octgmx_set_filter(sc->sc_gmx_port);
1075 1080
1076 callout_reset(&sc->sc_tick_misc_ch, hz, cnmac_tick_misc, sc); 1081 callout_schedule(&sc->sc_tick_misc_ch, hz);
1077 callout_reset(&sc->sc_tick_free_ch, hz, cnmac_tick_free, sc); 1082 callout_schedule(&sc->sc_tick_free_ch, hz);
1078 1083
1079 SET(ifp->if_flags, IFF_RUNNING); 1084 SET(ifp->if_flags, IFF_RUNNING);
1080 CLR(ifp->if_flags, IFF_OACTIVE); 1085 CLR(ifp->if_flags, IFF_OACTIVE);
1081 1086
1082 return 0; 1087 return 0;
1083} 1088}
1084 1089
1085static void 1090static void
1086cnmac_stop(struct ifnet *ifp, int disable) 1091cnmac_stop(struct ifnet *ifp, int disable)
1087{ 1092{
1088 struct cnmac_softc *sc = ifp->if_softc; 1093 struct cnmac_softc *sc = ifp->if_softc;
1089 1094
1090 callout_stop(&sc->sc_tick_misc_ch); 1095 callout_stop(&sc->sc_tick_misc_ch);
1091 callout_stop(&sc->sc_tick_free_ch); 1096 callout_stop(&sc->sc_tick_free_ch);
1092 1097
1093 mii_down(&sc->sc_mii); 1098 mii_down(&sc->sc_mii);
1094 1099
1095 octgmx_port_enable(sc->sc_gmx_port, 0); 1100 octgmx_port_enable(sc->sc_gmx_port, 0);
1096 1101
1097 /* Mark the interface as down and cancel the watchdog timer. */ 1102 /* Mark the interface as down and cancel the watchdog timer. */
1098 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE); 1103 CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE);
1099 ifp->if_timer = 0; 1104 ifp->if_timer = 0;
1100 
1101} 1105}
1102 1106
1103/* ---- misc */ 1107/* ---- misc */
1104 1108
1105static int 1109static int
1106cnmac_reset(struct cnmac_softc *sc) 1110cnmac_reset(struct cnmac_softc *sc)
1107{ 1111{
1108 octgmx_reset_speed(sc->sc_gmx_port); 1112 octgmx_reset_speed(sc->sc_gmx_port);
1109 octgmx_reset_flowctl(sc->sc_gmx_port); 1113 octgmx_reset_flowctl(sc->sc_gmx_port);
1110 octgmx_reset_timing(sc->sc_gmx_port); 1114 octgmx_reset_timing(sc->sc_gmx_port);
1111 1115
1112 return 0; 1116 return 0;
1113} 1117}
1114 1118
1115static int 1119static int
1116cnmac_configure(struct cnmac_softc *sc) 1120cnmac_configure(struct cnmac_softc *sc)
1117{ 1121{
1118 octgmx_port_enable(sc->sc_gmx_port, 0); 1122 octgmx_port_enable(sc->sc_gmx_port, 0);
1119 1123
1120 cnmac_reset(sc); 1124 cnmac_reset(sc);
1121 1125
1122 cnmac_configure_common(sc); 1126 cnmac_configure_common(sc);
1123 1127
1124 octpko_port_config(sc->sc_pko); 1128 octpko_port_config(sc->sc_pko);
1125 octpko_port_enable(sc->sc_pko, 1); 1129 octpko_port_enable(sc->sc_pko, 1);
1126 octpow_config(sc->sc_pow, sc->sc_powgroup); 1130 octpow_config(sc->sc_pow, sc->sc_powgroup);
1127 1131
1128 octgmx_tx_stats_rd_clr(sc->sc_gmx_port, 1); 1132 octgmx_tx_stats_rd_clr(sc->sc_gmx_port, 1);
1129 octgmx_rx_stats_rd_clr(sc->sc_gmx_port, 1); 1133 octgmx_rx_stats_rd_clr(sc->sc_gmx_port, 1);
1130 1134
1131 octgmx_port_enable(sc->sc_gmx_port, 1); 1135 octgmx_port_enable(sc->sc_gmx_port, 1);
1132 1136
1133 return 0; 1137 return 0;
1134} 1138}
1135 1139
1136static int 1140static int
1137cnmac_configure_common(struct cnmac_softc *sc) 1141cnmac_configure_common(struct cnmac_softc *sc)
1138{ 1142{
1139 static int once; 1143 static int once;
1140 1144
1141 if (once == 1) 1145 if (once == 1)
1142 return 0; 1146 return 0;
1143 once = 1; 1147 once = 1;
1144 1148
1145 octipd_config(sc->sc_ipd); 1149 octipd_config(sc->sc_ipd);
1146 octpko_config(sc->sc_pko); 1150 octpko_config(sc->sc_pko);
1147 1151
1148 return 0; 1152 return 0;
1149} 1153}
1150 1154
1151/* ---- receive (input) */ 1155/* ---- receive (input) */
1152 1156
1153static inline int 1157static inline int
1154cnmac_recv_mbuf(struct cnmac_softc *sc, uint64_t *work, struct mbuf **rm) 1158cnmac_recv_mbuf(struct cnmac_softc *sc, uint64_t *work, struct mbuf **rm)
1155{ 1159{
1156 struct mbuf *m; 1160 struct mbuf *m;
1157 vaddr_t addr; 1161 vaddr_t addr;
1158 vaddr_t ext_buf; 1162 vaddr_t ext_buf;
1159 size_t ext_size; 1163 size_t ext_size;
1160 uint64_t word1 = work[1]; 1164 uint64_t word1 = work[1];
1161 uint64_t word2 = work[2]; 1165 uint64_t word2 = work[2];
1162 uint64_t word3 = work[3]; 1166 uint64_t word3 = work[3];
1163 1167
1164 MGETHDR(m, M_NOWAIT, MT_DATA); 1168 MGETHDR(m, M_NOWAIT, MT_DATA);
1165 if (m == NULL) 1169 if (m == NULL)
1166 return 1; 1170 return 1;
1167 1171
1168 octfpa_buf_put(cnmac_fb_wqe, work); 1172 octfpa_buf_put(cnmac_fb_wqe, work);
1169 1173
1170 if (__SHIFTOUT(word2, PIP_WQE_WORD2_IP_BUFS) != 1) 1174 if (__SHIFTOUT(word2, PIP_WQE_WORD2_IP_BUFS) != 1)
1171 panic("%s: expected one buffer, got %" PRId64, __func__, 1175 panic("%s: expected one buffer, got %" PRId64, __func__,
1172 __SHIFTOUT(word2, PIP_WQE_WORD2_IP_BUFS)); 1176 __SHIFTOUT(word2, PIP_WQE_WORD2_IP_BUFS));
1173 1177
1174 1178
1175#ifdef __mips_n32 1179#ifdef __mips_n32
1176 KASSERT((word3 & ~MIPS_PHYS_MASK) == 0); 1180 KASSERT((word3 & ~MIPS_PHYS_MASK) == 0);
1177 addr = MIPS_PHYS_TO_KSEG0(word3 & PIP_WQE_WORD3_ADDR); 1181 addr = MIPS_PHYS_TO_KSEG0(word3 & PIP_WQE_WORD3_ADDR);
1178#else 1182#else
1179 addr = MIPS_PHYS_TO_XKPHYS_CACHED(word3 & PIP_WQE_WORD3_ADDR); 1183 addr = MIPS_PHYS_TO_XKPHYS_CACHED(word3 & PIP_WQE_WORD3_ADDR);
1180#endif 1184#endif
1181 1185
1182 ext_size = OCTEON_POOL_SIZE_PKT; 1186 ext_size = OCTEON_POOL_SIZE_PKT;
1183 ext_buf = addr & ~(ext_size - 1); 1187 ext_buf = addr & ~(ext_size - 1);
1184 MEXTADD(m, ext_buf, ext_size, 0, cnmac_buf_ext_free, NULL); 1188 MEXTADD(m, ext_buf, ext_size, 0, cnmac_buf_ext_free, NULL);
1185 1189
1186 m->m_data = (void *)addr; 1190 m->m_data = (void *)addr;
1187 m->m_len = m->m_pkthdr.len = (word1 & PIP_WQE_WORD1_LEN) >> 48; 1191 m->m_len = m->m_pkthdr.len = (word1 & PIP_WQE_WORD1_LEN) >> 48;
1188 m_set_rcvif(m, &sc->sc_ethercom.ec_if); 1192 m_set_rcvif(m, &sc->sc_ethercom.ec_if);
1189 1193
1190 /* Not readonly buffer */ 1194 /* Not readonly buffer */
1191 m->m_flags |= M_EXT_RW; 1195 m->m_flags |= M_EXT_RW;
1192 1196
1193 *rm = m; 1197 *rm = m;
1194 1198
1195 KASSERT(*rm != NULL); 1199 KASSERT(*rm != NULL);
1196 1200
1197 return 0; 1201 return 0;
1198} 1202}
1199 1203
1200static inline int 1204static inline int
1201cnmac_recv_check(struct cnmac_softc *sc, uint64_t word2) 1205cnmac_recv_check(struct cnmac_softc *sc, uint64_t word2)
1202{ 1206{
1203 static struct timeval rxerr_log_interval = { 0, 2500000 }; 1207 static struct timeval rxerr_log_interval = { 0, 2500000 };
1204 uint64_t opecode; 1208 uint64_t opecode;
1205 1209
1206 if (__predict_true(!ISSET(word2, PIP_WQE_WORD2_NOIP_RE))) 1210 if (__predict_true(!ISSET(word2, PIP_WQE_WORD2_NOIP_RE)))
1207 return 0; 1211 return 0;
1208 1212
1209 opecode = word2 & PIP_WQE_WORD2_NOIP_OPECODE; 1213 opecode = word2 & PIP_WQE_WORD2_NOIP_OPECODE;
1210 if ((sc->sc_ethercom.ec_if.if_flags & IFF_DEBUG) && 1214 if ((sc->sc_ethercom.ec_if.if_flags & IFF_DEBUG) &&
1211 ratecheck(&sc->sc_rxerr_log_last, &rxerr_log_interval)) 1215 ratecheck(&sc->sc_rxerr_log_last, &rxerr_log_interval))
1212 log(LOG_DEBUG, "%s: rx error (%"PRId64")\n", 1216 log(LOG_DEBUG, "%s: rx error (%"PRId64")\n",
1213 device_xname(sc->sc_dev), opecode); 1217 device_xname(sc->sc_dev), opecode);
1214 1218
1215 /* This error is harmless */ 1219 /* This error is harmless */
1216 if (opecode == PIP_WQE_WORD2_RE_OPCODE_OVRRUN) 1220 if (opecode == PIP_WQE_WORD2_RE_OPCODE_OVRRUN)
1217 return 0; 1221 return 0;
1218 1222
1219 return 1; 1223 return 1;
1220} 1224}
1221 1225
1222static inline int 1226static inline int
1223cnmac_recv(struct cnmac_softc *sc, uint64_t *work) 1227cnmac_recv(struct cnmac_softc *sc, uint64_t *work)
1224{ 1228{
1225 struct ifnet *ifp; 1229 struct ifnet *ifp;
1226 struct mbuf *m; 1230 struct mbuf *m;
1227 uint64_t word2; 1231 uint64_t word2;
1228 1232
1229 KASSERT(sc != NULL); 1233 KASSERT(sc != NULL);
1230 KASSERT(work != NULL); 1234 KASSERT(work != NULL);
1231 1235
1232 word2 = work[2]; 1236 word2 = work[2];
1233 ifp = &sc->sc_ethercom.ec_if; 1237 ifp = &sc->sc_ethercom.ec_if;
1234 1238
1235 KASSERT(ifp != NULL); 1239 KASSERT(ifp != NULL);
1236 1240
1237 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 1241 if (!ISSET(ifp->if_flags, IFF_RUNNING))
1238 goto drop; 1242 goto drop;
1239 1243
1240 if (__predict_false(cnmac_recv_check(sc, word2) != 0)) { 1244 if (__predict_false(cnmac_recv_check(sc, word2) != 0)) {
1241 if_statinc(ifp, if_ierrors); 1245 if_statinc(ifp, if_ierrors);
1242 goto drop; 1246 goto drop;
1243 } 1247 }
1244 1248
1245 if (__predict_false(cnmac_recv_mbuf(sc, work, &m) != 0)) { 1249 if (__predict_false(cnmac_recv_mbuf(sc, work, &m) != 0)) {
1246 if_statinc(ifp, if_ierrors); 1250 if_statinc(ifp, if_ierrors);
1247 goto drop; 1251 goto drop;
1248 } 1252 }
1249 1253
1250 /* work[0] .. work[3] may not be valid any more */ 1254 /* work[0] .. work[3] may not be valid any more */
1251 1255
1252 KASSERT(m != NULL); 1256 KASSERT(m != NULL);
1253 1257
1254 octipd_offload(word2, m->m_data, &m->m_pkthdr.csum_flags); 1258 octipd_offload(word2, m->m_data, &m->m_pkthdr.csum_flags);
1255 1259
1256 if_percpuq_enqueue(ifp->if_percpuq, m); 1260 if_percpuq_enqueue(ifp->if_percpuq, m);
1257 1261
1258 return 0; 1262 return 0;
1259 1263
1260drop: 1264drop:
1261 cnmac_buf_free_work(sc, work); 1265 cnmac_buf_free_work(sc, work);
1262 return 1; 1266 return 1;
1263} 1267}
1264 1268
1265static int 1269static int
1266cnmac_intr(void *arg) 1270cnmac_intr(void *arg)
1267{ 1271{
1268 struct cnmac_softc *sc = arg; 1272 struct cnmac_softc *sc = arg;
1269 uint64_t *work; 1273 uint64_t *work;
1270 uint64_t wqmask = __BIT(sc->sc_powgroup); 1274 uint64_t wqmask = __BIT(sc->sc_powgroup);
1271 uint32_t coreid = 0; /* XXX octeon_get_coreid() */ 1275 uint32_t coreid = 0; /* XXX octeon_get_coreid() */
1272 uint32_t port; 1276 uint32_t port;
1273 1277
1274 _POW_WR8(sc->sc_pow, POW_PP_GRP_MSK_OFFSET(coreid), wqmask); 1278 _POW_WR8(sc->sc_pow, POW_PP_GRP_MSK_OFFSET(coreid), wqmask);
1275 1279
1276 octpow_tag_sw_wait(); 1280 octpow_tag_sw_wait();
1277 octpow_work_request_async(OCTEON_CVMSEG_OFFSET(csm_pow_intr), 1281 octpow_work_request_async(OCTEON_CVMSEG_OFFSET(csm_pow_intr),
1278 POW_NO_WAIT); 1282 POW_NO_WAIT);
1279 1283
1280 for (;;) { 1284 for (;;) {
1281 work = (uint64_t *)octpow_work_response_async( 1285 work = (uint64_t *)octpow_work_response_async(
1282 OCTEON_CVMSEG_OFFSET(csm_pow_intr)); 1286 OCTEON_CVMSEG_OFFSET(csm_pow_intr));
1283 if (work == NULL) 1287 if (work == NULL)
1284 break; 1288 break;
1285 1289
1286 octpow_tag_sw_wait(); 1290 octpow_tag_sw_wait();
1287 octpow_work_request_async(OCTEON_CVMSEG_OFFSET(csm_pow_intr), 1291 octpow_work_request_async(OCTEON_CVMSEG_OFFSET(csm_pow_intr),
1288 POW_NO_WAIT); 1292 POW_NO_WAIT);
1289 1293
1290 port = __SHIFTOUT(work[1], PIP_WQE_WORD1_IPRT); 1294 port = __SHIFTOUT(work[1], PIP_WQE_WORD1_IPRT);
1291 if (port != sc->sc_port) { 1295 if (port != sc->sc_port) {
1292 printf("%s: unexpected wqe port %u, should be %u\n", 1296 printf("%s: unexpected wqe port %u, should be %u\n",
1293 device_xname(sc->sc_dev), port, sc->sc_port); 1297 device_xname(sc->sc_dev), port, sc->sc_port);
1294 goto wqe_error; 1298 goto wqe_error;
1295 } 1299 }
1296 1300
1297 (void)cnmac_recv(sc, work); 1301 (void)cnmac_recv(sc, work);
1298 } 1302 }
1299 1303
1300 _POW_WR8(sc->sc_pow, POW_WQ_INT_OFFSET, wqmask); 1304 _POW_WR8(sc->sc_pow, POW_WQ_INT_OFFSET, wqmask);
1301 1305
1302 return 1; 1306 return 1;
1303 1307
1304wqe_error: 1308wqe_error:
1305 printf("word0: 0x%016" PRIx64 "\n", work[0]); 1309 printf("word0: 0x%016" PRIx64 "\n", work[0]);
1306 printf("word1: 0x%016" PRIx64 "\n", work[1]); 1310 printf("word1: 0x%016" PRIx64 "\n", work[1]);
1307 printf("word2: 0x%016" PRIx64 "\n", work[2]); 1311 printf("word2: 0x%016" PRIx64 "\n", work[2]);
1308 printf("word3: 0x%016" PRIx64 "\n", work[3]); 1312 printf("word3: 0x%016" PRIx64 "\n", work[3]);
1309 panic("wqe_error"); 1313 panic("wqe_error");
1310} 1314}
1311 1315
1312/* ---- tick */ 1316/* ---- tick */
1313 1317
1314/* 1318/*
1315 * cnmac_tick_free 1319 * cnmac_tick_free
1316 * 1320 *
1317 * => garbage collect send gather buffer / mbuf 1321 * => garbage collect send gather buffer / mbuf
1318 * => called at softclock 1322 * => called at softclock
1319 */ 1323 */
1320static void 1324static void
1321cnmac_tick_free(void *arg) 1325cnmac_tick_free(void *arg)
1322{ 1326{
1323 struct cnmac_softc *sc = arg; 1327 struct cnmac_softc *sc = arg;
1324 int timo; 1328 int timo;
1325 int s; 1329 int s;
1326 1330
1327 s = splnet(); 1331 s = splnet();
1328 /* XXX XXX XXX */ 1332 /* XXX XXX XXX */
1329 if (sc->sc_soft_req_cnt > 0) { 1333 if (sc->sc_soft_req_cnt > 0) {
1330 cnmac_send_queue_flush_prefetch(sc); 1334 cnmac_send_queue_flush_prefetch(sc);
1331 cnmac_send_queue_flush_fetch(sc); 1335 cnmac_send_queue_flush_fetch(sc);
1332 cnmac_send_queue_flush(sc); 1336 cnmac_send_queue_flush(sc);
1333 cnmac_send_queue_flush_sync(sc); 1337 cnmac_send_queue_flush_sync(sc);
1334 } 1338 }
1335 /* XXX XXX XXX */ 1339 /* XXX XXX XXX */
1336 1340
1337 /* XXX XXX XXX */ 1341 timo = (sc->sc_ext_callback_cnt > 0) ? 1 : hz;
1338 /* ??? */ 
1339 timo = hz - (100 * sc->sc_ext_callback_cnt); 
1340 if (timo < 10) 
1341 timo = 10; 
1342 callout_schedule(&sc->sc_tick_free_ch, timo); 1342 callout_schedule(&sc->sc_tick_free_ch, timo);
1343 /* XXX XXX XXX */ 
1344 splx(s); 1343 splx(s);
1345} 1344}
1346 1345
1347/* 1346/*
1348 * cnmac_tick_misc 1347 * cnmac_tick_misc
1349 * 1348 *
1350 * => collect statistics 1349 * => collect statistics
1351 * => check link status 1350 * => check link status
1352 * => called at softclock 1351 * => called at softclock
1353 */ 1352 */
1354static void 1353static void
1355cnmac_tick_misc(void *arg) 1354cnmac_tick_misc(void *arg)
1356{ 1355{
1357 struct cnmac_softc *sc = arg; 1356 struct cnmac_softc *sc = arg;
1358 struct ifnet *ifp; 1357 struct ifnet *ifp;
1359 int s; 1358 int s;
1360 1359
1361 s = splnet(); 1360 s = splnet();
1362 1361
1363 ifp = &sc->sc_ethercom.ec_if; 1362 ifp = &sc->sc_ethercom.ec_if;
1364 1363
1365 octgmx_stats(sc->sc_gmx_port); 1364 octgmx_stats(sc->sc_gmx_port);
1366 octpip_stats(sc->sc_pip, ifp, sc->sc_port); 1365 octpip_stats(sc->sc_pip, ifp, sc->sc_port);
1367 mii_tick(&sc->sc_mii); 1366 mii_tick(&sc->sc_mii);
1368 1367
1369 splx(s); 1368 splx(s);
1370 1369
1371 callout_schedule(&sc->sc_tick_misc_ch, hz); 1370 callout_schedule(&sc->sc_tick_misc_ch, hz);
1372} 1371}