1 /* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */
2
3 #ifndef _FUNETH_TXRX_H
4 #define _FUNETH_TXRX_H
5
6 #include <linux/netdevice.h>
7 #include <linux/u64_stats_sync.h>
8
9 /* Tx descriptor size */
10 #define FUNETH_SQE_SIZE 64U
11
12 /* Size of device headers per Tx packet */
13 #define FUNETH_FUNOS_HDR_SZ (sizeof(struct fun_eth_tx_req))
14
15 /* Number of gather list entries per Tx descriptor */
16 #define FUNETH_GLE_PER_DESC (FUNETH_SQE_SIZE / sizeof(struct fun_dataop_gl))
17
18 /* Max gather list size in bytes for an sk_buff. */
19 #define FUNETH_MAX_GL_SZ ((MAX_SKB_FRAGS + 1) * sizeof(struct fun_dataop_gl))
20
21 #if IS_ENABLED(CONFIG_TLS_DEVICE)
22 # define FUNETH_TLS_SZ sizeof(struct fun_eth_tls)
23 #else
24 # define FUNETH_TLS_SZ 0
25 #endif
26
27 /* Max number of Tx descriptors for an sk_buff using a gather list. */
28 #define FUNETH_MAX_GL_DESC \
29 DIV_ROUND_UP((FUNETH_FUNOS_HDR_SZ + FUNETH_MAX_GL_SZ + FUNETH_TLS_SZ), \
30 FUNETH_SQE_SIZE)
31
32 /* Max number of Tx descriptors for any packet. */
33 #define FUNETH_MAX_PKT_DESC FUNETH_MAX_GL_DESC
34
35 /* Rx CQ descriptor size. */
36 #define FUNETH_CQE_SIZE 64U
37
38 /* Offset of cqe_info within a CQE. */
39 #define FUNETH_CQE_INFO_OFFSET (FUNETH_CQE_SIZE - sizeof(struct fun_cqe_info))
40
41 /* Construct the IRQ portion of a CQ doorbell. The resulting value arms the
42 * interrupt with the supplied time delay and packet count moderation settings.
43 */
44 #define FUN_IRQ_CQ_DB(usec, pkts) \
45 (FUN_DB_IRQ_ARM_F | ((usec) << FUN_DB_INTCOAL_USEC_S) | \
46 ((pkts) << FUN_DB_INTCOAL_ENTRIES_S))
47
48 /* As above for SQ doorbells. */
49 #define FUN_IRQ_SQ_DB(usec, pkts) \
50 (FUN_DB_IRQ_ARM_F | \
51 ((usec) << FUN_DB_INTCOAL_USEC_S) | \
52 ((pkts) << FUN_DB_INTCOAL_ENTRIES_S))
53
54 /* Per packet tailroom. Present only for 1-frag packets. */
55 #define FUN_RX_TAILROOM SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
56
57 /* Per packet headroom for XDP. Preferred over XDP_PACKET_HEADROOM to
58 * accommodate two packets per buffer for 4K pages and 1500B MTUs.
59 */
60 #define FUN_XDP_HEADROOM 192
61
62 /* Initialization state of a queue. */
63 enum {
64 FUN_QSTATE_DESTROYED, /* what queue? */
65 FUN_QSTATE_INIT_SW, /* exists in SW, not on the device */
66 FUN_QSTATE_INIT_FULL, /* exists both in SW and on device */
67 };
68
69 /* Initialization state of an interrupt. */
70 enum {
71 FUN_IRQ_INIT, /* initialized and in the XArray but inactive */
72 FUN_IRQ_REQUESTED, /* request_irq() done */
73 FUN_IRQ_ENABLED, /* processing enabled */
74 FUN_IRQ_DISABLED, /* processing disabled */
75 };
76
77 struct bpf_prog;
78
79 struct funeth_txq_stats { /* per Tx queue SW counters */
80 u64 tx_pkts; /* # of Tx packets */
81 u64 tx_bytes; /* total bytes of Tx packets */
82 u64 tx_cso; /* # of packets with checksum offload */
83 u64 tx_tso; /* # of non-encapsulated TSO super-packets */
84 u64 tx_encap_tso; /* # of encapsulated TSO super-packets */
85 u64 tx_more; /* # of DBs elided due to xmit_more */
86 u64 tx_nstops; /* # of times the queue has stopped */
87 u64 tx_nrestarts; /* # of times the queue has restarted */
88 u64 tx_map_err; /* # of packets dropped due to DMA mapping errors */
89 u64 tx_xdp_full; /* # of XDP packets that could not be enqueued */
90 u64 tx_tls_pkts; /* # of Tx TLS packets offloaded to HW */
91 u64 tx_tls_bytes; /* Tx bytes of HW-handled TLS payload */
92 u64 tx_tls_fallback; /* attempted Tx TLS offloads punted to SW */
93 u64 tx_tls_drops; /* attempted Tx TLS offloads dropped */
94 };
95
96 struct funeth_tx_info { /* per Tx descriptor state */
97 union {
98 struct sk_buff *skb; /* associated packet (sk_buff path) */
99 struct xdp_frame *xdpf; /* associated XDP frame (XDP path) */
100 };
101 };
102
103 struct funeth_txq {
104 /* RO cacheline of frequently accessed data */
105 u32 mask; /* queue depth - 1 */
106 u32 hw_qid; /* device ID of the queue */
107 void *desc; /* base address of descriptor ring */
108 struct funeth_tx_info *info;
109 struct device *dma_dev; /* device for DMA mappings */
110 volatile __be64 *hw_wb; /* HW write-back location */
111 u32 __iomem *db; /* SQ doorbell register address */
112 struct netdev_queue *ndq;
113 dma_addr_t dma_addr; /* DMA address of descriptor ring */
114 /* producer R/W cacheline */
115 u16 qidx; /* queue index within net_device */
116 u16 ethid;
117 u32 prod_cnt; /* producer counter */
118 struct funeth_txq_stats stats;
119 /* shared R/W cacheline, primarily accessed by consumer */
120 u32 irq_db_val; /* value written to IRQ doorbell */
121 u32 cons_cnt; /* consumer (cleanup) counter */
122 struct net_device *netdev;
123 struct fun_irq *irq;
124 int numa_node;
125 u8 init_state; /* queue initialization state */
126 struct u64_stats_sync syncp;
127 };
128
129 struct funeth_rxq_stats { /* per Rx queue SW counters */
130 u64 rx_pkts; /* # of received packets, including SW drops */
131 u64 rx_bytes; /* total size of received packets */
132 u64 rx_cso; /* # of packets with checksum offload */
133 u64 rx_bufs; /* total # of Rx buffers provided to device */
134 u64 gro_pkts; /* # of GRO superpackets */
135 u64 gro_merged; /* # of pkts merged into existing GRO superpackets */
136 u64 rx_page_alloc; /* # of page allocations for Rx buffers */
137 u64 rx_budget; /* NAPI iterations that exhausted their budget */
138 u64 rx_mem_drops; /* # of packets dropped due to memory shortage */
139 u64 rx_map_err; /* # of page DMA mapping errors */
140 u64 xdp_drops; /* XDP_DROPped packets */
141 u64 xdp_tx; /* successful XDP transmits */
142 u64 xdp_redir; /* successful XDP redirects */
143 u64 xdp_err; /* packets dropped due to XDP errors */
144 };
145
146 struct funeth_rxbuf { /* per Rx buffer state */
147 struct page *page; /* associated page */
148 dma_addr_t dma_addr; /* DMA address of page start */
149 int pg_refs; /* page refs held by driver */
150 int node; /* page node, or -1 if it is PF_MEMALLOC */
151 };
152
153 struct funeth_rx_cache { /* cache of DMA-mapped previously used buffers */
154 struct funeth_rxbuf *bufs; /* base of Rx buffer state ring */
155 unsigned int prod_cnt; /* producer counter */
156 unsigned int cons_cnt; /* consumer counter */
157 unsigned int mask; /* depth - 1 */
158 };
159
160 /* An Rx queue consists of a CQ and an SQ used to provide Rx buffers. */
161 struct funeth_rxq {
162 struct net_device *netdev;
163 struct napi_struct *napi;
164 struct device *dma_dev; /* device for DMA mappings */
165 void *cqes; /* base of CQ descriptor ring */
166 const void *next_cqe_info; /* fun_cqe_info of next CQE */
167 u32 __iomem *cq_db; /* CQ doorbell register address */
168 unsigned int cq_head; /* CQ head index */
169 unsigned int cq_mask; /* CQ depth - 1 */
170 u16 phase; /* CQ phase tag */
171 u16 qidx; /* queue index within net_device */
172 unsigned int irq_db_val; /* IRQ info for CQ doorbell */
173 struct fun_eprq_rqbuf *rqes; /* base of RQ descriptor ring */
174 struct funeth_rxbuf *bufs; /* base of Rx buffer state ring */
175 struct funeth_rxbuf *cur_buf; /* currently active buffer */
176 u32 __iomem *rq_db; /* RQ doorbell register address */
177 unsigned int rq_cons; /* RQ consumer counter */
178 unsigned int rq_mask; /* RQ depth - 1 */
179 unsigned int buf_offset; /* offset of next pkt in head buffer */
180 u8 xdp_flush; /* XDP flush types needed at NAPI end */
181 u8 init_state; /* queue initialization state */
182 u16 headroom; /* per packet headroom */
183 unsigned int rq_cons_db; /* value of rq_cons at last RQ db */
184 unsigned int rq_db_thres; /* # of new buffers needed to write RQ db */
185 struct funeth_rxbuf spare_buf; /* spare for next buffer replacement */
186 struct funeth_rx_cache cache; /* used buffer cache */
187 struct bpf_prog *xdp_prog; /* optional XDP BPF program */
188 struct funeth_rxq_stats stats;
189 dma_addr_t cq_dma_addr; /* DMA address of CQE ring */
190 dma_addr_t rq_dma_addr; /* DMA address of RQE ring */
191 u16 irq_cnt;
192 u32 hw_cqid; /* device ID of the queue's CQ */
193 u32 hw_sqid; /* device ID of the queue's SQ */
194 int numa_node;
195 struct u64_stats_sync syncp;
196 struct xdp_rxq_info xdp_rxq;
197 };
198
199 #define FUN_QSTAT_INC(q, counter) \
200 do { \
201 u64_stats_update_begin(&(q)->syncp); \
202 (q)->stats.counter++; \
203 u64_stats_update_end(&(q)->syncp); \
204 } while (0)
205
206 #define FUN_QSTAT_READ(q, seq, stats_copy) \
207 do { \
208 seq = u64_stats_fetch_begin_irq(&(q)->syncp); \
209 stats_copy = (q)->stats; \
210 } while (u64_stats_fetch_retry_irq(&(q)->syncp, (seq)))
211
212 #define FUN_INT_NAME_LEN (IFNAMSIZ + 16)
213
214 struct fun_irq {
215 struct napi_struct napi;
216 struct funeth_txq *txq;
217 struct funeth_rxq *rxq;
218 u8 state;
219 u16 irq_idx; /* index of MSI-X interrupt */
220 int irq; /* Linux IRQ vector */
221 cpumask_t affinity_mask; /* IRQ affinity */
222 struct irq_affinity_notify aff_notify;
223 char name[FUN_INT_NAME_LEN];
224 } ____cacheline_internodealigned_in_smp;
225
226 /* Return the start address of the idx-th Tx descriptor. */
fun_tx_desc_addr(const struct funeth_txq * q,unsigned int idx)227 static inline void *fun_tx_desc_addr(const struct funeth_txq *q,
228 unsigned int idx)
229 {
230 return q->desc + idx * FUNETH_SQE_SIZE;
231 }
232
fun_txq_wr_db(const struct funeth_txq * q)233 static inline void fun_txq_wr_db(const struct funeth_txq *q)
234 {
235 unsigned int tail = q->prod_cnt & q->mask;
236
237 writel(tail, q->db);
238 }
239
fun_irq_node(const struct fun_irq * p)240 static inline int fun_irq_node(const struct fun_irq *p)
241 {
242 return cpu_to_mem(cpumask_first(&p->affinity_mask));
243 }
244
245 int fun_rxq_napi_poll(struct napi_struct *napi, int budget);
246 int fun_txq_napi_poll(struct napi_struct *napi, int budget);
247 netdev_tx_t fun_start_xmit(struct sk_buff *skb, struct net_device *netdev);
248 bool fun_xdp_tx(struct funeth_txq *q, struct xdp_frame *xdpf);
249 int fun_xdp_xmit_frames(struct net_device *dev, int n,
250 struct xdp_frame **frames, u32 flags);
251
252 int funeth_txq_create(struct net_device *dev, unsigned int qidx,
253 unsigned int ndesc, struct fun_irq *irq, int state,
254 struct funeth_txq **qp);
255 int fun_txq_create_dev(struct funeth_txq *q, struct fun_irq *irq);
256 struct funeth_txq *funeth_txq_free(struct funeth_txq *q, int state);
257 int funeth_rxq_create(struct net_device *dev, unsigned int qidx,
258 unsigned int ncqe, unsigned int nrqe, struct fun_irq *irq,
259 int state, struct funeth_rxq **qp);
260 int fun_rxq_create_dev(struct funeth_rxq *q, struct fun_irq *irq);
261 struct funeth_rxq *funeth_rxq_free(struct funeth_rxq *q, int state);
262 int fun_rxq_set_bpf(struct funeth_rxq *q, struct bpf_prog *prog);
263
264 #endif /* _FUNETH_TXRX_H */
265