1 /**********************************************************************
2 * Author: Cavium, Inc.
3 *
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
6 *
7 * Copyright (c) 2003-2016 Cavium, Inc.
8 *
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
12 *
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more
17 * details.
18 **********************************************************************/
19
20 /*! \file octeon_network.h
21 * \brief Host NIC Driver: Structure and Macro definitions used by NIC Module.
22 */
23
24 #ifndef __OCTEON_NETWORK_H__
25 #define __OCTEON_NETWORK_H__
26 #include <linux/ptp_clock_kernel.h>
27
28 #define LIO_MAX_MTU_SIZE (OCTNET_MAX_FRM_SIZE - OCTNET_FRM_HEADER_SIZE)
29 #define LIO_MIN_MTU_SIZE ETH_MIN_MTU
30
31 /* Bit mask values for lio->ifstate */
32 #define LIO_IFSTATE_DROQ_OPS 0x01
33 #define LIO_IFSTATE_REGISTERED 0x02
34 #define LIO_IFSTATE_RUNNING 0x04
35 #define LIO_IFSTATE_RX_TIMESTAMP_ENABLED 0x08
36 #define LIO_IFSTATE_RESETTING 0x10
37
38 struct liquidio_if_cfg_resp {
39 u64 rh;
40 struct liquidio_if_cfg_info cfg_info;
41 u64 status;
42 };
43
44 #define LIO_IFCFG_WAIT_TIME 3000 /* In milli seconds */
45 #define LIQUIDIO_NDEV_STATS_POLL_TIME_MS 200
46
47 /* Structure of a node in list of gather components maintained by
48 * NIC driver for each network device.
49 */
50 struct octnic_gather {
51 /* List manipulation. Next and prev pointers. */
52 struct list_head list;
53
54 /* Size of the gather component at sg in bytes. */
55 int sg_size;
56
57 /* Number of bytes that sg was adjusted to make it 8B-aligned. */
58 int adjust;
59
60 /* Gather component that can accommodate max sized fragment list
61 * received from the IP layer.
62 */
63 struct octeon_sg_entry *sg;
64
65 dma_addr_t sg_dma_ptr;
66 };
67
68 struct oct_nic_stats_resp {
69 u64 rh;
70 struct oct_link_stats stats;
71 u64 status;
72 };
73
74 struct oct_nic_vf_stats_resp {
75 u64 rh;
76 u64 spoofmac_cnt;
77 u64 status;
78 };
79
80 struct oct_nic_stats_ctrl {
81 struct completion complete;
82 struct net_device *netdev;
83 };
84
85 struct oct_nic_seapi_resp {
86 u64 rh;
87 union {
88 u32 fec_setting;
89 u32 speed;
90 };
91 u64 status;
92 };
93
94 /** LiquidIO per-interface network private data */
95 struct lio {
96 /** State of the interface. Rx/Tx happens only in the RUNNING state. */
97 atomic_t ifstate;
98
99 /** Octeon Interface index number. This device will be represented as
100 * oct<ifidx> in the system.
101 */
102 int ifidx;
103
104 /** Octeon Input queue to use to transmit for this network interface. */
105 int txq;
106
107 /** Octeon Output queue from which pkts arrive
108 * for this network interface.
109 */
110 int rxq;
111
112 /** Guards each glist */
113 spinlock_t *glist_lock;
114
115 /** Array of gather component linked lists */
116 struct list_head *glist;
117 void **glists_virt_base;
118 dma_addr_t *glists_dma_base;
119 u32 glist_entry_size;
120
121 /** Pointer to the NIC properties for the Octeon device this network
122 * interface is associated with.
123 */
124 struct octdev_props *octprops;
125
126 /** Pointer to the octeon device structure. */
127 struct octeon_device *oct_dev;
128
129 struct net_device *netdev;
130
131 /** Link information sent by the core application for this interface. */
132 struct oct_link_info linfo;
133
134 /** counter of link changes */
135 u64 link_changes;
136
137 /** Size of Tx queue for this octeon device. */
138 u32 tx_qsize;
139
140 /** Size of Rx queue for this octeon device. */
141 u32 rx_qsize;
142
143 /** Size of MTU this octeon device. */
144 u32 mtu;
145
146 /** msg level flag per interface. */
147 u32 msg_enable;
148
149 /** Copy of Interface capabilities: TSO, TSO6, LRO, Chescksums . */
150 u64 dev_capability;
151
152 /* Copy of transmit encapsulation capabilities:
153 * TSO, TSO6, Checksums for this device for Kernel
154 * 3.10.0 onwards
155 */
156 u64 enc_dev_capability;
157
158 /** Copy of beacaon reg in phy */
159 u32 phy_beacon_val;
160
161 /** Copy of ctrl reg in phy */
162 u32 led_ctrl_val;
163
164 /* PTP clock information */
165 struct ptp_clock_info ptp_info;
166 struct ptp_clock *ptp_clock;
167 s64 ptp_adjust;
168
169 /* for atomic access to Octeon PTP reg and data struct */
170 spinlock_t ptp_lock;
171
172 /* Interface info */
173 u32 intf_open;
174
175 /* work queue for txq status */
176 struct cavium_wq txq_status_wq;
177
178 /* work queue for rxq oom status */
179 struct cavium_wq rxq_status_wq[MAX_POSSIBLE_OCTEON_OUTPUT_QUEUES];
180
181 /* work queue for link status */
182 struct cavium_wq link_status_wq;
183
184 /* work queue to regularly send local time to octeon firmware */
185 struct cavium_wq sync_octeon_time_wq;
186
187 int netdev_uc_count;
188 struct cavium_wk stats_wk;
189 };
190
191 #define LIO_SIZE (sizeof(struct lio))
192 #define GET_LIO(netdev) ((struct lio *)netdev_priv(netdev))
193
194 #define LIO_MAX_CORES 16
195
196 /**
197 * \brief Enable or disable feature
198 * @param netdev pointer to network device
199 * @param cmd Command that just requires acknowledgment
200 * @param param1 Parameter to command
201 */
202 int liquidio_set_feature(struct net_device *netdev, int cmd, u16 param1);
203
204 int setup_rx_oom_poll_fn(struct net_device *netdev);
205
206 void cleanup_rx_oom_poll_fn(struct net_device *netdev);
207
208 /**
209 * \brief Link control command completion callback
210 * @param nctrl_ptr pointer to control packet structure
211 *
212 * This routine is called by the callback function when a ctrl pkt sent to
213 * core app completes. The nctrl_ptr contains a copy of the command type
214 * and data sent to the core app. This routine is only called if the ctrl
215 * pkt was sent successfully to the core app.
216 */
217 void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr);
218
219 int liquidio_setup_io_queues(struct octeon_device *octeon_dev, int ifidx,
220 u32 num_iqs, u32 num_oqs);
221
222 irqreturn_t liquidio_msix_intr_handler(int irq __attribute__((unused)),
223 void *dev);
224
225 int octeon_setup_interrupt(struct octeon_device *oct, u32 num_ioqs);
226
227 void lio_fetch_stats(struct work_struct *work);
228
229 int lio_wait_for_clean_oq(struct octeon_device *oct);
230 /**
231 * \brief Register ethtool operations
232 * @param netdev pointer to network device
233 */
234 void liquidio_set_ethtool_ops(struct net_device *netdev);
235
236 void lio_delete_glists(struct lio *lio);
237
238 int lio_setup_glists(struct octeon_device *oct, struct lio *lio, int num_qs);
239
240 int liquidio_get_speed(struct lio *lio);
241 int liquidio_set_speed(struct lio *lio, int speed);
242 int liquidio_get_fec(struct lio *lio);
243 int liquidio_set_fec(struct lio *lio, int on_off);
244
245 /**
246 * \brief Net device change_mtu
247 * @param netdev network device
248 */
249 int liquidio_change_mtu(struct net_device *netdev, int new_mtu);
250 #define LIO_CHANGE_MTU_SUCCESS 1
251 #define LIO_CHANGE_MTU_FAIL 2
252
253 #define SKB_ADJ_MASK 0x3F
254 #define SKB_ADJ (SKB_ADJ_MASK + 1)
255
256 #define MIN_SKB_SIZE 256 /* 8 bytes and more - 8 bytes for PTP */
257 #define LIO_RXBUFFER_SZ 2048
258
259 static inline void
recv_buffer_alloc(struct octeon_device * oct,struct octeon_skb_page_info * pg_info)260 *recv_buffer_alloc(struct octeon_device *oct,
261 struct octeon_skb_page_info *pg_info)
262 {
263 struct page *page;
264 struct sk_buff *skb;
265 struct octeon_skb_page_info *skb_pg_info;
266
267 page = alloc_page(GFP_ATOMIC);
268 if (unlikely(!page))
269 return NULL;
270
271 skb = dev_alloc_skb(MIN_SKB_SIZE + SKB_ADJ);
272 if (unlikely(!skb)) {
273 __free_page(page);
274 pg_info->page = NULL;
275 return NULL;
276 }
277
278 if ((unsigned long)skb->data & SKB_ADJ_MASK) {
279 u32 r = SKB_ADJ - ((unsigned long)skb->data & SKB_ADJ_MASK);
280
281 skb_reserve(skb, r);
282 }
283
284 skb_pg_info = ((struct octeon_skb_page_info *)(skb->cb));
285 /* Get DMA info */
286 pg_info->dma = dma_map_page(&oct->pci_dev->dev, page, 0,
287 PAGE_SIZE, DMA_FROM_DEVICE);
288
289 /* Mapping failed!! */
290 if (dma_mapping_error(&oct->pci_dev->dev, pg_info->dma)) {
291 __free_page(page);
292 dev_kfree_skb_any((struct sk_buff *)skb);
293 pg_info->page = NULL;
294 return NULL;
295 }
296
297 pg_info->page = page;
298 pg_info->page_offset = 0;
299 skb_pg_info->page = page;
300 skb_pg_info->page_offset = 0;
301 skb_pg_info->dma = pg_info->dma;
302
303 return (void *)skb;
304 }
305
306 static inline void
recv_buffer_fast_alloc(u32 size)307 *recv_buffer_fast_alloc(u32 size)
308 {
309 struct sk_buff *skb;
310 struct octeon_skb_page_info *skb_pg_info;
311
312 skb = dev_alloc_skb(size + SKB_ADJ);
313 if (unlikely(!skb))
314 return NULL;
315
316 if ((unsigned long)skb->data & SKB_ADJ_MASK) {
317 u32 r = SKB_ADJ - ((unsigned long)skb->data & SKB_ADJ_MASK);
318
319 skb_reserve(skb, r);
320 }
321
322 skb_pg_info = ((struct octeon_skb_page_info *)(skb->cb));
323 skb_pg_info->page = NULL;
324 skb_pg_info->page_offset = 0;
325 skb_pg_info->dma = 0;
326
327 return skb;
328 }
329
330 static inline int
recv_buffer_recycle(struct octeon_device * oct,void * buf)331 recv_buffer_recycle(struct octeon_device *oct, void *buf)
332 {
333 struct octeon_skb_page_info *pg_info = buf;
334
335 if (!pg_info->page) {
336 dev_err(&oct->pci_dev->dev, "%s: pg_info->page NULL\n",
337 __func__);
338 return -ENOMEM;
339 }
340
341 if (unlikely(page_count(pg_info->page) != 1) ||
342 unlikely(page_to_nid(pg_info->page) != numa_node_id())) {
343 dma_unmap_page(&oct->pci_dev->dev,
344 pg_info->dma, (PAGE_SIZE << 0),
345 DMA_FROM_DEVICE);
346 pg_info->dma = 0;
347 pg_info->page = NULL;
348 pg_info->page_offset = 0;
349 return -ENOMEM;
350 }
351
352 /* Flip to other half of the buffer */
353 if (pg_info->page_offset == 0)
354 pg_info->page_offset = LIO_RXBUFFER_SZ;
355 else
356 pg_info->page_offset = 0;
357 page_ref_inc(pg_info->page);
358
359 return 0;
360 }
361
362 static inline void
recv_buffer_reuse(struct octeon_device * oct,void * buf)363 *recv_buffer_reuse(struct octeon_device *oct, void *buf)
364 {
365 struct octeon_skb_page_info *pg_info = buf, *skb_pg_info;
366 struct sk_buff *skb;
367
368 skb = dev_alloc_skb(MIN_SKB_SIZE + SKB_ADJ);
369 if (unlikely(!skb)) {
370 dma_unmap_page(&oct->pci_dev->dev,
371 pg_info->dma, (PAGE_SIZE << 0),
372 DMA_FROM_DEVICE);
373 return NULL;
374 }
375
376 if ((unsigned long)skb->data & SKB_ADJ_MASK) {
377 u32 r = SKB_ADJ - ((unsigned long)skb->data & SKB_ADJ_MASK);
378
379 skb_reserve(skb, r);
380 }
381
382 skb_pg_info = ((struct octeon_skb_page_info *)(skb->cb));
383 skb_pg_info->page = pg_info->page;
384 skb_pg_info->page_offset = pg_info->page_offset;
385 skb_pg_info->dma = pg_info->dma;
386
387 return skb;
388 }
389
390 static inline void
recv_buffer_destroy(void * buffer,struct octeon_skb_page_info * pg_info)391 recv_buffer_destroy(void *buffer, struct octeon_skb_page_info *pg_info)
392 {
393 struct sk_buff *skb = (struct sk_buff *)buffer;
394
395 put_page(pg_info->page);
396 pg_info->dma = 0;
397 pg_info->page = NULL;
398 pg_info->page_offset = 0;
399
400 if (skb)
401 dev_kfree_skb_any(skb);
402 }
403
recv_buffer_free(void * buffer)404 static inline void recv_buffer_free(void *buffer)
405 {
406 struct sk_buff *skb = (struct sk_buff *)buffer;
407 struct octeon_skb_page_info *pg_info;
408
409 pg_info = ((struct octeon_skb_page_info *)(skb->cb));
410
411 if (pg_info->page) {
412 put_page(pg_info->page);
413 pg_info->dma = 0;
414 pg_info->page = NULL;
415 pg_info->page_offset = 0;
416 }
417
418 dev_kfree_skb_any((struct sk_buff *)buffer);
419 }
420
421 static inline void
recv_buffer_fast_free(void * buffer)422 recv_buffer_fast_free(void *buffer)
423 {
424 dev_kfree_skb_any((struct sk_buff *)buffer);
425 }
426
tx_buffer_free(void * buffer)427 static inline void tx_buffer_free(void *buffer)
428 {
429 dev_kfree_skb_any((struct sk_buff *)buffer);
430 }
431
432 #define lio_dma_alloc(oct, size, dma_addr) \
433 dma_alloc_coherent(&(oct)->pci_dev->dev, size, dma_addr, GFP_KERNEL)
434 #define lio_dma_free(oct, size, virt_addr, dma_addr) \
435 dma_free_coherent(&(oct)->pci_dev->dev, size, virt_addr, dma_addr)
436
437 static inline
get_rbd(struct sk_buff * skb)438 void *get_rbd(struct sk_buff *skb)
439 {
440 struct octeon_skb_page_info *pg_info;
441 unsigned char *va;
442
443 pg_info = ((struct octeon_skb_page_info *)(skb->cb));
444 va = page_address(pg_info->page) + pg_info->page_offset;
445
446 return va;
447 }
448
449 static inline u64
lio_map_ring(void * buf)450 lio_map_ring(void *buf)
451 {
452 dma_addr_t dma_addr;
453
454 struct sk_buff *skb = (struct sk_buff *)buf;
455 struct octeon_skb_page_info *pg_info;
456
457 pg_info = ((struct octeon_skb_page_info *)(skb->cb));
458 if (!pg_info->page) {
459 pr_err("%s: pg_info->page NULL\n", __func__);
460 WARN_ON(1);
461 }
462
463 /* Get DMA info */
464 dma_addr = pg_info->dma;
465 if (!pg_info->dma) {
466 pr_err("%s: ERROR it should be already available\n",
467 __func__);
468 WARN_ON(1);
469 }
470 dma_addr += pg_info->page_offset;
471
472 return (u64)dma_addr;
473 }
474
475 static inline void
lio_unmap_ring(struct pci_dev * pci_dev,u64 buf_ptr)476 lio_unmap_ring(struct pci_dev *pci_dev,
477 u64 buf_ptr)
478
479 {
480 dma_unmap_page(&pci_dev->dev,
481 buf_ptr, (PAGE_SIZE << 0),
482 DMA_FROM_DEVICE);
483 }
484
octeon_fast_packet_alloc(u32 size)485 static inline void *octeon_fast_packet_alloc(u32 size)
486 {
487 return recv_buffer_fast_alloc(size);
488 }
489
octeon_fast_packet_next(struct octeon_droq * droq,struct sk_buff * nicbuf,int copy_len,int idx)490 static inline void octeon_fast_packet_next(struct octeon_droq *droq,
491 struct sk_buff *nicbuf,
492 int copy_len,
493 int idx)
494 {
495 skb_put_data(nicbuf, get_rbd(droq->recv_buf_list[idx].buffer),
496 copy_len);
497 }
498
499 /**
500 * \brief check interface state
501 * @param lio per-network private data
502 * @param state_flag flag state to check
503 */
ifstate_check(struct lio * lio,int state_flag)504 static inline int ifstate_check(struct lio *lio, int state_flag)
505 {
506 return atomic_read(&lio->ifstate) & state_flag;
507 }
508
509 /**
510 * \brief set interface state
511 * @param lio per-network private data
512 * @param state_flag flag state to set
513 */
ifstate_set(struct lio * lio,int state_flag)514 static inline void ifstate_set(struct lio *lio, int state_flag)
515 {
516 atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) | state_flag));
517 }
518
519 /**
520 * \brief clear interface state
521 * @param lio per-network private data
522 * @param state_flag flag state to clear
523 */
ifstate_reset(struct lio * lio,int state_flag)524 static inline void ifstate_reset(struct lio *lio, int state_flag)
525 {
526 atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) & ~(state_flag)));
527 }
528
529 /**
530 * \brief wait for all pending requests to complete
531 * @param oct Pointer to Octeon device
532 *
533 * Called during shutdown sequence
534 */
wait_for_pending_requests(struct octeon_device * oct)535 static inline int wait_for_pending_requests(struct octeon_device *oct)
536 {
537 int i, pcount = 0;
538
539 for (i = 0; i < MAX_IO_PENDING_PKT_COUNT; i++) {
540 pcount = atomic_read(
541 &oct->response_list[OCTEON_ORDERED_SC_LIST]
542 .pending_req_count);
543 if (pcount)
544 schedule_timeout_uninterruptible(HZ / 10);
545 else
546 break;
547 }
548
549 if (pcount)
550 return 1;
551
552 return 0;
553 }
554
555 /**
556 * \brief Stop Tx queues
557 * @param netdev network device
558 */
stop_txqs(struct net_device * netdev)559 static inline void stop_txqs(struct net_device *netdev)
560 {
561 int i;
562
563 for (i = 0; i < netdev->real_num_tx_queues; i++)
564 netif_stop_subqueue(netdev, i);
565 }
566
567 /**
568 * \brief Wake Tx queues
569 * @param netdev network device
570 */
wake_txqs(struct net_device * netdev)571 static inline void wake_txqs(struct net_device *netdev)
572 {
573 struct lio *lio = GET_LIO(netdev);
574 int i, qno;
575
576 for (i = 0; i < netdev->real_num_tx_queues; i++) {
577 qno = lio->linfo.txpciq[i % lio->oct_dev->num_iqs].s.q_no;
578
579 if (__netif_subqueue_stopped(netdev, i)) {
580 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, qno,
581 tx_restart, 1);
582 netif_wake_subqueue(netdev, i);
583 }
584 }
585 }
586
587 /**
588 * \brief Start Tx queues
589 * @param netdev network device
590 */
start_txqs(struct net_device * netdev)591 static inline void start_txqs(struct net_device *netdev)
592 {
593 struct lio *lio = GET_LIO(netdev);
594 int i;
595
596 if (lio->linfo.link.s.link_up) {
597 for (i = 0; i < netdev->real_num_tx_queues; i++)
598 netif_start_subqueue(netdev, i);
599 }
600 }
601
skb_iq(struct octeon_device * oct,struct sk_buff * skb)602 static inline int skb_iq(struct octeon_device *oct, struct sk_buff *skb)
603 {
604 return skb->queue_mapping % oct->num_iqs;
605 }
606
607 /**
608 * Remove the node at the head of the list. The list would be empty at
609 * the end of this call if there are no more nodes in the list.
610 */
lio_list_delete_head(struct list_head * root)611 static inline struct list_head *lio_list_delete_head(struct list_head *root)
612 {
613 struct list_head *node;
614
615 if (list_empty_careful(root))
616 node = NULL;
617 else
618 node = root->next;
619
620 if (node)
621 list_del(node);
622
623 return node;
624 }
625
626 #endif
627