1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c)  2018 Intel Corporation */
3 
4 #include <linux/module.h>
5 #include <linux/types.h>
6 #include <linux/if_vlan.h>
7 #include <linux/aer.h>
8 #include <linux/tcp.h>
9 #include <linux/udp.h>
10 #include <linux/ip.h>
11 #include <linux/pm_runtime.h>
12 #include <net/pkt_sched.h>
13 #include <linux/bpf_trace.h>
14 #include <net/xdp_sock_drv.h>
15 #include <linux/pci.h>
16 
17 #include <net/ipv6.h>
18 
19 #include "igc.h"
20 #include "igc_hw.h"
21 #include "igc_tsn.h"
22 #include "igc_xdp.h"
23 
24 #define DRV_SUMMARY	"Intel(R) 2.5G Ethernet Linux Driver"
25 
26 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
27 
28 #define IGC_XDP_PASS		0
29 #define IGC_XDP_CONSUMED	BIT(0)
30 #define IGC_XDP_TX		BIT(1)
31 #define IGC_XDP_REDIRECT	BIT(2)
32 
33 static int debug = -1;
34 
35 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
36 MODULE_DESCRIPTION(DRV_SUMMARY);
37 MODULE_LICENSE("GPL v2");
38 module_param(debug, int, 0);
39 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
40 
41 char igc_driver_name[] = "igc";
42 static const char igc_driver_string[] = DRV_SUMMARY;
43 static const char igc_copyright[] =
44 	"Copyright(c) 2018 Intel Corporation.";
45 
46 static const struct igc_info *igc_info_tbl[] = {
47 	[board_base] = &igc_base_info,
48 };
49 
50 static const struct pci_device_id igc_pci_tbl[] = {
51 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base },
52 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base },
53 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base },
54 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base },
55 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base },
56 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base },
57 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_K), board_base },
58 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base },
59 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LMVP), board_base },
60 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base },
61 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), board_base },
62 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), board_base },
63 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), board_base },
64 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), board_base },
65 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), board_base },
66 	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base },
67 	/* required last entry */
68 	{0, }
69 };
70 
71 MODULE_DEVICE_TABLE(pci, igc_pci_tbl);
72 
73 enum latency_range {
74 	lowest_latency = 0,
75 	low_latency = 1,
76 	bulk_latency = 2,
77 	latency_invalid = 255
78 };
79 
igc_reset(struct igc_adapter * adapter)80 void igc_reset(struct igc_adapter *adapter)
81 {
82 	struct net_device *dev = adapter->netdev;
83 	struct igc_hw *hw = &adapter->hw;
84 	struct igc_fc_info *fc = &hw->fc;
85 	u32 pba, hwm;
86 
87 	/* Repartition PBA for greater than 9k MTU if required */
88 	pba = IGC_PBA_34K;
89 
90 	/* flow control settings
91 	 * The high water mark must be low enough to fit one full frame
92 	 * after transmitting the pause frame.  As such we must have enough
93 	 * space to allow for us to complete our current transmit and then
94 	 * receive the frame that is in progress from the link partner.
95 	 * Set it to:
96 	 * - the full Rx FIFO size minus one full Tx plus one full Rx frame
97 	 */
98 	hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
99 
100 	fc->high_water = hwm & 0xFFFFFFF0;	/* 16-byte granularity */
101 	fc->low_water = fc->high_water - 16;
102 	fc->pause_time = 0xFFFF;
103 	fc->send_xon = 1;
104 	fc->current_mode = fc->requested_mode;
105 
106 	hw->mac.ops.reset_hw(hw);
107 
108 	if (hw->mac.ops.init_hw(hw))
109 		netdev_err(dev, "Error on hardware initialization\n");
110 
111 	/* Re-establish EEE setting */
112 	igc_set_eee_i225(hw, true, true, true);
113 
114 	if (!netif_running(adapter->netdev))
115 		igc_power_down_phy_copper_base(&adapter->hw);
116 
117 	/* Enable HW to recognize an 802.1Q VLAN Ethernet packet */
118 	wr32(IGC_VET, ETH_P_8021Q);
119 
120 	/* Re-enable PTP, where applicable. */
121 	igc_ptp_reset(adapter);
122 
123 	/* Re-enable TSN offloading, where applicable. */
124 	igc_tsn_reset(adapter);
125 
126 	igc_get_phy_info(hw);
127 }
128 
129 /**
130  * igc_power_up_link - Power up the phy link
131  * @adapter: address of board private structure
132  */
igc_power_up_link(struct igc_adapter * adapter)133 static void igc_power_up_link(struct igc_adapter *adapter)
134 {
135 	igc_reset_phy(&adapter->hw);
136 
137 	igc_power_up_phy_copper(&adapter->hw);
138 
139 	igc_setup_link(&adapter->hw);
140 }
141 
142 /**
143  * igc_release_hw_control - release control of the h/w to f/w
144  * @adapter: address of board private structure
145  *
146  * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
147  * For ASF and Pass Through versions of f/w this means that the
148  * driver is no longer loaded.
149  */
igc_release_hw_control(struct igc_adapter * adapter)150 static void igc_release_hw_control(struct igc_adapter *adapter)
151 {
152 	struct igc_hw *hw = &adapter->hw;
153 	u32 ctrl_ext;
154 
155 	if (!pci_device_is_present(adapter->pdev))
156 		return;
157 
158 	/* Let firmware take over control of h/w */
159 	ctrl_ext = rd32(IGC_CTRL_EXT);
160 	wr32(IGC_CTRL_EXT,
161 	     ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
162 }
163 
164 /**
165  * igc_get_hw_control - get control of the h/w from f/w
166  * @adapter: address of board private structure
167  *
168  * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
169  * For ASF and Pass Through versions of f/w this means that
170  * the driver is loaded.
171  */
igc_get_hw_control(struct igc_adapter * adapter)172 static void igc_get_hw_control(struct igc_adapter *adapter)
173 {
174 	struct igc_hw *hw = &adapter->hw;
175 	u32 ctrl_ext;
176 
177 	/* Let firmware know the driver has taken over */
178 	ctrl_ext = rd32(IGC_CTRL_EXT);
179 	wr32(IGC_CTRL_EXT,
180 	     ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
181 }
182 
igc_unmap_tx_buffer(struct device * dev,struct igc_tx_buffer * buf)183 static void igc_unmap_tx_buffer(struct device *dev, struct igc_tx_buffer *buf)
184 {
185 	dma_unmap_single(dev, dma_unmap_addr(buf, dma),
186 			 dma_unmap_len(buf, len), DMA_TO_DEVICE);
187 
188 	dma_unmap_len_set(buf, len, 0);
189 }
190 
191 /**
192  * igc_clean_tx_ring - Free Tx Buffers
193  * @tx_ring: ring to be cleaned
194  */
igc_clean_tx_ring(struct igc_ring * tx_ring)195 static void igc_clean_tx_ring(struct igc_ring *tx_ring)
196 {
197 	u16 i = tx_ring->next_to_clean;
198 	struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
199 	u32 xsk_frames = 0;
200 
201 	while (i != tx_ring->next_to_use) {
202 		union igc_adv_tx_desc *eop_desc, *tx_desc;
203 
204 		switch (tx_buffer->type) {
205 		case IGC_TX_BUFFER_TYPE_XSK:
206 			xsk_frames++;
207 			break;
208 		case IGC_TX_BUFFER_TYPE_XDP:
209 			xdp_return_frame(tx_buffer->xdpf);
210 			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
211 			break;
212 		case IGC_TX_BUFFER_TYPE_SKB:
213 			dev_kfree_skb_any(tx_buffer->skb);
214 			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
215 			break;
216 		default:
217 			netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n");
218 			break;
219 		}
220 
221 		/* check for eop_desc to determine the end of the packet */
222 		eop_desc = tx_buffer->next_to_watch;
223 		tx_desc = IGC_TX_DESC(tx_ring, i);
224 
225 		/* unmap remaining buffers */
226 		while (tx_desc != eop_desc) {
227 			tx_buffer++;
228 			tx_desc++;
229 			i++;
230 			if (unlikely(i == tx_ring->count)) {
231 				i = 0;
232 				tx_buffer = tx_ring->tx_buffer_info;
233 				tx_desc = IGC_TX_DESC(tx_ring, 0);
234 			}
235 
236 			/* unmap any remaining paged data */
237 			if (dma_unmap_len(tx_buffer, len))
238 				igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
239 		}
240 
241 		tx_buffer->next_to_watch = NULL;
242 
243 		/* move us one more past the eop_desc for start of next pkt */
244 		tx_buffer++;
245 		i++;
246 		if (unlikely(i == tx_ring->count)) {
247 			i = 0;
248 			tx_buffer = tx_ring->tx_buffer_info;
249 		}
250 	}
251 
252 	if (tx_ring->xsk_pool && xsk_frames)
253 		xsk_tx_completed(tx_ring->xsk_pool, xsk_frames);
254 
255 	/* reset BQL for queue */
256 	netdev_tx_reset_queue(txring_txq(tx_ring));
257 
258 	/* reset next_to_use and next_to_clean */
259 	tx_ring->next_to_use = 0;
260 	tx_ring->next_to_clean = 0;
261 }
262 
263 /**
264  * igc_free_tx_resources - Free Tx Resources per Queue
265  * @tx_ring: Tx descriptor ring for a specific queue
266  *
267  * Free all transmit software resources
268  */
igc_free_tx_resources(struct igc_ring * tx_ring)269 void igc_free_tx_resources(struct igc_ring *tx_ring)
270 {
271 	igc_clean_tx_ring(tx_ring);
272 
273 	vfree(tx_ring->tx_buffer_info);
274 	tx_ring->tx_buffer_info = NULL;
275 
276 	/* if not set, then don't free */
277 	if (!tx_ring->desc)
278 		return;
279 
280 	dma_free_coherent(tx_ring->dev, tx_ring->size,
281 			  tx_ring->desc, tx_ring->dma);
282 
283 	tx_ring->desc = NULL;
284 }
285 
286 /**
287  * igc_free_all_tx_resources - Free Tx Resources for All Queues
288  * @adapter: board private structure
289  *
290  * Free all transmit software resources
291  */
igc_free_all_tx_resources(struct igc_adapter * adapter)292 static void igc_free_all_tx_resources(struct igc_adapter *adapter)
293 {
294 	int i;
295 
296 	for (i = 0; i < adapter->num_tx_queues; i++)
297 		igc_free_tx_resources(adapter->tx_ring[i]);
298 }
299 
300 /**
301  * igc_clean_all_tx_rings - Free Tx Buffers for all queues
302  * @adapter: board private structure
303  */
igc_clean_all_tx_rings(struct igc_adapter * adapter)304 static void igc_clean_all_tx_rings(struct igc_adapter *adapter)
305 {
306 	int i;
307 
308 	for (i = 0; i < adapter->num_tx_queues; i++)
309 		if (adapter->tx_ring[i])
310 			igc_clean_tx_ring(adapter->tx_ring[i]);
311 }
312 
313 /**
314  * igc_setup_tx_resources - allocate Tx resources (Descriptors)
315  * @tx_ring: tx descriptor ring (for a specific queue) to setup
316  *
317  * Return 0 on success, negative on failure
318  */
igc_setup_tx_resources(struct igc_ring * tx_ring)319 int igc_setup_tx_resources(struct igc_ring *tx_ring)
320 {
321 	struct net_device *ndev = tx_ring->netdev;
322 	struct device *dev = tx_ring->dev;
323 	int size = 0;
324 
325 	size = sizeof(struct igc_tx_buffer) * tx_ring->count;
326 	tx_ring->tx_buffer_info = vzalloc(size);
327 	if (!tx_ring->tx_buffer_info)
328 		goto err;
329 
330 	/* round up to nearest 4K */
331 	tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc);
332 	tx_ring->size = ALIGN(tx_ring->size, 4096);
333 
334 	tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
335 					   &tx_ring->dma, GFP_KERNEL);
336 
337 	if (!tx_ring->desc)
338 		goto err;
339 
340 	tx_ring->next_to_use = 0;
341 	tx_ring->next_to_clean = 0;
342 
343 	return 0;
344 
345 err:
346 	vfree(tx_ring->tx_buffer_info);
347 	netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n");
348 	return -ENOMEM;
349 }
350 
351 /**
352  * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues
353  * @adapter: board private structure
354  *
355  * Return 0 on success, negative on failure
356  */
igc_setup_all_tx_resources(struct igc_adapter * adapter)357 static int igc_setup_all_tx_resources(struct igc_adapter *adapter)
358 {
359 	struct net_device *dev = adapter->netdev;
360 	int i, err = 0;
361 
362 	for (i = 0; i < adapter->num_tx_queues; i++) {
363 		err = igc_setup_tx_resources(adapter->tx_ring[i]);
364 		if (err) {
365 			netdev_err(dev, "Error on Tx queue %u setup\n", i);
366 			for (i--; i >= 0; i--)
367 				igc_free_tx_resources(adapter->tx_ring[i]);
368 			break;
369 		}
370 	}
371 
372 	return err;
373 }
374 
igc_clean_rx_ring_page_shared(struct igc_ring * rx_ring)375 static void igc_clean_rx_ring_page_shared(struct igc_ring *rx_ring)
376 {
377 	u16 i = rx_ring->next_to_clean;
378 
379 	dev_kfree_skb(rx_ring->skb);
380 	rx_ring->skb = NULL;
381 
382 	/* Free all the Rx ring sk_buffs */
383 	while (i != rx_ring->next_to_alloc) {
384 		struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
385 
386 		/* Invalidate cache lines that may have been written to by
387 		 * device so that we avoid corrupting memory.
388 		 */
389 		dma_sync_single_range_for_cpu(rx_ring->dev,
390 					      buffer_info->dma,
391 					      buffer_info->page_offset,
392 					      igc_rx_bufsz(rx_ring),
393 					      DMA_FROM_DEVICE);
394 
395 		/* free resources associated with mapping */
396 		dma_unmap_page_attrs(rx_ring->dev,
397 				     buffer_info->dma,
398 				     igc_rx_pg_size(rx_ring),
399 				     DMA_FROM_DEVICE,
400 				     IGC_RX_DMA_ATTR);
401 		__page_frag_cache_drain(buffer_info->page,
402 					buffer_info->pagecnt_bias);
403 
404 		i++;
405 		if (i == rx_ring->count)
406 			i = 0;
407 	}
408 }
409 
igc_clean_rx_ring_xsk_pool(struct igc_ring * ring)410 static void igc_clean_rx_ring_xsk_pool(struct igc_ring *ring)
411 {
412 	struct igc_rx_buffer *bi;
413 	u16 i;
414 
415 	for (i = 0; i < ring->count; i++) {
416 		bi = &ring->rx_buffer_info[i];
417 		if (!bi->xdp)
418 			continue;
419 
420 		xsk_buff_free(bi->xdp);
421 		bi->xdp = NULL;
422 	}
423 }
424 
425 /**
426  * igc_clean_rx_ring - Free Rx Buffers per Queue
427  * @ring: ring to free buffers from
428  */
igc_clean_rx_ring(struct igc_ring * ring)429 static void igc_clean_rx_ring(struct igc_ring *ring)
430 {
431 	if (ring->xsk_pool)
432 		igc_clean_rx_ring_xsk_pool(ring);
433 	else
434 		igc_clean_rx_ring_page_shared(ring);
435 
436 	clear_ring_uses_large_buffer(ring);
437 
438 	ring->next_to_alloc = 0;
439 	ring->next_to_clean = 0;
440 	ring->next_to_use = 0;
441 }
442 
443 /**
444  * igc_clean_all_rx_rings - Free Rx Buffers for all queues
445  * @adapter: board private structure
446  */
igc_clean_all_rx_rings(struct igc_adapter * adapter)447 static void igc_clean_all_rx_rings(struct igc_adapter *adapter)
448 {
449 	int i;
450 
451 	for (i = 0; i < adapter->num_rx_queues; i++)
452 		if (adapter->rx_ring[i])
453 			igc_clean_rx_ring(adapter->rx_ring[i]);
454 }
455 
456 /**
457  * igc_free_rx_resources - Free Rx Resources
458  * @rx_ring: ring to clean the resources from
459  *
460  * Free all receive software resources
461  */
igc_free_rx_resources(struct igc_ring * rx_ring)462 void igc_free_rx_resources(struct igc_ring *rx_ring)
463 {
464 	igc_clean_rx_ring(rx_ring);
465 
466 	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
467 
468 	vfree(rx_ring->rx_buffer_info);
469 	rx_ring->rx_buffer_info = NULL;
470 
471 	/* if not set, then don't free */
472 	if (!rx_ring->desc)
473 		return;
474 
475 	dma_free_coherent(rx_ring->dev, rx_ring->size,
476 			  rx_ring->desc, rx_ring->dma);
477 
478 	rx_ring->desc = NULL;
479 }
480 
481 /**
482  * igc_free_all_rx_resources - Free Rx Resources for All Queues
483  * @adapter: board private structure
484  *
485  * Free all receive software resources
486  */
igc_free_all_rx_resources(struct igc_adapter * adapter)487 static void igc_free_all_rx_resources(struct igc_adapter *adapter)
488 {
489 	int i;
490 
491 	for (i = 0; i < adapter->num_rx_queues; i++)
492 		igc_free_rx_resources(adapter->rx_ring[i]);
493 }
494 
495 /**
496  * igc_setup_rx_resources - allocate Rx resources (Descriptors)
497  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
498  *
499  * Returns 0 on success, negative on failure
500  */
igc_setup_rx_resources(struct igc_ring * rx_ring)501 int igc_setup_rx_resources(struct igc_ring *rx_ring)
502 {
503 	struct net_device *ndev = rx_ring->netdev;
504 	struct device *dev = rx_ring->dev;
505 	u8 index = rx_ring->queue_index;
506 	int size, desc_len, res;
507 
508 	/* XDP RX-queue info */
509 	if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))
510 		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
511 	res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, ndev, index,
512 			       rx_ring->q_vector->napi.napi_id);
513 	if (res < 0) {
514 		netdev_err(ndev, "Failed to register xdp_rxq index %u\n",
515 			   index);
516 		return res;
517 	}
518 
519 	size = sizeof(struct igc_rx_buffer) * rx_ring->count;
520 	rx_ring->rx_buffer_info = vzalloc(size);
521 	if (!rx_ring->rx_buffer_info)
522 		goto err;
523 
524 	desc_len = sizeof(union igc_adv_rx_desc);
525 
526 	/* Round up to nearest 4K */
527 	rx_ring->size = rx_ring->count * desc_len;
528 	rx_ring->size = ALIGN(rx_ring->size, 4096);
529 
530 	rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
531 					   &rx_ring->dma, GFP_KERNEL);
532 
533 	if (!rx_ring->desc)
534 		goto err;
535 
536 	rx_ring->next_to_alloc = 0;
537 	rx_ring->next_to_clean = 0;
538 	rx_ring->next_to_use = 0;
539 
540 	return 0;
541 
542 err:
543 	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
544 	vfree(rx_ring->rx_buffer_info);
545 	rx_ring->rx_buffer_info = NULL;
546 	netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n");
547 	return -ENOMEM;
548 }
549 
550 /**
551  * igc_setup_all_rx_resources - wrapper to allocate Rx resources
552  *                                (Descriptors) for all queues
553  * @adapter: board private structure
554  *
555  * Return 0 on success, negative on failure
556  */
igc_setup_all_rx_resources(struct igc_adapter * adapter)557 static int igc_setup_all_rx_resources(struct igc_adapter *adapter)
558 {
559 	struct net_device *dev = adapter->netdev;
560 	int i, err = 0;
561 
562 	for (i = 0; i < adapter->num_rx_queues; i++) {
563 		err = igc_setup_rx_resources(adapter->rx_ring[i]);
564 		if (err) {
565 			netdev_err(dev, "Error on Rx queue %u setup\n", i);
566 			for (i--; i >= 0; i--)
567 				igc_free_rx_resources(adapter->rx_ring[i]);
568 			break;
569 		}
570 	}
571 
572 	return err;
573 }
574 
igc_get_xsk_pool(struct igc_adapter * adapter,struct igc_ring * ring)575 static struct xsk_buff_pool *igc_get_xsk_pool(struct igc_adapter *adapter,
576 					      struct igc_ring *ring)
577 {
578 	if (!igc_xdp_is_enabled(adapter) ||
579 	    !test_bit(IGC_RING_FLAG_AF_XDP_ZC, &ring->flags))
580 		return NULL;
581 
582 	return xsk_get_pool_from_qid(ring->netdev, ring->queue_index);
583 }
584 
585 /**
586  * igc_configure_rx_ring - Configure a receive ring after Reset
587  * @adapter: board private structure
588  * @ring: receive ring to be configured
589  *
590  * Configure the Rx unit of the MAC after a reset.
591  */
igc_configure_rx_ring(struct igc_adapter * adapter,struct igc_ring * ring)592 static void igc_configure_rx_ring(struct igc_adapter *adapter,
593 				  struct igc_ring *ring)
594 {
595 	struct igc_hw *hw = &adapter->hw;
596 	union igc_adv_rx_desc *rx_desc;
597 	int reg_idx = ring->reg_idx;
598 	u32 srrctl = 0, rxdctl = 0;
599 	u64 rdba = ring->dma;
600 	u32 buf_size;
601 
602 	xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
603 	ring->xsk_pool = igc_get_xsk_pool(adapter, ring);
604 	if (ring->xsk_pool) {
605 		WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
606 						   MEM_TYPE_XSK_BUFF_POOL,
607 						   NULL));
608 		xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
609 	} else {
610 		WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
611 						   MEM_TYPE_PAGE_SHARED,
612 						   NULL));
613 	}
614 
615 	if (igc_xdp_is_enabled(adapter))
616 		set_ring_uses_large_buffer(ring);
617 
618 	/* disable the queue */
619 	wr32(IGC_RXDCTL(reg_idx), 0);
620 
621 	/* Set DMA base address registers */
622 	wr32(IGC_RDBAL(reg_idx),
623 	     rdba & 0x00000000ffffffffULL);
624 	wr32(IGC_RDBAH(reg_idx), rdba >> 32);
625 	wr32(IGC_RDLEN(reg_idx),
626 	     ring->count * sizeof(union igc_adv_rx_desc));
627 
628 	/* initialize head and tail */
629 	ring->tail = adapter->io_addr + IGC_RDT(reg_idx);
630 	wr32(IGC_RDH(reg_idx), 0);
631 	writel(0, ring->tail);
632 
633 	/* reset next-to- use/clean to place SW in sync with hardware */
634 	ring->next_to_clean = 0;
635 	ring->next_to_use = 0;
636 
637 	if (ring->xsk_pool)
638 		buf_size = xsk_pool_get_rx_frame_size(ring->xsk_pool);
639 	else if (ring_uses_large_buffer(ring))
640 		buf_size = IGC_RXBUFFER_3072;
641 	else
642 		buf_size = IGC_RXBUFFER_2048;
643 
644 	srrctl = IGC_RX_HDR_LEN << IGC_SRRCTL_BSIZEHDRSIZE_SHIFT;
645 	srrctl |= buf_size >> IGC_SRRCTL_BSIZEPKT_SHIFT;
646 	srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF;
647 
648 	wr32(IGC_SRRCTL(reg_idx), srrctl);
649 
650 	rxdctl |= IGC_RX_PTHRESH;
651 	rxdctl |= IGC_RX_HTHRESH << 8;
652 	rxdctl |= IGC_RX_WTHRESH << 16;
653 
654 	/* initialize rx_buffer_info */
655 	memset(ring->rx_buffer_info, 0,
656 	       sizeof(struct igc_rx_buffer) * ring->count);
657 
658 	/* initialize Rx descriptor 0 */
659 	rx_desc = IGC_RX_DESC(ring, 0);
660 	rx_desc->wb.upper.length = 0;
661 
662 	/* enable receive descriptor fetching */
663 	rxdctl |= IGC_RXDCTL_QUEUE_ENABLE;
664 
665 	wr32(IGC_RXDCTL(reg_idx), rxdctl);
666 }
667 
668 /**
669  * igc_configure_rx - Configure receive Unit after Reset
670  * @adapter: board private structure
671  *
672  * Configure the Rx unit of the MAC after a reset.
673  */
igc_configure_rx(struct igc_adapter * adapter)674 static void igc_configure_rx(struct igc_adapter *adapter)
675 {
676 	int i;
677 
678 	/* Setup the HW Rx Head and Tail Descriptor Pointers and
679 	 * the Base and Length of the Rx Descriptor Ring
680 	 */
681 	for (i = 0; i < adapter->num_rx_queues; i++)
682 		igc_configure_rx_ring(adapter, adapter->rx_ring[i]);
683 }
684 
685 /**
686  * igc_configure_tx_ring - Configure transmit ring after Reset
687  * @adapter: board private structure
688  * @ring: tx ring to configure
689  *
690  * Configure a transmit ring after a reset.
691  */
igc_configure_tx_ring(struct igc_adapter * adapter,struct igc_ring * ring)692 static void igc_configure_tx_ring(struct igc_adapter *adapter,
693 				  struct igc_ring *ring)
694 {
695 	struct igc_hw *hw = &adapter->hw;
696 	int reg_idx = ring->reg_idx;
697 	u64 tdba = ring->dma;
698 	u32 txdctl = 0;
699 
700 	ring->xsk_pool = igc_get_xsk_pool(adapter, ring);
701 
702 	/* disable the queue */
703 	wr32(IGC_TXDCTL(reg_idx), 0);
704 	wrfl();
705 	mdelay(10);
706 
707 	wr32(IGC_TDLEN(reg_idx),
708 	     ring->count * sizeof(union igc_adv_tx_desc));
709 	wr32(IGC_TDBAL(reg_idx),
710 	     tdba & 0x00000000ffffffffULL);
711 	wr32(IGC_TDBAH(reg_idx), tdba >> 32);
712 
713 	ring->tail = adapter->io_addr + IGC_TDT(reg_idx);
714 	wr32(IGC_TDH(reg_idx), 0);
715 	writel(0, ring->tail);
716 
717 	txdctl |= IGC_TX_PTHRESH;
718 	txdctl |= IGC_TX_HTHRESH << 8;
719 	txdctl |= IGC_TX_WTHRESH << 16;
720 
721 	txdctl |= IGC_TXDCTL_QUEUE_ENABLE;
722 	wr32(IGC_TXDCTL(reg_idx), txdctl);
723 }
724 
725 /**
726  * igc_configure_tx - Configure transmit Unit after Reset
727  * @adapter: board private structure
728  *
729  * Configure the Tx unit of the MAC after a reset.
730  */
igc_configure_tx(struct igc_adapter * adapter)731 static void igc_configure_tx(struct igc_adapter *adapter)
732 {
733 	int i;
734 
735 	for (i = 0; i < adapter->num_tx_queues; i++)
736 		igc_configure_tx_ring(adapter, adapter->tx_ring[i]);
737 }
738 
739 /**
740  * igc_setup_mrqc - configure the multiple receive queue control registers
741  * @adapter: Board private structure
742  */
igc_setup_mrqc(struct igc_adapter * adapter)743 static void igc_setup_mrqc(struct igc_adapter *adapter)
744 {
745 	struct igc_hw *hw = &adapter->hw;
746 	u32 j, num_rx_queues;
747 	u32 mrqc, rxcsum;
748 	u32 rss_key[10];
749 
750 	netdev_rss_key_fill(rss_key, sizeof(rss_key));
751 	for (j = 0; j < 10; j++)
752 		wr32(IGC_RSSRK(j), rss_key[j]);
753 
754 	num_rx_queues = adapter->rss_queues;
755 
756 	if (adapter->rss_indir_tbl_init != num_rx_queues) {
757 		for (j = 0; j < IGC_RETA_SIZE; j++)
758 			adapter->rss_indir_tbl[j] =
759 			(j * num_rx_queues) / IGC_RETA_SIZE;
760 		adapter->rss_indir_tbl_init = num_rx_queues;
761 	}
762 	igc_write_rss_indir_tbl(adapter);
763 
764 	/* Disable raw packet checksumming so that RSS hash is placed in
765 	 * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
766 	 * offloads as they are enabled by default
767 	 */
768 	rxcsum = rd32(IGC_RXCSUM);
769 	rxcsum |= IGC_RXCSUM_PCSD;
770 
771 	/* Enable Receive Checksum Offload for SCTP */
772 	rxcsum |= IGC_RXCSUM_CRCOFL;
773 
774 	/* Don't need to set TUOFL or IPOFL, they default to 1 */
775 	wr32(IGC_RXCSUM, rxcsum);
776 
777 	/* Generate RSS hash based on packet types, TCP/UDP
778 	 * port numbers and/or IPv4/v6 src and dst addresses
779 	 */
780 	mrqc = IGC_MRQC_RSS_FIELD_IPV4 |
781 	       IGC_MRQC_RSS_FIELD_IPV4_TCP |
782 	       IGC_MRQC_RSS_FIELD_IPV6 |
783 	       IGC_MRQC_RSS_FIELD_IPV6_TCP |
784 	       IGC_MRQC_RSS_FIELD_IPV6_TCP_EX;
785 
786 	if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP)
787 		mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP;
788 	if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP)
789 		mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP;
790 
791 	mrqc |= IGC_MRQC_ENABLE_RSS_MQ;
792 
793 	wr32(IGC_MRQC, mrqc);
794 }
795 
796 /**
797  * igc_setup_rctl - configure the receive control registers
798  * @adapter: Board private structure
799  */
igc_setup_rctl(struct igc_adapter * adapter)800 static void igc_setup_rctl(struct igc_adapter *adapter)
801 {
802 	struct igc_hw *hw = &adapter->hw;
803 	u32 rctl;
804 
805 	rctl = rd32(IGC_RCTL);
806 
807 	rctl &= ~(3 << IGC_RCTL_MO_SHIFT);
808 	rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC);
809 
810 	rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF |
811 		(hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT);
812 
813 	/* enable stripping of CRC. Newer features require
814 	 * that the HW strips the CRC.
815 	 */
816 	rctl |= IGC_RCTL_SECRC;
817 
818 	/* disable store bad packets and clear size bits. */
819 	rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256);
820 
821 	/* enable LPE to allow for reception of jumbo frames */
822 	rctl |= IGC_RCTL_LPE;
823 
824 	/* disable queue 0 to prevent tail write w/o re-config */
825 	wr32(IGC_RXDCTL(0), 0);
826 
827 	/* This is useful for sniffing bad packets. */
828 	if (adapter->netdev->features & NETIF_F_RXALL) {
829 		/* UPE and MPE will be handled by normal PROMISC logic
830 		 * in set_rx_mode
831 		 */
832 		rctl |= (IGC_RCTL_SBP | /* Receive bad packets */
833 			 IGC_RCTL_BAM | /* RX All Bcast Pkts */
834 			 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
835 
836 		rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */
837 			  IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */
838 	}
839 
840 	wr32(IGC_RCTL, rctl);
841 }
842 
843 /**
844  * igc_setup_tctl - configure the transmit control registers
845  * @adapter: Board private structure
846  */
igc_setup_tctl(struct igc_adapter * adapter)847 static void igc_setup_tctl(struct igc_adapter *adapter)
848 {
849 	struct igc_hw *hw = &adapter->hw;
850 	u32 tctl;
851 
852 	/* disable queue 0 which icould be enabled by default */
853 	wr32(IGC_TXDCTL(0), 0);
854 
855 	/* Program the Transmit Control Register */
856 	tctl = rd32(IGC_TCTL);
857 	tctl &= ~IGC_TCTL_CT;
858 	tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC |
859 		(IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT);
860 
861 	/* Enable transmits */
862 	tctl |= IGC_TCTL_EN;
863 
864 	wr32(IGC_TCTL, tctl);
865 }
866 
867 /**
868  * igc_set_mac_filter_hw() - Set MAC address filter in hardware
869  * @adapter: Pointer to adapter where the filter should be set
870  * @index: Filter index
871  * @type: MAC address filter type (source or destination)
872  * @addr: MAC address
873  * @queue: If non-negative, queue assignment feature is enabled and frames
874  *         matching the filter are enqueued onto 'queue'. Otherwise, queue
875  *         assignment is disabled.
876  */
igc_set_mac_filter_hw(struct igc_adapter * adapter,int index,enum igc_mac_filter_type type,const u8 * addr,int queue)877 static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index,
878 				  enum igc_mac_filter_type type,
879 				  const u8 *addr, int queue)
880 {
881 	struct net_device *dev = adapter->netdev;
882 	struct igc_hw *hw = &adapter->hw;
883 	u32 ral, rah;
884 
885 	if (WARN_ON(index >= hw->mac.rar_entry_count))
886 		return;
887 
888 	ral = le32_to_cpup((__le32 *)(addr));
889 	rah = le16_to_cpup((__le16 *)(addr + 4));
890 
891 	if (type == IGC_MAC_FILTER_TYPE_SRC) {
892 		rah &= ~IGC_RAH_ASEL_MASK;
893 		rah |= IGC_RAH_ASEL_SRC_ADDR;
894 	}
895 
896 	if (queue >= 0) {
897 		rah &= ~IGC_RAH_QSEL_MASK;
898 		rah |= (queue << IGC_RAH_QSEL_SHIFT);
899 		rah |= IGC_RAH_QSEL_ENABLE;
900 	}
901 
902 	rah |= IGC_RAH_AV;
903 
904 	wr32(IGC_RAL(index), ral);
905 	wr32(IGC_RAH(index), rah);
906 
907 	netdev_dbg(dev, "MAC address filter set in HW: index %d", index);
908 }
909 
910 /**
911  * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware
912  * @adapter: Pointer to adapter where the filter should be cleared
913  * @index: Filter index
914  */
igc_clear_mac_filter_hw(struct igc_adapter * adapter,int index)915 static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index)
916 {
917 	struct net_device *dev = adapter->netdev;
918 	struct igc_hw *hw = &adapter->hw;
919 
920 	if (WARN_ON(index >= hw->mac.rar_entry_count))
921 		return;
922 
923 	wr32(IGC_RAL(index), 0);
924 	wr32(IGC_RAH(index), 0);
925 
926 	netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index);
927 }
928 
929 /* Set default MAC address for the PF in the first RAR entry */
igc_set_default_mac_filter(struct igc_adapter * adapter)930 static void igc_set_default_mac_filter(struct igc_adapter *adapter)
931 {
932 	struct net_device *dev = adapter->netdev;
933 	u8 *addr = adapter->hw.mac.addr;
934 
935 	netdev_dbg(dev, "Set default MAC address filter: address %pM", addr);
936 
937 	igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1);
938 }
939 
940 /**
941  * igc_set_mac - Change the Ethernet Address of the NIC
942  * @netdev: network interface device structure
943  * @p: pointer to an address structure
944  *
945  * Returns 0 on success, negative on failure
946  */
igc_set_mac(struct net_device * netdev,void * p)947 static int igc_set_mac(struct net_device *netdev, void *p)
948 {
949 	struct igc_adapter *adapter = netdev_priv(netdev);
950 	struct igc_hw *hw = &adapter->hw;
951 	struct sockaddr *addr = p;
952 
953 	if (!is_valid_ether_addr(addr->sa_data))
954 		return -EADDRNOTAVAIL;
955 
956 	eth_hw_addr_set(netdev, addr->sa_data);
957 	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
958 
959 	/* set the correct pool for the new PF MAC address in entry 0 */
960 	igc_set_default_mac_filter(adapter);
961 
962 	return 0;
963 }
964 
965 /**
966  *  igc_write_mc_addr_list - write multicast addresses to MTA
967  *  @netdev: network interface device structure
968  *
969  *  Writes multicast address list to the MTA hash table.
970  *  Returns: -ENOMEM on failure
971  *           0 on no addresses written
972  *           X on writing X addresses to MTA
973  **/
igc_write_mc_addr_list(struct net_device * netdev)974 static int igc_write_mc_addr_list(struct net_device *netdev)
975 {
976 	struct igc_adapter *adapter = netdev_priv(netdev);
977 	struct igc_hw *hw = &adapter->hw;
978 	struct netdev_hw_addr *ha;
979 	u8  *mta_list;
980 	int i;
981 
982 	if (netdev_mc_empty(netdev)) {
983 		/* nothing to program, so clear mc list */
984 		igc_update_mc_addr_list(hw, NULL, 0);
985 		return 0;
986 	}
987 
988 	mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC);
989 	if (!mta_list)
990 		return -ENOMEM;
991 
992 	/* The shared function expects a packed array of only addresses. */
993 	i = 0;
994 	netdev_for_each_mc_addr(ha, netdev)
995 		memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
996 
997 	igc_update_mc_addr_list(hw, mta_list, i);
998 	kfree(mta_list);
999 
1000 	return netdev_mc_count(netdev);
1001 }
1002 
igc_tx_launchtime(struct igc_ring * ring,ktime_t txtime,bool * first_flag,bool * insert_empty)1003 static __le32 igc_tx_launchtime(struct igc_ring *ring, ktime_t txtime,
1004 				bool *first_flag, bool *insert_empty)
1005 {
1006 	struct igc_adapter *adapter = netdev_priv(ring->netdev);
1007 	ktime_t cycle_time = adapter->cycle_time;
1008 	ktime_t base_time = adapter->base_time;
1009 	ktime_t now = ktime_get_clocktai();
1010 	ktime_t baset_est, end_of_cycle;
1011 	u32 launchtime;
1012 	s64 n;
1013 
1014 	n = div64_s64(ktime_sub_ns(now, base_time), cycle_time);
1015 
1016 	baset_est = ktime_add_ns(base_time, cycle_time * (n));
1017 	end_of_cycle = ktime_add_ns(baset_est, cycle_time);
1018 
1019 	if (ktime_compare(txtime, end_of_cycle) >= 0) {
1020 		if (baset_est != ring->last_ff_cycle) {
1021 			*first_flag = true;
1022 			ring->last_ff_cycle = baset_est;
1023 
1024 			if (ktime_compare(txtime, ring->last_tx_cycle) > 0)
1025 				*insert_empty = true;
1026 		}
1027 	}
1028 
1029 	/* Introducing a window at end of cycle on which packets
1030 	 * potentially not honor launchtime. Window of 5us chosen
1031 	 * considering software update the tail pointer and packets
1032 	 * are dma'ed to packet buffer.
1033 	 */
1034 	if ((ktime_sub_ns(end_of_cycle, now) < 5 * NSEC_PER_USEC))
1035 		netdev_warn(ring->netdev, "Packet with txtime=%llu may not be honoured\n",
1036 			    txtime);
1037 
1038 	ring->last_tx_cycle = end_of_cycle;
1039 
1040 	launchtime = ktime_sub_ns(txtime, baset_est);
1041 	if (launchtime > 0)
1042 		div_s64_rem(launchtime, cycle_time, &launchtime);
1043 	else
1044 		launchtime = 0;
1045 
1046 	return cpu_to_le32(launchtime);
1047 }
1048 
igc_init_empty_frame(struct igc_ring * ring,struct igc_tx_buffer * buffer,struct sk_buff * skb)1049 static int igc_init_empty_frame(struct igc_ring *ring,
1050 				struct igc_tx_buffer *buffer,
1051 				struct sk_buff *skb)
1052 {
1053 	unsigned int size;
1054 	dma_addr_t dma;
1055 
1056 	size = skb_headlen(skb);
1057 
1058 	dma = dma_map_single(ring->dev, skb->data, size, DMA_TO_DEVICE);
1059 	if (dma_mapping_error(ring->dev, dma)) {
1060 		netdev_err_once(ring->netdev, "Failed to map DMA for TX\n");
1061 		return -ENOMEM;
1062 	}
1063 
1064 	buffer->skb = skb;
1065 	buffer->protocol = 0;
1066 	buffer->bytecount = skb->len;
1067 	buffer->gso_segs = 1;
1068 	buffer->time_stamp = jiffies;
1069 	dma_unmap_len_set(buffer, len, skb->len);
1070 	dma_unmap_addr_set(buffer, dma, dma);
1071 
1072 	return 0;
1073 }
1074 
igc_init_tx_empty_descriptor(struct igc_ring * ring,struct sk_buff * skb,struct igc_tx_buffer * first)1075 static int igc_init_tx_empty_descriptor(struct igc_ring *ring,
1076 					struct sk_buff *skb,
1077 					struct igc_tx_buffer *first)
1078 {
1079 	union igc_adv_tx_desc *desc;
1080 	u32 cmd_type, olinfo_status;
1081 	int err;
1082 
1083 	if (!igc_desc_unused(ring))
1084 		return -EBUSY;
1085 
1086 	err = igc_init_empty_frame(ring, first, skb);
1087 	if (err)
1088 		return err;
1089 
1090 	cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
1091 		   IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD |
1092 		   first->bytecount;
1093 	olinfo_status = first->bytecount << IGC_ADVTXD_PAYLEN_SHIFT;
1094 
1095 	desc = IGC_TX_DESC(ring, ring->next_to_use);
1096 	desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1097 	desc->read.olinfo_status = cpu_to_le32(olinfo_status);
1098 	desc->read.buffer_addr = cpu_to_le64(dma_unmap_addr(first, dma));
1099 
1100 	netdev_tx_sent_queue(txring_txq(ring), skb->len);
1101 
1102 	first->next_to_watch = desc;
1103 
1104 	ring->next_to_use++;
1105 	if (ring->next_to_use == ring->count)
1106 		ring->next_to_use = 0;
1107 
1108 	return 0;
1109 }
1110 
1111 #define IGC_EMPTY_FRAME_SIZE 60
1112 
igc_tx_ctxtdesc(struct igc_ring * tx_ring,__le32 launch_time,bool first_flag,u32 vlan_macip_lens,u32 type_tucmd,u32 mss_l4len_idx)1113 static void igc_tx_ctxtdesc(struct igc_ring *tx_ring,
1114 			    __le32 launch_time, bool first_flag,
1115 			    u32 vlan_macip_lens, u32 type_tucmd,
1116 			    u32 mss_l4len_idx)
1117 {
1118 	struct igc_adv_tx_context_desc *context_desc;
1119 	u16 i = tx_ring->next_to_use;
1120 
1121 	context_desc = IGC_TX_CTXTDESC(tx_ring, i);
1122 
1123 	i++;
1124 	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1125 
1126 	/* set bits to identify this as an advanced context descriptor */
1127 	type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT;
1128 
1129 	/* For i225, context index must be unique per ring. */
1130 	if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
1131 		mss_l4len_idx |= tx_ring->reg_idx << 4;
1132 
1133 	if (first_flag)
1134 		mss_l4len_idx |= IGC_ADVTXD_TSN_CNTX_FIRST;
1135 
1136 	context_desc->vlan_macip_lens	= cpu_to_le32(vlan_macip_lens);
1137 	context_desc->type_tucmd_mlhl	= cpu_to_le32(type_tucmd);
1138 	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);
1139 	context_desc->launch_time	= launch_time;
1140 }
1141 
igc_tx_csum(struct igc_ring * tx_ring,struct igc_tx_buffer * first,__le32 launch_time,bool first_flag)1142 static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first,
1143 			__le32 launch_time, bool first_flag)
1144 {
1145 	struct sk_buff *skb = first->skb;
1146 	u32 vlan_macip_lens = 0;
1147 	u32 type_tucmd = 0;
1148 
1149 	if (skb->ip_summed != CHECKSUM_PARTIAL) {
1150 csum_failed:
1151 		if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) &&
1152 		    !tx_ring->launchtime_enable)
1153 			return;
1154 		goto no_csum;
1155 	}
1156 
1157 	switch (skb->csum_offset) {
1158 	case offsetof(struct tcphdr, check):
1159 		type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP;
1160 		fallthrough;
1161 	case offsetof(struct udphdr, check):
1162 		break;
1163 	case offsetof(struct sctphdr, checksum):
1164 		/* validate that this is actually an SCTP request */
1165 		if (skb_csum_is_sctp(skb)) {
1166 			type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP;
1167 			break;
1168 		}
1169 		fallthrough;
1170 	default:
1171 		skb_checksum_help(skb);
1172 		goto csum_failed;
1173 	}
1174 
1175 	/* update TX checksum flag */
1176 	first->tx_flags |= IGC_TX_FLAGS_CSUM;
1177 	vlan_macip_lens = skb_checksum_start_offset(skb) -
1178 			  skb_network_offset(skb);
1179 no_csum:
1180 	vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT;
1181 	vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK;
1182 
1183 	igc_tx_ctxtdesc(tx_ring, launch_time, first_flag,
1184 			vlan_macip_lens, type_tucmd, 0);
1185 }
1186 
__igc_maybe_stop_tx(struct igc_ring * tx_ring,const u16 size)1187 static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
1188 {
1189 	struct net_device *netdev = tx_ring->netdev;
1190 
1191 	netif_stop_subqueue(netdev, tx_ring->queue_index);
1192 
1193 	/* memory barriier comment */
1194 	smp_mb();
1195 
1196 	/* We need to check again in a case another CPU has just
1197 	 * made room available.
1198 	 */
1199 	if (igc_desc_unused(tx_ring) < size)
1200 		return -EBUSY;
1201 
1202 	/* A reprieve! */
1203 	netif_wake_subqueue(netdev, tx_ring->queue_index);
1204 
1205 	u64_stats_update_begin(&tx_ring->tx_syncp2);
1206 	tx_ring->tx_stats.restart_queue2++;
1207 	u64_stats_update_end(&tx_ring->tx_syncp2);
1208 
1209 	return 0;
1210 }
1211 
igc_maybe_stop_tx(struct igc_ring * tx_ring,const u16 size)1212 static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
1213 {
1214 	if (igc_desc_unused(tx_ring) >= size)
1215 		return 0;
1216 	return __igc_maybe_stop_tx(tx_ring, size);
1217 }
1218 
1219 #define IGC_SET_FLAG(_input, _flag, _result) \
1220 	(((_flag) <= (_result)) ?				\
1221 	 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) :	\
1222 	 ((u32)((_input) & (_flag)) / ((_flag) / (_result))))
1223 
igc_tx_cmd_type(struct sk_buff * skb,u32 tx_flags)1224 static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
1225 {
1226 	/* set type for advanced descriptor with frame checksum insertion */
1227 	u32 cmd_type = IGC_ADVTXD_DTYP_DATA |
1228 		       IGC_ADVTXD_DCMD_DEXT |
1229 		       IGC_ADVTXD_DCMD_IFCS;
1230 
1231 	/* set HW vlan bit if vlan is present */
1232 	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_VLAN,
1233 				 IGC_ADVTXD_DCMD_VLE);
1234 
1235 	/* set segmentation bits for TSO */
1236 	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO,
1237 				 (IGC_ADVTXD_DCMD_TSE));
1238 
1239 	/* set timestamp bit if present */
1240 	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP,
1241 				 (IGC_ADVTXD_MAC_TSTAMP));
1242 
1243 	/* insert frame checksum */
1244 	cmd_type ^= IGC_SET_FLAG(skb->no_fcs, 1, IGC_ADVTXD_DCMD_IFCS);
1245 
1246 	return cmd_type;
1247 }
1248 
igc_tx_olinfo_status(struct igc_ring * tx_ring,union igc_adv_tx_desc * tx_desc,u32 tx_flags,unsigned int paylen)1249 static void igc_tx_olinfo_status(struct igc_ring *tx_ring,
1250 				 union igc_adv_tx_desc *tx_desc,
1251 				 u32 tx_flags, unsigned int paylen)
1252 {
1253 	u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT;
1254 
1255 	/* insert L4 checksum */
1256 	olinfo_status |= (tx_flags & IGC_TX_FLAGS_CSUM) *
1257 			  ((IGC_TXD_POPTS_TXSM << 8) /
1258 			  IGC_TX_FLAGS_CSUM);
1259 
1260 	/* insert IPv4 checksum */
1261 	olinfo_status |= (tx_flags & IGC_TX_FLAGS_IPV4) *
1262 			  (((IGC_TXD_POPTS_IXSM << 8)) /
1263 			  IGC_TX_FLAGS_IPV4);
1264 
1265 	tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
1266 }
1267 
igc_tx_map(struct igc_ring * tx_ring,struct igc_tx_buffer * first,const u8 hdr_len)1268 static int igc_tx_map(struct igc_ring *tx_ring,
1269 		      struct igc_tx_buffer *first,
1270 		      const u8 hdr_len)
1271 {
1272 	struct sk_buff *skb = first->skb;
1273 	struct igc_tx_buffer *tx_buffer;
1274 	union igc_adv_tx_desc *tx_desc;
1275 	u32 tx_flags = first->tx_flags;
1276 	skb_frag_t *frag;
1277 	u16 i = tx_ring->next_to_use;
1278 	unsigned int data_len, size;
1279 	dma_addr_t dma;
1280 	u32 cmd_type;
1281 
1282 	cmd_type = igc_tx_cmd_type(skb, tx_flags);
1283 	tx_desc = IGC_TX_DESC(tx_ring, i);
1284 
1285 	igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
1286 
1287 	size = skb_headlen(skb);
1288 	data_len = skb->data_len;
1289 
1290 	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
1291 
1292 	tx_buffer = first;
1293 
1294 	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
1295 		if (dma_mapping_error(tx_ring->dev, dma))
1296 			goto dma_error;
1297 
1298 		/* record length, and DMA address */
1299 		dma_unmap_len_set(tx_buffer, len, size);
1300 		dma_unmap_addr_set(tx_buffer, dma, dma);
1301 
1302 		tx_desc->read.buffer_addr = cpu_to_le64(dma);
1303 
1304 		while (unlikely(size > IGC_MAX_DATA_PER_TXD)) {
1305 			tx_desc->read.cmd_type_len =
1306 				cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD);
1307 
1308 			i++;
1309 			tx_desc++;
1310 			if (i == tx_ring->count) {
1311 				tx_desc = IGC_TX_DESC(tx_ring, 0);
1312 				i = 0;
1313 			}
1314 			tx_desc->read.olinfo_status = 0;
1315 
1316 			dma += IGC_MAX_DATA_PER_TXD;
1317 			size -= IGC_MAX_DATA_PER_TXD;
1318 
1319 			tx_desc->read.buffer_addr = cpu_to_le64(dma);
1320 		}
1321 
1322 		if (likely(!data_len))
1323 			break;
1324 
1325 		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
1326 
1327 		i++;
1328 		tx_desc++;
1329 		if (i == tx_ring->count) {
1330 			tx_desc = IGC_TX_DESC(tx_ring, 0);
1331 			i = 0;
1332 		}
1333 		tx_desc->read.olinfo_status = 0;
1334 
1335 		size = skb_frag_size(frag);
1336 		data_len -= size;
1337 
1338 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
1339 				       size, DMA_TO_DEVICE);
1340 
1341 		tx_buffer = &tx_ring->tx_buffer_info[i];
1342 	}
1343 
1344 	/* write last descriptor with RS and EOP bits */
1345 	cmd_type |= size | IGC_TXD_DCMD;
1346 	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1347 
1348 	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
1349 
1350 	/* set the timestamp */
1351 	first->time_stamp = jiffies;
1352 
1353 	skb_tx_timestamp(skb);
1354 
1355 	/* Force memory writes to complete before letting h/w know there
1356 	 * are new descriptors to fetch.  (Only applicable for weak-ordered
1357 	 * memory model archs, such as IA-64).
1358 	 *
1359 	 * We also need this memory barrier to make certain all of the
1360 	 * status bits have been updated before next_to_watch is written.
1361 	 */
1362 	wmb();
1363 
1364 	/* set next_to_watch value indicating a packet is present */
1365 	first->next_to_watch = tx_desc;
1366 
1367 	i++;
1368 	if (i == tx_ring->count)
1369 		i = 0;
1370 
1371 	tx_ring->next_to_use = i;
1372 
1373 	/* Make sure there is space in the ring for the next send. */
1374 	igc_maybe_stop_tx(tx_ring, DESC_NEEDED);
1375 
1376 	if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
1377 		writel(i, tx_ring->tail);
1378 	}
1379 
1380 	return 0;
1381 dma_error:
1382 	netdev_err(tx_ring->netdev, "TX DMA map failed\n");
1383 	tx_buffer = &tx_ring->tx_buffer_info[i];
1384 
1385 	/* clear dma mappings for failed tx_buffer_info map */
1386 	while (tx_buffer != first) {
1387 		if (dma_unmap_len(tx_buffer, len))
1388 			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
1389 
1390 		if (i-- == 0)
1391 			i += tx_ring->count;
1392 		tx_buffer = &tx_ring->tx_buffer_info[i];
1393 	}
1394 
1395 	if (dma_unmap_len(tx_buffer, len))
1396 		igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
1397 
1398 	dev_kfree_skb_any(tx_buffer->skb);
1399 	tx_buffer->skb = NULL;
1400 
1401 	tx_ring->next_to_use = i;
1402 
1403 	return -1;
1404 }
1405 
igc_tso(struct igc_ring * tx_ring,struct igc_tx_buffer * first,__le32 launch_time,bool first_flag,u8 * hdr_len)1406 static int igc_tso(struct igc_ring *tx_ring,
1407 		   struct igc_tx_buffer *first,
1408 		   __le32 launch_time, bool first_flag,
1409 		   u8 *hdr_len)
1410 {
1411 	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
1412 	struct sk_buff *skb = first->skb;
1413 	union {
1414 		struct iphdr *v4;
1415 		struct ipv6hdr *v6;
1416 		unsigned char *hdr;
1417 	} ip;
1418 	union {
1419 		struct tcphdr *tcp;
1420 		struct udphdr *udp;
1421 		unsigned char *hdr;
1422 	} l4;
1423 	u32 paylen, l4_offset;
1424 	int err;
1425 
1426 	if (skb->ip_summed != CHECKSUM_PARTIAL)
1427 		return 0;
1428 
1429 	if (!skb_is_gso(skb))
1430 		return 0;
1431 
1432 	err = skb_cow_head(skb, 0);
1433 	if (err < 0)
1434 		return err;
1435 
1436 	ip.hdr = skb_network_header(skb);
1437 	l4.hdr = skb_checksum_start(skb);
1438 
1439 	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
1440 	type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP;
1441 
1442 	/* initialize outer IP header fields */
1443 	if (ip.v4->version == 4) {
1444 		unsigned char *csum_start = skb_checksum_start(skb);
1445 		unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
1446 
1447 		/* IP header will have to cancel out any data that
1448 		 * is not a part of the outer IP header
1449 		 */
1450 		ip.v4->check = csum_fold(csum_partial(trans_start,
1451 						      csum_start - trans_start,
1452 						      0));
1453 		type_tucmd |= IGC_ADVTXD_TUCMD_IPV4;
1454 
1455 		ip.v4->tot_len = 0;
1456 		first->tx_flags |= IGC_TX_FLAGS_TSO |
1457 				   IGC_TX_FLAGS_CSUM |
1458 				   IGC_TX_FLAGS_IPV4;
1459 	} else {
1460 		ip.v6->payload_len = 0;
1461 		first->tx_flags |= IGC_TX_FLAGS_TSO |
1462 				   IGC_TX_FLAGS_CSUM;
1463 	}
1464 
1465 	/* determine offset of inner transport header */
1466 	l4_offset = l4.hdr - skb->data;
1467 
1468 	/* remove payload length from inner checksum */
1469 	paylen = skb->len - l4_offset;
1470 	if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) {
1471 		/* compute length of segmentation header */
1472 		*hdr_len = (l4.tcp->doff * 4) + l4_offset;
1473 		csum_replace_by_diff(&l4.tcp->check,
1474 				     (__force __wsum)htonl(paylen));
1475 	} else {
1476 		/* compute length of segmentation header */
1477 		*hdr_len = sizeof(*l4.udp) + l4_offset;
1478 		csum_replace_by_diff(&l4.udp->check,
1479 				     (__force __wsum)htonl(paylen));
1480 	}
1481 
1482 	/* update gso size and bytecount with header size */
1483 	first->gso_segs = skb_shinfo(skb)->gso_segs;
1484 	first->bytecount += (first->gso_segs - 1) * *hdr_len;
1485 
1486 	/* MSS L4LEN IDX */
1487 	mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT;
1488 	mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT;
1489 
1490 	/* VLAN MACLEN IPLEN */
1491 	vlan_macip_lens = l4.hdr - ip.hdr;
1492 	vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT;
1493 	vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK;
1494 
1495 	igc_tx_ctxtdesc(tx_ring, launch_time, first_flag,
1496 			vlan_macip_lens, type_tucmd, mss_l4len_idx);
1497 
1498 	return 1;
1499 }
1500 
igc_xmit_frame_ring(struct sk_buff * skb,struct igc_ring * tx_ring)1501 static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb,
1502 				       struct igc_ring *tx_ring)
1503 {
1504 	bool first_flag = false, insert_empty = false;
1505 	u16 count = TXD_USE_COUNT(skb_headlen(skb));
1506 	__be16 protocol = vlan_get_protocol(skb);
1507 	struct igc_tx_buffer *first;
1508 	__le32 launch_time = 0;
1509 	u32 tx_flags = 0;
1510 	unsigned short f;
1511 	ktime_t txtime;
1512 	u8 hdr_len = 0;
1513 	int tso = 0;
1514 
1515 	/* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD,
1516 	 *	+ 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD,
1517 	 *	+ 2 desc gap to keep tail from touching head,
1518 	 *	+ 1 desc for context descriptor,
1519 	 * otherwise try next time
1520 	 */
1521 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1522 		count += TXD_USE_COUNT(skb_frag_size(
1523 						&skb_shinfo(skb)->frags[f]));
1524 
1525 	if (igc_maybe_stop_tx(tx_ring, count + 5)) {
1526 		/* this is a hard error */
1527 		return NETDEV_TX_BUSY;
1528 	}
1529 
1530 	if (!tx_ring->launchtime_enable)
1531 		goto done;
1532 
1533 	txtime = skb->tstamp;
1534 	skb->tstamp = ktime_set(0, 0);
1535 	launch_time = igc_tx_launchtime(tx_ring, txtime, &first_flag, &insert_empty);
1536 
1537 	if (insert_empty) {
1538 		struct igc_tx_buffer *empty_info;
1539 		struct sk_buff *empty;
1540 		void *data;
1541 
1542 		empty_info = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1543 		empty = alloc_skb(IGC_EMPTY_FRAME_SIZE, GFP_ATOMIC);
1544 		if (!empty)
1545 			goto done;
1546 
1547 		data = skb_put(empty, IGC_EMPTY_FRAME_SIZE);
1548 		memset(data, 0, IGC_EMPTY_FRAME_SIZE);
1549 
1550 		igc_tx_ctxtdesc(tx_ring, 0, false, 0, 0, 0);
1551 
1552 		if (igc_init_tx_empty_descriptor(tx_ring,
1553 						 empty,
1554 						 empty_info) < 0)
1555 			dev_kfree_skb_any(empty);
1556 	}
1557 
1558 done:
1559 	/* record the location of the first descriptor for this packet */
1560 	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1561 	first->type = IGC_TX_BUFFER_TYPE_SKB;
1562 	first->skb = skb;
1563 	first->bytecount = skb->len;
1564 	first->gso_segs = 1;
1565 
1566 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
1567 		struct igc_adapter *adapter = netdev_priv(tx_ring->netdev);
1568 
1569 		/* FIXME: add support for retrieving timestamps from
1570 		 * the other timer registers before skipping the
1571 		 * timestamping request.
1572 		 */
1573 		if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
1574 		    !test_and_set_bit_lock(__IGC_PTP_TX_IN_PROGRESS,
1575 					   &adapter->state)) {
1576 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1577 			tx_flags |= IGC_TX_FLAGS_TSTAMP;
1578 
1579 			adapter->ptp_tx_skb = skb_get(skb);
1580 			adapter->ptp_tx_start = jiffies;
1581 		} else {
1582 			adapter->tx_hwtstamp_skipped++;
1583 		}
1584 	}
1585 
1586 	if (skb_vlan_tag_present(skb)) {
1587 		tx_flags |= IGC_TX_FLAGS_VLAN;
1588 		tx_flags |= (skb_vlan_tag_get(skb) << IGC_TX_FLAGS_VLAN_SHIFT);
1589 	}
1590 
1591 	/* record initial flags and protocol */
1592 	first->tx_flags = tx_flags;
1593 	first->protocol = protocol;
1594 
1595 	tso = igc_tso(tx_ring, first, launch_time, first_flag, &hdr_len);
1596 	if (tso < 0)
1597 		goto out_drop;
1598 	else if (!tso)
1599 		igc_tx_csum(tx_ring, first, launch_time, first_flag);
1600 
1601 	igc_tx_map(tx_ring, first, hdr_len);
1602 
1603 	return NETDEV_TX_OK;
1604 
1605 out_drop:
1606 	dev_kfree_skb_any(first->skb);
1607 	first->skb = NULL;
1608 
1609 	return NETDEV_TX_OK;
1610 }
1611 
igc_tx_queue_mapping(struct igc_adapter * adapter,struct sk_buff * skb)1612 static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter,
1613 						    struct sk_buff *skb)
1614 {
1615 	unsigned int r_idx = skb->queue_mapping;
1616 
1617 	if (r_idx >= adapter->num_tx_queues)
1618 		r_idx = r_idx % adapter->num_tx_queues;
1619 
1620 	return adapter->tx_ring[r_idx];
1621 }
1622 
igc_xmit_frame(struct sk_buff * skb,struct net_device * netdev)1623 static netdev_tx_t igc_xmit_frame(struct sk_buff *skb,
1624 				  struct net_device *netdev)
1625 {
1626 	struct igc_adapter *adapter = netdev_priv(netdev);
1627 
1628 	/* The minimum packet size with TCTL.PSP set is 17 so pad the skb
1629 	 * in order to meet this minimum size requirement.
1630 	 */
1631 	if (skb->len < 17) {
1632 		if (skb_padto(skb, 17))
1633 			return NETDEV_TX_OK;
1634 		skb->len = 17;
1635 	}
1636 
1637 	return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb));
1638 }
1639 
igc_rx_checksum(struct igc_ring * ring,union igc_adv_rx_desc * rx_desc,struct sk_buff * skb)1640 static void igc_rx_checksum(struct igc_ring *ring,
1641 			    union igc_adv_rx_desc *rx_desc,
1642 			    struct sk_buff *skb)
1643 {
1644 	skb_checksum_none_assert(skb);
1645 
1646 	/* Ignore Checksum bit is set */
1647 	if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM))
1648 		return;
1649 
1650 	/* Rx checksum disabled via ethtool */
1651 	if (!(ring->netdev->features & NETIF_F_RXCSUM))
1652 		return;
1653 
1654 	/* TCP/UDP checksum error bit is set */
1655 	if (igc_test_staterr(rx_desc,
1656 			     IGC_RXDEXT_STATERR_L4E |
1657 			     IGC_RXDEXT_STATERR_IPE)) {
1658 		/* work around errata with sctp packets where the TCPE aka
1659 		 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
1660 		 * packets (aka let the stack check the crc32c)
1661 		 */
1662 		if (!(skb->len == 60 &&
1663 		      test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
1664 			u64_stats_update_begin(&ring->rx_syncp);
1665 			ring->rx_stats.csum_err++;
1666 			u64_stats_update_end(&ring->rx_syncp);
1667 		}
1668 		/* let the stack verify checksum errors */
1669 		return;
1670 	}
1671 	/* It must be a TCP or UDP packet with a valid checksum */
1672 	if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS |
1673 				      IGC_RXD_STAT_UDPCS))
1674 		skb->ip_summed = CHECKSUM_UNNECESSARY;
1675 
1676 	netdev_dbg(ring->netdev, "cksum success: bits %08X\n",
1677 		   le32_to_cpu(rx_desc->wb.upper.status_error));
1678 }
1679 
igc_rx_hash(struct igc_ring * ring,union igc_adv_rx_desc * rx_desc,struct sk_buff * skb)1680 static inline void igc_rx_hash(struct igc_ring *ring,
1681 			       union igc_adv_rx_desc *rx_desc,
1682 			       struct sk_buff *skb)
1683 {
1684 	if (ring->netdev->features & NETIF_F_RXHASH)
1685 		skb_set_hash(skb,
1686 			     le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
1687 			     PKT_HASH_TYPE_L3);
1688 }
1689 
igc_rx_vlan(struct igc_ring * rx_ring,union igc_adv_rx_desc * rx_desc,struct sk_buff * skb)1690 static void igc_rx_vlan(struct igc_ring *rx_ring,
1691 			union igc_adv_rx_desc *rx_desc,
1692 			struct sk_buff *skb)
1693 {
1694 	struct net_device *dev = rx_ring->netdev;
1695 	u16 vid;
1696 
1697 	if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
1698 	    igc_test_staterr(rx_desc, IGC_RXD_STAT_VP)) {
1699 		if (igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_LB) &&
1700 		    test_bit(IGC_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
1701 			vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan);
1702 		else
1703 			vid = le16_to_cpu(rx_desc->wb.upper.vlan);
1704 
1705 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
1706 	}
1707 }
1708 
1709 /**
1710  * igc_process_skb_fields - Populate skb header fields from Rx descriptor
1711  * @rx_ring: rx descriptor ring packet is being transacted on
1712  * @rx_desc: pointer to the EOP Rx descriptor
1713  * @skb: pointer to current skb being populated
1714  *
1715  * This function checks the ring, descriptor, and packet information in order
1716  * to populate the hash, checksum, VLAN, protocol, and other fields within the
1717  * skb.
1718  */
igc_process_skb_fields(struct igc_ring * rx_ring,union igc_adv_rx_desc * rx_desc,struct sk_buff * skb)1719 static void igc_process_skb_fields(struct igc_ring *rx_ring,
1720 				   union igc_adv_rx_desc *rx_desc,
1721 				   struct sk_buff *skb)
1722 {
1723 	igc_rx_hash(rx_ring, rx_desc, skb);
1724 
1725 	igc_rx_checksum(rx_ring, rx_desc, skb);
1726 
1727 	igc_rx_vlan(rx_ring, rx_desc, skb);
1728 
1729 	skb_record_rx_queue(skb, rx_ring->queue_index);
1730 
1731 	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1732 }
1733 
igc_vlan_mode(struct net_device * netdev,netdev_features_t features)1734 static void igc_vlan_mode(struct net_device *netdev, netdev_features_t features)
1735 {
1736 	bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
1737 	struct igc_adapter *adapter = netdev_priv(netdev);
1738 	struct igc_hw *hw = &adapter->hw;
1739 	u32 ctrl;
1740 
1741 	ctrl = rd32(IGC_CTRL);
1742 
1743 	if (enable) {
1744 		/* enable VLAN tag insert/strip */
1745 		ctrl |= IGC_CTRL_VME;
1746 	} else {
1747 		/* disable VLAN tag insert/strip */
1748 		ctrl &= ~IGC_CTRL_VME;
1749 	}
1750 	wr32(IGC_CTRL, ctrl);
1751 }
1752 
igc_restore_vlan(struct igc_adapter * adapter)1753 static void igc_restore_vlan(struct igc_adapter *adapter)
1754 {
1755 	igc_vlan_mode(adapter->netdev, adapter->netdev->features);
1756 }
1757 
igc_get_rx_buffer(struct igc_ring * rx_ring,const unsigned int size,int * rx_buffer_pgcnt)1758 static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring,
1759 					       const unsigned int size,
1760 					       int *rx_buffer_pgcnt)
1761 {
1762 	struct igc_rx_buffer *rx_buffer;
1763 
1764 	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
1765 	*rx_buffer_pgcnt =
1766 #if (PAGE_SIZE < 8192)
1767 		page_count(rx_buffer->page);
1768 #else
1769 		0;
1770 #endif
1771 	prefetchw(rx_buffer->page);
1772 
1773 	/* we are reusing so sync this buffer for CPU use */
1774 	dma_sync_single_range_for_cpu(rx_ring->dev,
1775 				      rx_buffer->dma,
1776 				      rx_buffer->page_offset,
1777 				      size,
1778 				      DMA_FROM_DEVICE);
1779 
1780 	rx_buffer->pagecnt_bias--;
1781 
1782 	return rx_buffer;
1783 }
1784 
igc_rx_buffer_flip(struct igc_rx_buffer * buffer,unsigned int truesize)1785 static void igc_rx_buffer_flip(struct igc_rx_buffer *buffer,
1786 			       unsigned int truesize)
1787 {
1788 #if (PAGE_SIZE < 8192)
1789 	buffer->page_offset ^= truesize;
1790 #else
1791 	buffer->page_offset += truesize;
1792 #endif
1793 }
1794 
igc_get_rx_frame_truesize(struct igc_ring * ring,unsigned int size)1795 static unsigned int igc_get_rx_frame_truesize(struct igc_ring *ring,
1796 					      unsigned int size)
1797 {
1798 	unsigned int truesize;
1799 
1800 #if (PAGE_SIZE < 8192)
1801 	truesize = igc_rx_pg_size(ring) / 2;
1802 #else
1803 	truesize = ring_uses_build_skb(ring) ?
1804 		   SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1805 		   SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
1806 		   SKB_DATA_ALIGN(size);
1807 #endif
1808 	return truesize;
1809 }
1810 
1811 /**
1812  * igc_add_rx_frag - Add contents of Rx buffer to sk_buff
1813  * @rx_ring: rx descriptor ring to transact packets on
1814  * @rx_buffer: buffer containing page to add
1815  * @skb: sk_buff to place the data into
1816  * @size: size of buffer to be added
1817  *
1818  * This function will add the data contained in rx_buffer->page to the skb.
1819  */
igc_add_rx_frag(struct igc_ring * rx_ring,struct igc_rx_buffer * rx_buffer,struct sk_buff * skb,unsigned int size)1820 static void igc_add_rx_frag(struct igc_ring *rx_ring,
1821 			    struct igc_rx_buffer *rx_buffer,
1822 			    struct sk_buff *skb,
1823 			    unsigned int size)
1824 {
1825 	unsigned int truesize;
1826 
1827 #if (PAGE_SIZE < 8192)
1828 	truesize = igc_rx_pg_size(rx_ring) / 2;
1829 #else
1830 	truesize = ring_uses_build_skb(rx_ring) ?
1831 		   SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
1832 		   SKB_DATA_ALIGN(size);
1833 #endif
1834 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1835 			rx_buffer->page_offset, size, truesize);
1836 
1837 	igc_rx_buffer_flip(rx_buffer, truesize);
1838 }
1839 
igc_build_skb(struct igc_ring * rx_ring,struct igc_rx_buffer * rx_buffer,struct xdp_buff * xdp)1840 static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring,
1841 				     struct igc_rx_buffer *rx_buffer,
1842 				     struct xdp_buff *xdp)
1843 {
1844 	unsigned int size = xdp->data_end - xdp->data;
1845 	unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size);
1846 	unsigned int metasize = xdp->data - xdp->data_meta;
1847 	struct sk_buff *skb;
1848 
1849 	/* prefetch first cache line of first page */
1850 	net_prefetch(xdp->data_meta);
1851 
1852 	/* build an skb around the page buffer */
1853 	skb = napi_build_skb(xdp->data_hard_start, truesize);
1854 	if (unlikely(!skb))
1855 		return NULL;
1856 
1857 	/* update pointers within the skb to store the data */
1858 	skb_reserve(skb, xdp->data - xdp->data_hard_start);
1859 	__skb_put(skb, size);
1860 	if (metasize)
1861 		skb_metadata_set(skb, metasize);
1862 
1863 	igc_rx_buffer_flip(rx_buffer, truesize);
1864 	return skb;
1865 }
1866 
igc_construct_skb(struct igc_ring * rx_ring,struct igc_rx_buffer * rx_buffer,struct xdp_buff * xdp,ktime_t timestamp)1867 static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring,
1868 					 struct igc_rx_buffer *rx_buffer,
1869 					 struct xdp_buff *xdp,
1870 					 ktime_t timestamp)
1871 {
1872 	unsigned int metasize = xdp->data - xdp->data_meta;
1873 	unsigned int size = xdp->data_end - xdp->data;
1874 	unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size);
1875 	void *va = xdp->data;
1876 	unsigned int headlen;
1877 	struct sk_buff *skb;
1878 
1879 	/* prefetch first cache line of first page */
1880 	net_prefetch(xdp->data_meta);
1881 
1882 	/* allocate a skb to store the frags */
1883 	skb = napi_alloc_skb(&rx_ring->q_vector->napi,
1884 			     IGC_RX_HDR_LEN + metasize);
1885 	if (unlikely(!skb))
1886 		return NULL;
1887 
1888 	if (timestamp)
1889 		skb_hwtstamps(skb)->hwtstamp = timestamp;
1890 
1891 	/* Determine available headroom for copy */
1892 	headlen = size;
1893 	if (headlen > IGC_RX_HDR_LEN)
1894 		headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN);
1895 
1896 	/* align pull length to size of long to optimize memcpy performance */
1897 	memcpy(__skb_put(skb, headlen + metasize), xdp->data_meta,
1898 	       ALIGN(headlen + metasize, sizeof(long)));
1899 
1900 	if (metasize) {
1901 		skb_metadata_set(skb, metasize);
1902 		__skb_pull(skb, metasize);
1903 	}
1904 
1905 	/* update all of the pointers */
1906 	size -= headlen;
1907 	if (size) {
1908 		skb_add_rx_frag(skb, 0, rx_buffer->page,
1909 				(va + headlen) - page_address(rx_buffer->page),
1910 				size, truesize);
1911 		igc_rx_buffer_flip(rx_buffer, truesize);
1912 	} else {
1913 		rx_buffer->pagecnt_bias++;
1914 	}
1915 
1916 	return skb;
1917 }
1918 
1919 /**
1920  * igc_reuse_rx_page - page flip buffer and store it back on the ring
1921  * @rx_ring: rx descriptor ring to store buffers on
1922  * @old_buff: donor buffer to have page reused
1923  *
1924  * Synchronizes page for reuse by the adapter
1925  */
igc_reuse_rx_page(struct igc_ring * rx_ring,struct igc_rx_buffer * old_buff)1926 static void igc_reuse_rx_page(struct igc_ring *rx_ring,
1927 			      struct igc_rx_buffer *old_buff)
1928 {
1929 	u16 nta = rx_ring->next_to_alloc;
1930 	struct igc_rx_buffer *new_buff;
1931 
1932 	new_buff = &rx_ring->rx_buffer_info[nta];
1933 
1934 	/* update, and store next to alloc */
1935 	nta++;
1936 	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
1937 
1938 	/* Transfer page from old buffer to new buffer.
1939 	 * Move each member individually to avoid possible store
1940 	 * forwarding stalls.
1941 	 */
1942 	new_buff->dma		= old_buff->dma;
1943 	new_buff->page		= old_buff->page;
1944 	new_buff->page_offset	= old_buff->page_offset;
1945 	new_buff->pagecnt_bias	= old_buff->pagecnt_bias;
1946 }
1947 
igc_can_reuse_rx_page(struct igc_rx_buffer * rx_buffer,int rx_buffer_pgcnt)1948 static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer,
1949 				  int rx_buffer_pgcnt)
1950 {
1951 	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
1952 	struct page *page = rx_buffer->page;
1953 
1954 	/* avoid re-using remote and pfmemalloc pages */
1955 	if (!dev_page_is_reusable(page))
1956 		return false;
1957 
1958 #if (PAGE_SIZE < 8192)
1959 	/* if we are only owner of page we can reuse it */
1960 	if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1))
1961 		return false;
1962 #else
1963 #define IGC_LAST_OFFSET \
1964 	(SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048)
1965 
1966 	if (rx_buffer->page_offset > IGC_LAST_OFFSET)
1967 		return false;
1968 #endif
1969 
1970 	/* If we have drained the page fragment pool we need to update
1971 	 * the pagecnt_bias and page count so that we fully restock the
1972 	 * number of references the driver holds.
1973 	 */
1974 	if (unlikely(pagecnt_bias == 1)) {
1975 		page_ref_add(page, USHRT_MAX - 1);
1976 		rx_buffer->pagecnt_bias = USHRT_MAX;
1977 	}
1978 
1979 	return true;
1980 }
1981 
1982 /**
1983  * igc_is_non_eop - process handling of non-EOP buffers
1984  * @rx_ring: Rx ring being processed
1985  * @rx_desc: Rx descriptor for current buffer
1986  *
1987  * This function updates next to clean.  If the buffer is an EOP buffer
1988  * this function exits returning false, otherwise it will place the
1989  * sk_buff in the next buffer to be chained and return true indicating
1990  * that this is in fact a non-EOP buffer.
1991  */
igc_is_non_eop(struct igc_ring * rx_ring,union igc_adv_rx_desc * rx_desc)1992 static bool igc_is_non_eop(struct igc_ring *rx_ring,
1993 			   union igc_adv_rx_desc *rx_desc)
1994 {
1995 	u32 ntc = rx_ring->next_to_clean + 1;
1996 
1997 	/* fetch, update, and store next to clean */
1998 	ntc = (ntc < rx_ring->count) ? ntc : 0;
1999 	rx_ring->next_to_clean = ntc;
2000 
2001 	prefetch(IGC_RX_DESC(rx_ring, ntc));
2002 
2003 	if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP)))
2004 		return false;
2005 
2006 	return true;
2007 }
2008 
2009 /**
2010  * igc_cleanup_headers - Correct corrupted or empty headers
2011  * @rx_ring: rx descriptor ring packet is being transacted on
2012  * @rx_desc: pointer to the EOP Rx descriptor
2013  * @skb: pointer to current skb being fixed
2014  *
2015  * Address the case where we are pulling data in on pages only
2016  * and as such no data is present in the skb header.
2017  *
2018  * In addition if skb is not at least 60 bytes we need to pad it so that
2019  * it is large enough to qualify as a valid Ethernet frame.
2020  *
2021  * Returns true if an error was encountered and skb was freed.
2022  */
igc_cleanup_headers(struct igc_ring * rx_ring,union igc_adv_rx_desc * rx_desc,struct sk_buff * skb)2023 static bool igc_cleanup_headers(struct igc_ring *rx_ring,
2024 				union igc_adv_rx_desc *rx_desc,
2025 				struct sk_buff *skb)
2026 {
2027 	/* XDP packets use error pointer so abort at this point */
2028 	if (IS_ERR(skb))
2029 		return true;
2030 
2031 	if (unlikely(igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_RXE))) {
2032 		struct net_device *netdev = rx_ring->netdev;
2033 
2034 		if (!(netdev->features & NETIF_F_RXALL)) {
2035 			dev_kfree_skb_any(skb);
2036 			return true;
2037 		}
2038 	}
2039 
2040 	/* if eth_skb_pad returns an error the skb was freed */
2041 	if (eth_skb_pad(skb))
2042 		return true;
2043 
2044 	return false;
2045 }
2046 
igc_put_rx_buffer(struct igc_ring * rx_ring,struct igc_rx_buffer * rx_buffer,int rx_buffer_pgcnt)2047 static void igc_put_rx_buffer(struct igc_ring *rx_ring,
2048 			      struct igc_rx_buffer *rx_buffer,
2049 			      int rx_buffer_pgcnt)
2050 {
2051 	if (igc_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) {
2052 		/* hand second half of page back to the ring */
2053 		igc_reuse_rx_page(rx_ring, rx_buffer);
2054 	} else {
2055 		/* We are not reusing the buffer so unmap it and free
2056 		 * any references we are holding to it
2057 		 */
2058 		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
2059 				     igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
2060 				     IGC_RX_DMA_ATTR);
2061 		__page_frag_cache_drain(rx_buffer->page,
2062 					rx_buffer->pagecnt_bias);
2063 	}
2064 
2065 	/* clear contents of rx_buffer */
2066 	rx_buffer->page = NULL;
2067 }
2068 
igc_rx_offset(struct igc_ring * rx_ring)2069 static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring)
2070 {
2071 	struct igc_adapter *adapter = rx_ring->q_vector->adapter;
2072 
2073 	if (ring_uses_build_skb(rx_ring))
2074 		return IGC_SKB_PAD;
2075 	if (igc_xdp_is_enabled(adapter))
2076 		return XDP_PACKET_HEADROOM;
2077 
2078 	return 0;
2079 }
2080 
igc_alloc_mapped_page(struct igc_ring * rx_ring,struct igc_rx_buffer * bi)2081 static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
2082 				  struct igc_rx_buffer *bi)
2083 {
2084 	struct page *page = bi->page;
2085 	dma_addr_t dma;
2086 
2087 	/* since we are recycling buffers we should seldom need to alloc */
2088 	if (likely(page))
2089 		return true;
2090 
2091 	/* alloc new page for storage */
2092 	page = dev_alloc_pages(igc_rx_pg_order(rx_ring));
2093 	if (unlikely(!page)) {
2094 		rx_ring->rx_stats.alloc_failed++;
2095 		return false;
2096 	}
2097 
2098 	/* map page for use */
2099 	dma = dma_map_page_attrs(rx_ring->dev, page, 0,
2100 				 igc_rx_pg_size(rx_ring),
2101 				 DMA_FROM_DEVICE,
2102 				 IGC_RX_DMA_ATTR);
2103 
2104 	/* if mapping failed free memory back to system since
2105 	 * there isn't much point in holding memory we can't use
2106 	 */
2107 	if (dma_mapping_error(rx_ring->dev, dma)) {
2108 		__free_page(page);
2109 
2110 		rx_ring->rx_stats.alloc_failed++;
2111 		return false;
2112 	}
2113 
2114 	bi->dma = dma;
2115 	bi->page = page;
2116 	bi->page_offset = igc_rx_offset(rx_ring);
2117 	page_ref_add(page, USHRT_MAX - 1);
2118 	bi->pagecnt_bias = USHRT_MAX;
2119 
2120 	return true;
2121 }
2122 
2123 /**
2124  * igc_alloc_rx_buffers - Replace used receive buffers; packet split
2125  * @rx_ring: rx descriptor ring
2126  * @cleaned_count: number of buffers to clean
2127  */
igc_alloc_rx_buffers(struct igc_ring * rx_ring,u16 cleaned_count)2128 static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count)
2129 {
2130 	union igc_adv_rx_desc *rx_desc;
2131 	u16 i = rx_ring->next_to_use;
2132 	struct igc_rx_buffer *bi;
2133 	u16 bufsz;
2134 
2135 	/* nothing to do */
2136 	if (!cleaned_count)
2137 		return;
2138 
2139 	rx_desc = IGC_RX_DESC(rx_ring, i);
2140 	bi = &rx_ring->rx_buffer_info[i];
2141 	i -= rx_ring->count;
2142 
2143 	bufsz = igc_rx_bufsz(rx_ring);
2144 
2145 	do {
2146 		if (!igc_alloc_mapped_page(rx_ring, bi))
2147 			break;
2148 
2149 		/* sync the buffer for use by the device */
2150 		dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
2151 						 bi->page_offset, bufsz,
2152 						 DMA_FROM_DEVICE);
2153 
2154 		/* Refresh the desc even if buffer_addrs didn't change
2155 		 * because each write-back erases this info.
2156 		 */
2157 		rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
2158 
2159 		rx_desc++;
2160 		bi++;
2161 		i++;
2162 		if (unlikely(!i)) {
2163 			rx_desc = IGC_RX_DESC(rx_ring, 0);
2164 			bi = rx_ring->rx_buffer_info;
2165 			i -= rx_ring->count;
2166 		}
2167 
2168 		/* clear the length for the next_to_use descriptor */
2169 		rx_desc->wb.upper.length = 0;
2170 
2171 		cleaned_count--;
2172 	} while (cleaned_count);
2173 
2174 	i += rx_ring->count;
2175 
2176 	if (rx_ring->next_to_use != i) {
2177 		/* record the next descriptor to use */
2178 		rx_ring->next_to_use = i;
2179 
2180 		/* update next to alloc since we have filled the ring */
2181 		rx_ring->next_to_alloc = i;
2182 
2183 		/* Force memory writes to complete before letting h/w
2184 		 * know there are new descriptors to fetch.  (Only
2185 		 * applicable for weak-ordered memory model archs,
2186 		 * such as IA-64).
2187 		 */
2188 		wmb();
2189 		writel(i, rx_ring->tail);
2190 	}
2191 }
2192 
igc_alloc_rx_buffers_zc(struct igc_ring * ring,u16 count)2193 static bool igc_alloc_rx_buffers_zc(struct igc_ring *ring, u16 count)
2194 {
2195 	union igc_adv_rx_desc *desc;
2196 	u16 i = ring->next_to_use;
2197 	struct igc_rx_buffer *bi;
2198 	dma_addr_t dma;
2199 	bool ok = true;
2200 
2201 	if (!count)
2202 		return ok;
2203 
2204 	desc = IGC_RX_DESC(ring, i);
2205 	bi = &ring->rx_buffer_info[i];
2206 	i -= ring->count;
2207 
2208 	do {
2209 		bi->xdp = xsk_buff_alloc(ring->xsk_pool);
2210 		if (!bi->xdp) {
2211 			ok = false;
2212 			break;
2213 		}
2214 
2215 		dma = xsk_buff_xdp_get_dma(bi->xdp);
2216 		desc->read.pkt_addr = cpu_to_le64(dma);
2217 
2218 		desc++;
2219 		bi++;
2220 		i++;
2221 		if (unlikely(!i)) {
2222 			desc = IGC_RX_DESC(ring, 0);
2223 			bi = ring->rx_buffer_info;
2224 			i -= ring->count;
2225 		}
2226 
2227 		/* Clear the length for the next_to_use descriptor. */
2228 		desc->wb.upper.length = 0;
2229 
2230 		count--;
2231 	} while (count);
2232 
2233 	i += ring->count;
2234 
2235 	if (ring->next_to_use != i) {
2236 		ring->next_to_use = i;
2237 
2238 		/* Force memory writes to complete before letting h/w
2239 		 * know there are new descriptors to fetch.  (Only
2240 		 * applicable for weak-ordered memory model archs,
2241 		 * such as IA-64).
2242 		 */
2243 		wmb();
2244 		writel(i, ring->tail);
2245 	}
2246 
2247 	return ok;
2248 }
2249 
2250 /* This function requires __netif_tx_lock is held by the caller. */
igc_xdp_init_tx_descriptor(struct igc_ring * ring,struct xdp_frame * xdpf)2251 static int igc_xdp_init_tx_descriptor(struct igc_ring *ring,
2252 				      struct xdp_frame *xdpf)
2253 {
2254 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
2255 	u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
2256 	u16 count, index = ring->next_to_use;
2257 	struct igc_tx_buffer *head = &ring->tx_buffer_info[index];
2258 	struct igc_tx_buffer *buffer = head;
2259 	union igc_adv_tx_desc *desc = IGC_TX_DESC(ring, index);
2260 	u32 olinfo_status, len = xdpf->len, cmd_type;
2261 	void *data = xdpf->data;
2262 	u16 i;
2263 
2264 	count = TXD_USE_COUNT(len);
2265 	for (i = 0; i < nr_frags; i++)
2266 		count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i]));
2267 
2268 	if (igc_maybe_stop_tx(ring, count + 3)) {
2269 		/* this is a hard error */
2270 		return -EBUSY;
2271 	}
2272 
2273 	i = 0;
2274 	head->bytecount = xdp_get_frame_len(xdpf);
2275 	head->type = IGC_TX_BUFFER_TYPE_XDP;
2276 	head->gso_segs = 1;
2277 	head->xdpf = xdpf;
2278 
2279 	olinfo_status = head->bytecount << IGC_ADVTXD_PAYLEN_SHIFT;
2280 	desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2281 
2282 	for (;;) {
2283 		dma_addr_t dma;
2284 
2285 		dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE);
2286 		if (dma_mapping_error(ring->dev, dma)) {
2287 			netdev_err_once(ring->netdev,
2288 					"Failed to map DMA for TX\n");
2289 			goto unmap;
2290 		}
2291 
2292 		dma_unmap_len_set(buffer, len, len);
2293 		dma_unmap_addr_set(buffer, dma, dma);
2294 
2295 		cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
2296 			   IGC_ADVTXD_DCMD_IFCS | len;
2297 
2298 		desc->read.cmd_type_len = cpu_to_le32(cmd_type);
2299 		desc->read.buffer_addr = cpu_to_le64(dma);
2300 
2301 		buffer->protocol = 0;
2302 
2303 		if (++index == ring->count)
2304 			index = 0;
2305 
2306 		if (i == nr_frags)
2307 			break;
2308 
2309 		buffer = &ring->tx_buffer_info[index];
2310 		desc = IGC_TX_DESC(ring, index);
2311 		desc->read.olinfo_status = 0;
2312 
2313 		data = skb_frag_address(&sinfo->frags[i]);
2314 		len = skb_frag_size(&sinfo->frags[i]);
2315 		i++;
2316 	}
2317 	desc->read.cmd_type_len |= cpu_to_le32(IGC_TXD_DCMD);
2318 
2319 	netdev_tx_sent_queue(txring_txq(ring), head->bytecount);
2320 	/* set the timestamp */
2321 	head->time_stamp = jiffies;
2322 	/* set next_to_watch value indicating a packet is present */
2323 	head->next_to_watch = desc;
2324 	ring->next_to_use = index;
2325 
2326 	return 0;
2327 
2328 unmap:
2329 	for (;;) {
2330 		buffer = &ring->tx_buffer_info[index];
2331 		if (dma_unmap_len(buffer, len))
2332 			dma_unmap_page(ring->dev,
2333 				       dma_unmap_addr(buffer, dma),
2334 				       dma_unmap_len(buffer, len),
2335 				       DMA_TO_DEVICE);
2336 		dma_unmap_len_set(buffer, len, 0);
2337 		if (buffer == head)
2338 			break;
2339 
2340 		if (!index)
2341 			index += ring->count;
2342 		index--;
2343 	}
2344 
2345 	return -ENOMEM;
2346 }
2347 
igc_xdp_get_tx_ring(struct igc_adapter * adapter,int cpu)2348 static struct igc_ring *igc_xdp_get_tx_ring(struct igc_adapter *adapter,
2349 					    int cpu)
2350 {
2351 	int index = cpu;
2352 
2353 	if (unlikely(index < 0))
2354 		index = 0;
2355 
2356 	while (index >= adapter->num_tx_queues)
2357 		index -= adapter->num_tx_queues;
2358 
2359 	return adapter->tx_ring[index];
2360 }
2361 
igc_xdp_xmit_back(struct igc_adapter * adapter,struct xdp_buff * xdp)2362 static int igc_xdp_xmit_back(struct igc_adapter *adapter, struct xdp_buff *xdp)
2363 {
2364 	struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
2365 	int cpu = smp_processor_id();
2366 	struct netdev_queue *nq;
2367 	struct igc_ring *ring;
2368 	int res;
2369 
2370 	if (unlikely(!xdpf))
2371 		return -EFAULT;
2372 
2373 	ring = igc_xdp_get_tx_ring(adapter, cpu);
2374 	nq = txring_txq(ring);
2375 
2376 	__netif_tx_lock(nq, cpu);
2377 	res = igc_xdp_init_tx_descriptor(ring, xdpf);
2378 	__netif_tx_unlock(nq);
2379 	return res;
2380 }
2381 
2382 /* This function assumes rcu_read_lock() is held by the caller. */
__igc_xdp_run_prog(struct igc_adapter * adapter,struct bpf_prog * prog,struct xdp_buff * xdp)2383 static int __igc_xdp_run_prog(struct igc_adapter *adapter,
2384 			      struct bpf_prog *prog,
2385 			      struct xdp_buff *xdp)
2386 {
2387 	u32 act = bpf_prog_run_xdp(prog, xdp);
2388 
2389 	switch (act) {
2390 	case XDP_PASS:
2391 		return IGC_XDP_PASS;
2392 	case XDP_TX:
2393 		if (igc_xdp_xmit_back(adapter, xdp) < 0)
2394 			goto out_failure;
2395 		return IGC_XDP_TX;
2396 	case XDP_REDIRECT:
2397 		if (xdp_do_redirect(adapter->netdev, xdp, prog) < 0)
2398 			goto out_failure;
2399 		return IGC_XDP_REDIRECT;
2400 		break;
2401 	default:
2402 		bpf_warn_invalid_xdp_action(adapter->netdev, prog, act);
2403 		fallthrough;
2404 	case XDP_ABORTED:
2405 out_failure:
2406 		trace_xdp_exception(adapter->netdev, prog, act);
2407 		fallthrough;
2408 	case XDP_DROP:
2409 		return IGC_XDP_CONSUMED;
2410 	}
2411 }
2412 
igc_xdp_run_prog(struct igc_adapter * adapter,struct xdp_buff * xdp)2413 static struct sk_buff *igc_xdp_run_prog(struct igc_adapter *adapter,
2414 					struct xdp_buff *xdp)
2415 {
2416 	struct bpf_prog *prog;
2417 	int res;
2418 
2419 	prog = READ_ONCE(adapter->xdp_prog);
2420 	if (!prog) {
2421 		res = IGC_XDP_PASS;
2422 		goto out;
2423 	}
2424 
2425 	res = __igc_xdp_run_prog(adapter, prog, xdp);
2426 
2427 out:
2428 	return ERR_PTR(-res);
2429 }
2430 
2431 /* This function assumes __netif_tx_lock is held by the caller. */
igc_flush_tx_descriptors(struct igc_ring * ring)2432 static void igc_flush_tx_descriptors(struct igc_ring *ring)
2433 {
2434 	/* Once tail pointer is updated, hardware can fetch the descriptors
2435 	 * any time so we issue a write membar here to ensure all memory
2436 	 * writes are complete before the tail pointer is updated.
2437 	 */
2438 	wmb();
2439 	writel(ring->next_to_use, ring->tail);
2440 }
2441 
igc_finalize_xdp(struct igc_adapter * adapter,int status)2442 static void igc_finalize_xdp(struct igc_adapter *adapter, int status)
2443 {
2444 	int cpu = smp_processor_id();
2445 	struct netdev_queue *nq;
2446 	struct igc_ring *ring;
2447 
2448 	if (status & IGC_XDP_TX) {
2449 		ring = igc_xdp_get_tx_ring(adapter, cpu);
2450 		nq = txring_txq(ring);
2451 
2452 		__netif_tx_lock(nq, cpu);
2453 		igc_flush_tx_descriptors(ring);
2454 		__netif_tx_unlock(nq);
2455 	}
2456 
2457 	if (status & IGC_XDP_REDIRECT)
2458 		xdp_do_flush();
2459 }
2460 
igc_update_rx_stats(struct igc_q_vector * q_vector,unsigned int packets,unsigned int bytes)2461 static void igc_update_rx_stats(struct igc_q_vector *q_vector,
2462 				unsigned int packets, unsigned int bytes)
2463 {
2464 	struct igc_ring *ring = q_vector->rx.ring;
2465 
2466 	u64_stats_update_begin(&ring->rx_syncp);
2467 	ring->rx_stats.packets += packets;
2468 	ring->rx_stats.bytes += bytes;
2469 	u64_stats_update_end(&ring->rx_syncp);
2470 
2471 	q_vector->rx.total_packets += packets;
2472 	q_vector->rx.total_bytes += bytes;
2473 }
2474 
igc_clean_rx_irq(struct igc_q_vector * q_vector,const int budget)2475 static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
2476 {
2477 	unsigned int total_bytes = 0, total_packets = 0;
2478 	struct igc_adapter *adapter = q_vector->adapter;
2479 	struct igc_ring *rx_ring = q_vector->rx.ring;
2480 	struct sk_buff *skb = rx_ring->skb;
2481 	u16 cleaned_count = igc_desc_unused(rx_ring);
2482 	int xdp_status = 0, rx_buffer_pgcnt;
2483 
2484 	while (likely(total_packets < budget)) {
2485 		union igc_adv_rx_desc *rx_desc;
2486 		struct igc_rx_buffer *rx_buffer;
2487 		unsigned int size, truesize;
2488 		ktime_t timestamp = 0;
2489 		struct xdp_buff xdp;
2490 		int pkt_offset = 0;
2491 		void *pktbuf;
2492 
2493 		/* return some buffers to hardware, one at a time is too slow */
2494 		if (cleaned_count >= IGC_RX_BUFFER_WRITE) {
2495 			igc_alloc_rx_buffers(rx_ring, cleaned_count);
2496 			cleaned_count = 0;
2497 		}
2498 
2499 		rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean);
2500 		size = le16_to_cpu(rx_desc->wb.upper.length);
2501 		if (!size)
2502 			break;
2503 
2504 		/* This memory barrier is needed to keep us from reading
2505 		 * any other fields out of the rx_desc until we know the
2506 		 * descriptor has been written back
2507 		 */
2508 		dma_rmb();
2509 
2510 		rx_buffer = igc_get_rx_buffer(rx_ring, size, &rx_buffer_pgcnt);
2511 		truesize = igc_get_rx_frame_truesize(rx_ring, size);
2512 
2513 		pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset;
2514 
2515 		if (igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP)) {
2516 			timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,
2517 							pktbuf);
2518 			pkt_offset = IGC_TS_HDR_LEN;
2519 			size -= IGC_TS_HDR_LEN;
2520 		}
2521 
2522 		if (!skb) {
2523 			xdp_init_buff(&xdp, truesize, &rx_ring->xdp_rxq);
2524 			xdp_prepare_buff(&xdp, pktbuf - igc_rx_offset(rx_ring),
2525 					 igc_rx_offset(rx_ring) + pkt_offset,
2526 					 size, true);
2527 			xdp_buff_clear_frags_flag(&xdp);
2528 
2529 			skb = igc_xdp_run_prog(adapter, &xdp);
2530 		}
2531 
2532 		if (IS_ERR(skb)) {
2533 			unsigned int xdp_res = -PTR_ERR(skb);
2534 
2535 			switch (xdp_res) {
2536 			case IGC_XDP_CONSUMED:
2537 				rx_buffer->pagecnt_bias++;
2538 				break;
2539 			case IGC_XDP_TX:
2540 			case IGC_XDP_REDIRECT:
2541 				igc_rx_buffer_flip(rx_buffer, truesize);
2542 				xdp_status |= xdp_res;
2543 				break;
2544 			}
2545 
2546 			total_packets++;
2547 			total_bytes += size;
2548 		} else if (skb)
2549 			igc_add_rx_frag(rx_ring, rx_buffer, skb, size);
2550 		else if (ring_uses_build_skb(rx_ring))
2551 			skb = igc_build_skb(rx_ring, rx_buffer, &xdp);
2552 		else
2553 			skb = igc_construct_skb(rx_ring, rx_buffer, &xdp,
2554 						timestamp);
2555 
2556 		/* exit if we failed to retrieve a buffer */
2557 		if (!skb) {
2558 			rx_ring->rx_stats.alloc_failed++;
2559 			rx_buffer->pagecnt_bias++;
2560 			break;
2561 		}
2562 
2563 		igc_put_rx_buffer(rx_ring, rx_buffer, rx_buffer_pgcnt);
2564 		cleaned_count++;
2565 
2566 		/* fetch next buffer in frame if non-eop */
2567 		if (igc_is_non_eop(rx_ring, rx_desc))
2568 			continue;
2569 
2570 		/* verify the packet layout is correct */
2571 		if (igc_cleanup_headers(rx_ring, rx_desc, skb)) {
2572 			skb = NULL;
2573 			continue;
2574 		}
2575 
2576 		/* probably a little skewed due to removing CRC */
2577 		total_bytes += skb->len;
2578 
2579 		/* populate checksum, VLAN, and protocol */
2580 		igc_process_skb_fields(rx_ring, rx_desc, skb);
2581 
2582 		napi_gro_receive(&q_vector->napi, skb);
2583 
2584 		/* reset skb pointer */
2585 		skb = NULL;
2586 
2587 		/* update budget accounting */
2588 		total_packets++;
2589 	}
2590 
2591 	if (xdp_status)
2592 		igc_finalize_xdp(adapter, xdp_status);
2593 
2594 	/* place incomplete frames back on ring for completion */
2595 	rx_ring->skb = skb;
2596 
2597 	igc_update_rx_stats(q_vector, total_packets, total_bytes);
2598 
2599 	if (cleaned_count)
2600 		igc_alloc_rx_buffers(rx_ring, cleaned_count);
2601 
2602 	return total_packets;
2603 }
2604 
igc_construct_skb_zc(struct igc_ring * ring,struct xdp_buff * xdp)2605 static struct sk_buff *igc_construct_skb_zc(struct igc_ring *ring,
2606 					    struct xdp_buff *xdp)
2607 {
2608 	unsigned int totalsize = xdp->data_end - xdp->data_meta;
2609 	unsigned int metasize = xdp->data - xdp->data_meta;
2610 	struct sk_buff *skb;
2611 
2612 	net_prefetch(xdp->data_meta);
2613 
2614 	skb = __napi_alloc_skb(&ring->q_vector->napi, totalsize,
2615 			       GFP_ATOMIC | __GFP_NOWARN);
2616 	if (unlikely(!skb))
2617 		return NULL;
2618 
2619 	memcpy(__skb_put(skb, totalsize), xdp->data_meta,
2620 	       ALIGN(totalsize, sizeof(long)));
2621 
2622 	if (metasize) {
2623 		skb_metadata_set(skb, metasize);
2624 		__skb_pull(skb, metasize);
2625 	}
2626 
2627 	return skb;
2628 }
2629 
igc_dispatch_skb_zc(struct igc_q_vector * q_vector,union igc_adv_rx_desc * desc,struct xdp_buff * xdp,ktime_t timestamp)2630 static void igc_dispatch_skb_zc(struct igc_q_vector *q_vector,
2631 				union igc_adv_rx_desc *desc,
2632 				struct xdp_buff *xdp,
2633 				ktime_t timestamp)
2634 {
2635 	struct igc_ring *ring = q_vector->rx.ring;
2636 	struct sk_buff *skb;
2637 
2638 	skb = igc_construct_skb_zc(ring, xdp);
2639 	if (!skb) {
2640 		ring->rx_stats.alloc_failed++;
2641 		return;
2642 	}
2643 
2644 	if (timestamp)
2645 		skb_hwtstamps(skb)->hwtstamp = timestamp;
2646 
2647 	if (igc_cleanup_headers(ring, desc, skb))
2648 		return;
2649 
2650 	igc_process_skb_fields(ring, desc, skb);
2651 	napi_gro_receive(&q_vector->napi, skb);
2652 }
2653 
igc_clean_rx_irq_zc(struct igc_q_vector * q_vector,const int budget)2654 static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget)
2655 {
2656 	struct igc_adapter *adapter = q_vector->adapter;
2657 	struct igc_ring *ring = q_vector->rx.ring;
2658 	u16 cleaned_count = igc_desc_unused(ring);
2659 	int total_bytes = 0, total_packets = 0;
2660 	u16 ntc = ring->next_to_clean;
2661 	struct bpf_prog *prog;
2662 	bool failure = false;
2663 	int xdp_status = 0;
2664 
2665 	rcu_read_lock();
2666 
2667 	prog = READ_ONCE(adapter->xdp_prog);
2668 
2669 	while (likely(total_packets < budget)) {
2670 		union igc_adv_rx_desc *desc;
2671 		struct igc_rx_buffer *bi;
2672 		ktime_t timestamp = 0;
2673 		unsigned int size;
2674 		int res;
2675 
2676 		desc = IGC_RX_DESC(ring, ntc);
2677 		size = le16_to_cpu(desc->wb.upper.length);
2678 		if (!size)
2679 			break;
2680 
2681 		/* This memory barrier is needed to keep us from reading
2682 		 * any other fields out of the rx_desc until we know the
2683 		 * descriptor has been written back
2684 		 */
2685 		dma_rmb();
2686 
2687 		bi = &ring->rx_buffer_info[ntc];
2688 
2689 		if (igc_test_staterr(desc, IGC_RXDADV_STAT_TSIP)) {
2690 			timestamp = igc_ptp_rx_pktstamp(q_vector->adapter,
2691 							bi->xdp->data);
2692 
2693 			bi->xdp->data += IGC_TS_HDR_LEN;
2694 
2695 			/* HW timestamp has been copied into local variable. Metadata
2696 			 * length when XDP program is called should be 0.
2697 			 */
2698 			bi->xdp->data_meta += IGC_TS_HDR_LEN;
2699 			size -= IGC_TS_HDR_LEN;
2700 		}
2701 
2702 		bi->xdp->data_end = bi->xdp->data + size;
2703 		xsk_buff_dma_sync_for_cpu(bi->xdp, ring->xsk_pool);
2704 
2705 		res = __igc_xdp_run_prog(adapter, prog, bi->xdp);
2706 		switch (res) {
2707 		case IGC_XDP_PASS:
2708 			igc_dispatch_skb_zc(q_vector, desc, bi->xdp, timestamp);
2709 			fallthrough;
2710 		case IGC_XDP_CONSUMED:
2711 			xsk_buff_free(bi->xdp);
2712 			break;
2713 		case IGC_XDP_TX:
2714 		case IGC_XDP_REDIRECT:
2715 			xdp_status |= res;
2716 			break;
2717 		}
2718 
2719 		bi->xdp = NULL;
2720 		total_bytes += size;
2721 		total_packets++;
2722 		cleaned_count++;
2723 		ntc++;
2724 		if (ntc == ring->count)
2725 			ntc = 0;
2726 	}
2727 
2728 	ring->next_to_clean = ntc;
2729 	rcu_read_unlock();
2730 
2731 	if (cleaned_count >= IGC_RX_BUFFER_WRITE)
2732 		failure = !igc_alloc_rx_buffers_zc(ring, cleaned_count);
2733 
2734 	if (xdp_status)
2735 		igc_finalize_xdp(adapter, xdp_status);
2736 
2737 	igc_update_rx_stats(q_vector, total_packets, total_bytes);
2738 
2739 	if (xsk_uses_need_wakeup(ring->xsk_pool)) {
2740 		if (failure || ring->next_to_clean == ring->next_to_use)
2741 			xsk_set_rx_need_wakeup(ring->xsk_pool);
2742 		else
2743 			xsk_clear_rx_need_wakeup(ring->xsk_pool);
2744 		return total_packets;
2745 	}
2746 
2747 	return failure ? budget : total_packets;
2748 }
2749 
igc_update_tx_stats(struct igc_q_vector * q_vector,unsigned int packets,unsigned int bytes)2750 static void igc_update_tx_stats(struct igc_q_vector *q_vector,
2751 				unsigned int packets, unsigned int bytes)
2752 {
2753 	struct igc_ring *ring = q_vector->tx.ring;
2754 
2755 	u64_stats_update_begin(&ring->tx_syncp);
2756 	ring->tx_stats.bytes += bytes;
2757 	ring->tx_stats.packets += packets;
2758 	u64_stats_update_end(&ring->tx_syncp);
2759 
2760 	q_vector->tx.total_bytes += bytes;
2761 	q_vector->tx.total_packets += packets;
2762 }
2763 
igc_xdp_xmit_zc(struct igc_ring * ring)2764 static void igc_xdp_xmit_zc(struct igc_ring *ring)
2765 {
2766 	struct xsk_buff_pool *pool = ring->xsk_pool;
2767 	struct netdev_queue *nq = txring_txq(ring);
2768 	union igc_adv_tx_desc *tx_desc = NULL;
2769 	int cpu = smp_processor_id();
2770 	u16 ntu = ring->next_to_use;
2771 	struct xdp_desc xdp_desc;
2772 	u16 budget;
2773 
2774 	if (!netif_carrier_ok(ring->netdev))
2775 		return;
2776 
2777 	__netif_tx_lock(nq, cpu);
2778 
2779 	budget = igc_desc_unused(ring);
2780 
2781 	while (xsk_tx_peek_desc(pool, &xdp_desc) && budget--) {
2782 		u32 cmd_type, olinfo_status;
2783 		struct igc_tx_buffer *bi;
2784 		dma_addr_t dma;
2785 
2786 		cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
2787 			   IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD |
2788 			   xdp_desc.len;
2789 		olinfo_status = xdp_desc.len << IGC_ADVTXD_PAYLEN_SHIFT;
2790 
2791 		dma = xsk_buff_raw_get_dma(pool, xdp_desc.addr);
2792 		xsk_buff_raw_dma_sync_for_device(pool, dma, xdp_desc.len);
2793 
2794 		tx_desc = IGC_TX_DESC(ring, ntu);
2795 		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
2796 		tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2797 		tx_desc->read.buffer_addr = cpu_to_le64(dma);
2798 
2799 		bi = &ring->tx_buffer_info[ntu];
2800 		bi->type = IGC_TX_BUFFER_TYPE_XSK;
2801 		bi->protocol = 0;
2802 		bi->bytecount = xdp_desc.len;
2803 		bi->gso_segs = 1;
2804 		bi->time_stamp = jiffies;
2805 		bi->next_to_watch = tx_desc;
2806 
2807 		netdev_tx_sent_queue(txring_txq(ring), xdp_desc.len);
2808 
2809 		ntu++;
2810 		if (ntu == ring->count)
2811 			ntu = 0;
2812 	}
2813 
2814 	ring->next_to_use = ntu;
2815 	if (tx_desc) {
2816 		igc_flush_tx_descriptors(ring);
2817 		xsk_tx_release(pool);
2818 	}
2819 
2820 	__netif_tx_unlock(nq);
2821 }
2822 
2823 /**
2824  * igc_clean_tx_irq - Reclaim resources after transmit completes
2825  * @q_vector: pointer to q_vector containing needed info
2826  * @napi_budget: Used to determine if we are in netpoll
2827  *
2828  * returns true if ring is completely cleaned
2829  */
igc_clean_tx_irq(struct igc_q_vector * q_vector,int napi_budget)2830 static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget)
2831 {
2832 	struct igc_adapter *adapter = q_vector->adapter;
2833 	unsigned int total_bytes = 0, total_packets = 0;
2834 	unsigned int budget = q_vector->tx.work_limit;
2835 	struct igc_ring *tx_ring = q_vector->tx.ring;
2836 	unsigned int i = tx_ring->next_to_clean;
2837 	struct igc_tx_buffer *tx_buffer;
2838 	union igc_adv_tx_desc *tx_desc;
2839 	u32 xsk_frames = 0;
2840 
2841 	if (test_bit(__IGC_DOWN, &adapter->state))
2842 		return true;
2843 
2844 	tx_buffer = &tx_ring->tx_buffer_info[i];
2845 	tx_desc = IGC_TX_DESC(tx_ring, i);
2846 	i -= tx_ring->count;
2847 
2848 	do {
2849 		union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
2850 
2851 		/* if next_to_watch is not set then there is no work pending */
2852 		if (!eop_desc)
2853 			break;
2854 
2855 		/* prevent any other reads prior to eop_desc */
2856 		smp_rmb();
2857 
2858 		/* if DD is not set pending work has not been completed */
2859 		if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD)))
2860 			break;
2861 
2862 		/* clear next_to_watch to prevent false hangs */
2863 		tx_buffer->next_to_watch = NULL;
2864 
2865 		/* update the statistics for this packet */
2866 		total_bytes += tx_buffer->bytecount;
2867 		total_packets += tx_buffer->gso_segs;
2868 
2869 		switch (tx_buffer->type) {
2870 		case IGC_TX_BUFFER_TYPE_XSK:
2871 			xsk_frames++;
2872 			break;
2873 		case IGC_TX_BUFFER_TYPE_XDP:
2874 			xdp_return_frame(tx_buffer->xdpf);
2875 			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
2876 			break;
2877 		case IGC_TX_BUFFER_TYPE_SKB:
2878 			napi_consume_skb(tx_buffer->skb, napi_budget);
2879 			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
2880 			break;
2881 		default:
2882 			netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n");
2883 			break;
2884 		}
2885 
2886 		/* clear last DMA location and unmap remaining buffers */
2887 		while (tx_desc != eop_desc) {
2888 			tx_buffer++;
2889 			tx_desc++;
2890 			i++;
2891 			if (unlikely(!i)) {
2892 				i -= tx_ring->count;
2893 				tx_buffer = tx_ring->tx_buffer_info;
2894 				tx_desc = IGC_TX_DESC(tx_ring, 0);
2895 			}
2896 
2897 			/* unmap any remaining paged data */
2898 			if (dma_unmap_len(tx_buffer, len))
2899 				igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
2900 		}
2901 
2902 		/* move us one more past the eop_desc for start of next pkt */
2903 		tx_buffer++;
2904 		tx_desc++;
2905 		i++;
2906 		if (unlikely(!i)) {
2907 			i -= tx_ring->count;
2908 			tx_buffer = tx_ring->tx_buffer_info;
2909 			tx_desc = IGC_TX_DESC(tx_ring, 0);
2910 		}
2911 
2912 		/* issue prefetch for next Tx descriptor */
2913 		prefetch(tx_desc);
2914 
2915 		/* update budget accounting */
2916 		budget--;
2917 	} while (likely(budget));
2918 
2919 	netdev_tx_completed_queue(txring_txq(tx_ring),
2920 				  total_packets, total_bytes);
2921 
2922 	i += tx_ring->count;
2923 	tx_ring->next_to_clean = i;
2924 
2925 	igc_update_tx_stats(q_vector, total_packets, total_bytes);
2926 
2927 	if (tx_ring->xsk_pool) {
2928 		if (xsk_frames)
2929 			xsk_tx_completed(tx_ring->xsk_pool, xsk_frames);
2930 		if (xsk_uses_need_wakeup(tx_ring->xsk_pool))
2931 			xsk_set_tx_need_wakeup(tx_ring->xsk_pool);
2932 		igc_xdp_xmit_zc(tx_ring);
2933 	}
2934 
2935 	if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
2936 		struct igc_hw *hw = &adapter->hw;
2937 
2938 		/* Detect a transmit hang in hardware, this serializes the
2939 		 * check with the clearing of time_stamp and movement of i
2940 		 */
2941 		clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
2942 		if (tx_buffer->next_to_watch &&
2943 		    time_after(jiffies, tx_buffer->time_stamp +
2944 		    (adapter->tx_timeout_factor * HZ)) &&
2945 		    !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF)) {
2946 			/* detected Tx unit hang */
2947 			netdev_err(tx_ring->netdev,
2948 				   "Detected Tx Unit Hang\n"
2949 				   "  Tx Queue             <%d>\n"
2950 				   "  TDH                  <%x>\n"
2951 				   "  TDT                  <%x>\n"
2952 				   "  next_to_use          <%x>\n"
2953 				   "  next_to_clean        <%x>\n"
2954 				   "buffer_info[next_to_clean]\n"
2955 				   "  time_stamp           <%lx>\n"
2956 				   "  next_to_watch        <%p>\n"
2957 				   "  jiffies              <%lx>\n"
2958 				   "  desc.status          <%x>\n",
2959 				   tx_ring->queue_index,
2960 				   rd32(IGC_TDH(tx_ring->reg_idx)),
2961 				   readl(tx_ring->tail),
2962 				   tx_ring->next_to_use,
2963 				   tx_ring->next_to_clean,
2964 				   tx_buffer->time_stamp,
2965 				   tx_buffer->next_to_watch,
2966 				   jiffies,
2967 				   tx_buffer->next_to_watch->wb.status);
2968 			netif_stop_subqueue(tx_ring->netdev,
2969 					    tx_ring->queue_index);
2970 
2971 			/* we are about to reset, no point in enabling stuff */
2972 			return true;
2973 		}
2974 	}
2975 
2976 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
2977 	if (unlikely(total_packets &&
2978 		     netif_carrier_ok(tx_ring->netdev) &&
2979 		     igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
2980 		/* Make sure that anybody stopping the queue after this
2981 		 * sees the new next_to_clean.
2982 		 */
2983 		smp_mb();
2984 		if (__netif_subqueue_stopped(tx_ring->netdev,
2985 					     tx_ring->queue_index) &&
2986 		    !(test_bit(__IGC_DOWN, &adapter->state))) {
2987 			netif_wake_subqueue(tx_ring->netdev,
2988 					    tx_ring->queue_index);
2989 
2990 			u64_stats_update_begin(&tx_ring->tx_syncp);
2991 			tx_ring->tx_stats.restart_queue++;
2992 			u64_stats_update_end(&tx_ring->tx_syncp);
2993 		}
2994 	}
2995 
2996 	return !!budget;
2997 }
2998 
igc_find_mac_filter(struct igc_adapter * adapter,enum igc_mac_filter_type type,const u8 * addr)2999 static int igc_find_mac_filter(struct igc_adapter *adapter,
3000 			       enum igc_mac_filter_type type, const u8 *addr)
3001 {
3002 	struct igc_hw *hw = &adapter->hw;
3003 	int max_entries = hw->mac.rar_entry_count;
3004 	u32 ral, rah;
3005 	int i;
3006 
3007 	for (i = 0; i < max_entries; i++) {
3008 		ral = rd32(IGC_RAL(i));
3009 		rah = rd32(IGC_RAH(i));
3010 
3011 		if (!(rah & IGC_RAH_AV))
3012 			continue;
3013 		if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type)
3014 			continue;
3015 		if ((rah & IGC_RAH_RAH_MASK) !=
3016 		    le16_to_cpup((__le16 *)(addr + 4)))
3017 			continue;
3018 		if (ral != le32_to_cpup((__le32 *)(addr)))
3019 			continue;
3020 
3021 		return i;
3022 	}
3023 
3024 	return -1;
3025 }
3026 
igc_get_avail_mac_filter_slot(struct igc_adapter * adapter)3027 static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter)
3028 {
3029 	struct igc_hw *hw = &adapter->hw;
3030 	int max_entries = hw->mac.rar_entry_count;
3031 	u32 rah;
3032 	int i;
3033 
3034 	for (i = 0; i < max_entries; i++) {
3035 		rah = rd32(IGC_RAH(i));
3036 
3037 		if (!(rah & IGC_RAH_AV))
3038 			return i;
3039 	}
3040 
3041 	return -1;
3042 }
3043 
3044 /**
3045  * igc_add_mac_filter() - Add MAC address filter
3046  * @adapter: Pointer to adapter where the filter should be added
3047  * @type: MAC address filter type (source or destination)
3048  * @addr: MAC address
3049  * @queue: If non-negative, queue assignment feature is enabled and frames
3050  *         matching the filter are enqueued onto 'queue'. Otherwise, queue
3051  *         assignment is disabled.
3052  *
3053  * Return: 0 in case of success, negative errno code otherwise.
3054  */
igc_add_mac_filter(struct igc_adapter * adapter,enum igc_mac_filter_type type,const u8 * addr,int queue)3055 static int igc_add_mac_filter(struct igc_adapter *adapter,
3056 			      enum igc_mac_filter_type type, const u8 *addr,
3057 			      int queue)
3058 {
3059 	struct net_device *dev = adapter->netdev;
3060 	int index;
3061 
3062 	index = igc_find_mac_filter(adapter, type, addr);
3063 	if (index >= 0)
3064 		goto update_filter;
3065 
3066 	index = igc_get_avail_mac_filter_slot(adapter);
3067 	if (index < 0)
3068 		return -ENOSPC;
3069 
3070 	netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n",
3071 		   index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src",
3072 		   addr, queue);
3073 
3074 update_filter:
3075 	igc_set_mac_filter_hw(adapter, index, type, addr, queue);
3076 	return 0;
3077 }
3078 
3079 /**
3080  * igc_del_mac_filter() - Delete MAC address filter
3081  * @adapter: Pointer to adapter where the filter should be deleted from
3082  * @type: MAC address filter type (source or destination)
3083  * @addr: MAC address
3084  */
igc_del_mac_filter(struct igc_adapter * adapter,enum igc_mac_filter_type type,const u8 * addr)3085 static void igc_del_mac_filter(struct igc_adapter *adapter,
3086 			       enum igc_mac_filter_type type, const u8 *addr)
3087 {
3088 	struct net_device *dev = adapter->netdev;
3089 	int index;
3090 
3091 	index = igc_find_mac_filter(adapter, type, addr);
3092 	if (index < 0)
3093 		return;
3094 
3095 	if (index == 0) {
3096 		/* If this is the default filter, we don't actually delete it.
3097 		 * We just reset to its default value i.e. disable queue
3098 		 * assignment.
3099 		 */
3100 		netdev_dbg(dev, "Disable default MAC filter queue assignment");
3101 
3102 		igc_set_mac_filter_hw(adapter, 0, type, addr, -1);
3103 	} else {
3104 		netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n",
3105 			   index,
3106 			   type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src",
3107 			   addr);
3108 
3109 		igc_clear_mac_filter_hw(adapter, index);
3110 	}
3111 }
3112 
3113 /**
3114  * igc_add_vlan_prio_filter() - Add VLAN priority filter
3115  * @adapter: Pointer to adapter where the filter should be added
3116  * @prio: VLAN priority value
3117  * @queue: Queue number which matching frames are assigned to
3118  *
3119  * Return: 0 in case of success, negative errno code otherwise.
3120  */
igc_add_vlan_prio_filter(struct igc_adapter * adapter,int prio,int queue)3121 static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio,
3122 				    int queue)
3123 {
3124 	struct net_device *dev = adapter->netdev;
3125 	struct igc_hw *hw = &adapter->hw;
3126 	u32 vlanpqf;
3127 
3128 	vlanpqf = rd32(IGC_VLANPQF);
3129 
3130 	if (vlanpqf & IGC_VLANPQF_VALID(prio)) {
3131 		netdev_dbg(dev, "VLAN priority filter already in use\n");
3132 		return -EEXIST;
3133 	}
3134 
3135 	vlanpqf |= IGC_VLANPQF_QSEL(prio, queue);
3136 	vlanpqf |= IGC_VLANPQF_VALID(prio);
3137 
3138 	wr32(IGC_VLANPQF, vlanpqf);
3139 
3140 	netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n",
3141 		   prio, queue);
3142 	return 0;
3143 }
3144 
3145 /**
3146  * igc_del_vlan_prio_filter() - Delete VLAN priority filter
3147  * @adapter: Pointer to adapter where the filter should be deleted from
3148  * @prio: VLAN priority value
3149  */
igc_del_vlan_prio_filter(struct igc_adapter * adapter,int prio)3150 static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio)
3151 {
3152 	struct igc_hw *hw = &adapter->hw;
3153 	u32 vlanpqf;
3154 
3155 	vlanpqf = rd32(IGC_VLANPQF);
3156 
3157 	vlanpqf &= ~IGC_VLANPQF_VALID(prio);
3158 	vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK);
3159 
3160 	wr32(IGC_VLANPQF, vlanpqf);
3161 
3162 	netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n",
3163 		   prio);
3164 }
3165 
igc_get_avail_etype_filter_slot(struct igc_adapter * adapter)3166 static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter)
3167 {
3168 	struct igc_hw *hw = &adapter->hw;
3169 	int i;
3170 
3171 	for (i = 0; i < MAX_ETYPE_FILTER; i++) {
3172 		u32 etqf = rd32(IGC_ETQF(i));
3173 
3174 		if (!(etqf & IGC_ETQF_FILTER_ENABLE))
3175 			return i;
3176 	}
3177 
3178 	return -1;
3179 }
3180 
3181 /**
3182  * igc_add_etype_filter() - Add ethertype filter
3183  * @adapter: Pointer to adapter where the filter should be added
3184  * @etype: Ethertype value
3185  * @queue: If non-negative, queue assignment feature is enabled and frames
3186  *         matching the filter are enqueued onto 'queue'. Otherwise, queue
3187  *         assignment is disabled.
3188  *
3189  * Return: 0 in case of success, negative errno code otherwise.
3190  */
igc_add_etype_filter(struct igc_adapter * adapter,u16 etype,int queue)3191 static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype,
3192 				int queue)
3193 {
3194 	struct igc_hw *hw = &adapter->hw;
3195 	int index;
3196 	u32 etqf;
3197 
3198 	index = igc_get_avail_etype_filter_slot(adapter);
3199 	if (index < 0)
3200 		return -ENOSPC;
3201 
3202 	etqf = rd32(IGC_ETQF(index));
3203 
3204 	etqf &= ~IGC_ETQF_ETYPE_MASK;
3205 	etqf |= etype;
3206 
3207 	if (queue >= 0) {
3208 		etqf &= ~IGC_ETQF_QUEUE_MASK;
3209 		etqf |= (queue << IGC_ETQF_QUEUE_SHIFT);
3210 		etqf |= IGC_ETQF_QUEUE_ENABLE;
3211 	}
3212 
3213 	etqf |= IGC_ETQF_FILTER_ENABLE;
3214 
3215 	wr32(IGC_ETQF(index), etqf);
3216 
3217 	netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n",
3218 		   etype, queue);
3219 	return 0;
3220 }
3221 
igc_find_etype_filter(struct igc_adapter * adapter,u16 etype)3222 static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype)
3223 {
3224 	struct igc_hw *hw = &adapter->hw;
3225 	int i;
3226 
3227 	for (i = 0; i < MAX_ETYPE_FILTER; i++) {
3228 		u32 etqf = rd32(IGC_ETQF(i));
3229 
3230 		if ((etqf & IGC_ETQF_ETYPE_MASK) == etype)
3231 			return i;
3232 	}
3233 
3234 	return -1;
3235 }
3236 
3237 /**
3238  * igc_del_etype_filter() - Delete ethertype filter
3239  * @adapter: Pointer to adapter where the filter should be deleted from
3240  * @etype: Ethertype value
3241  */
igc_del_etype_filter(struct igc_adapter * adapter,u16 etype)3242 static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype)
3243 {
3244 	struct igc_hw *hw = &adapter->hw;
3245 	int index;
3246 
3247 	index = igc_find_etype_filter(adapter, etype);
3248 	if (index < 0)
3249 		return;
3250 
3251 	wr32(IGC_ETQF(index), 0);
3252 
3253 	netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n",
3254 		   etype);
3255 }
3256 
igc_flex_filter_select(struct igc_adapter * adapter,struct igc_flex_filter * input,u32 * fhft)3257 static int igc_flex_filter_select(struct igc_adapter *adapter,
3258 				  struct igc_flex_filter *input,
3259 				  u32 *fhft)
3260 {
3261 	struct igc_hw *hw = &adapter->hw;
3262 	u8 fhft_index;
3263 	u32 fhftsl;
3264 
3265 	if (input->index >= MAX_FLEX_FILTER) {
3266 		dev_err(&adapter->pdev->dev, "Wrong Flex Filter index selected!\n");
3267 		return -EINVAL;
3268 	}
3269 
3270 	/* Indirect table select register */
3271 	fhftsl = rd32(IGC_FHFTSL);
3272 	fhftsl &= ~IGC_FHFTSL_FTSL_MASK;
3273 	switch (input->index) {
3274 	case 0 ... 7:
3275 		fhftsl |= 0x00;
3276 		break;
3277 	case 8 ... 15:
3278 		fhftsl |= 0x01;
3279 		break;
3280 	case 16 ... 23:
3281 		fhftsl |= 0x02;
3282 		break;
3283 	case 24 ... 31:
3284 		fhftsl |= 0x03;
3285 		break;
3286 	}
3287 	wr32(IGC_FHFTSL, fhftsl);
3288 
3289 	/* Normalize index down to host table register */
3290 	fhft_index = input->index % 8;
3291 
3292 	*fhft = (fhft_index < 4) ? IGC_FHFT(fhft_index) :
3293 		IGC_FHFT_EXT(fhft_index - 4);
3294 
3295 	return 0;
3296 }
3297 
igc_write_flex_filter_ll(struct igc_adapter * adapter,struct igc_flex_filter * input)3298 static int igc_write_flex_filter_ll(struct igc_adapter *adapter,
3299 				    struct igc_flex_filter *input)
3300 {
3301 	struct device *dev = &adapter->pdev->dev;
3302 	struct igc_hw *hw = &adapter->hw;
3303 	u8 *data = input->data;
3304 	u8 *mask = input->mask;
3305 	u32 queuing;
3306 	u32 fhft;
3307 	u32 wufc;
3308 	int ret;
3309 	int i;
3310 
3311 	/* Length has to be aligned to 8. Otherwise the filter will fail. Bail
3312 	 * out early to avoid surprises later.
3313 	 */
3314 	if (input->length % 8 != 0) {
3315 		dev_err(dev, "The length of a flex filter has to be 8 byte aligned!\n");
3316 		return -EINVAL;
3317 	}
3318 
3319 	/* Select corresponding flex filter register and get base for host table. */
3320 	ret = igc_flex_filter_select(adapter, input, &fhft);
3321 	if (ret)
3322 		return ret;
3323 
3324 	/* When adding a filter globally disable flex filter feature. That is
3325 	 * recommended within the datasheet.
3326 	 */
3327 	wufc = rd32(IGC_WUFC);
3328 	wufc &= ~IGC_WUFC_FLEX_HQ;
3329 	wr32(IGC_WUFC, wufc);
3330 
3331 	/* Configure filter */
3332 	queuing = input->length & IGC_FHFT_LENGTH_MASK;
3333 	queuing |= (input->rx_queue << IGC_FHFT_QUEUE_SHIFT) & IGC_FHFT_QUEUE_MASK;
3334 	queuing |= (input->prio << IGC_FHFT_PRIO_SHIFT) & IGC_FHFT_PRIO_MASK;
3335 
3336 	if (input->immediate_irq)
3337 		queuing |= IGC_FHFT_IMM_INT;
3338 
3339 	if (input->drop)
3340 		queuing |= IGC_FHFT_DROP;
3341 
3342 	wr32(fhft + 0xFC, queuing);
3343 
3344 	/* Write data (128 byte) and mask (128 bit) */
3345 	for (i = 0; i < 16; ++i) {
3346 		const size_t data_idx = i * 8;
3347 		const size_t row_idx = i * 16;
3348 		u32 dw0 =
3349 			(data[data_idx + 0] << 0) |
3350 			(data[data_idx + 1] << 8) |
3351 			(data[data_idx + 2] << 16) |
3352 			(data[data_idx + 3] << 24);
3353 		u32 dw1 =
3354 			(data[data_idx + 4] << 0) |
3355 			(data[data_idx + 5] << 8) |
3356 			(data[data_idx + 6] << 16) |
3357 			(data[data_idx + 7] << 24);
3358 		u32 tmp;
3359 
3360 		/* Write row: dw0, dw1 and mask */
3361 		wr32(fhft + row_idx, dw0);
3362 		wr32(fhft + row_idx + 4, dw1);
3363 
3364 		/* mask is only valid for MASK(7, 0) */
3365 		tmp = rd32(fhft + row_idx + 8);
3366 		tmp &= ~GENMASK(7, 0);
3367 		tmp |= mask[i];
3368 		wr32(fhft + row_idx + 8, tmp);
3369 	}
3370 
3371 	/* Enable filter. */
3372 	wufc |= IGC_WUFC_FLEX_HQ;
3373 	if (input->index > 8) {
3374 		/* Filter 0-7 are enabled via WUFC. The other 24 filters are not. */
3375 		u32 wufc_ext = rd32(IGC_WUFC_EXT);
3376 
3377 		wufc_ext |= (IGC_WUFC_EXT_FLX8 << (input->index - 8));
3378 
3379 		wr32(IGC_WUFC_EXT, wufc_ext);
3380 	} else {
3381 		wufc |= (IGC_WUFC_FLX0 << input->index);
3382 	}
3383 	wr32(IGC_WUFC, wufc);
3384 
3385 	dev_dbg(&adapter->pdev->dev, "Added flex filter %u to HW.\n",
3386 		input->index);
3387 
3388 	return 0;
3389 }
3390 
igc_flex_filter_add_field(struct igc_flex_filter * flex,const void * src,unsigned int offset,size_t len,const void * mask)3391 static void igc_flex_filter_add_field(struct igc_flex_filter *flex,
3392 				      const void *src, unsigned int offset,
3393 				      size_t len, const void *mask)
3394 {
3395 	int i;
3396 
3397 	/* data */
3398 	memcpy(&flex->data[offset], src, len);
3399 
3400 	/* mask */
3401 	for (i = 0; i < len; ++i) {
3402 		const unsigned int idx = i + offset;
3403 		const u8 *ptr = mask;
3404 
3405 		if (mask) {
3406 			if (ptr[i] & 0xff)
3407 				flex->mask[idx / 8] |= BIT(idx % 8);
3408 
3409 			continue;
3410 		}
3411 
3412 		flex->mask[idx / 8] |= BIT(idx % 8);
3413 	}
3414 }
3415 
igc_find_avail_flex_filter_slot(struct igc_adapter * adapter)3416 static int igc_find_avail_flex_filter_slot(struct igc_adapter *adapter)
3417 {
3418 	struct igc_hw *hw = &adapter->hw;
3419 	u32 wufc, wufc_ext;
3420 	int i;
3421 
3422 	wufc = rd32(IGC_WUFC);
3423 	wufc_ext = rd32(IGC_WUFC_EXT);
3424 
3425 	for (i = 0; i < MAX_FLEX_FILTER; i++) {
3426 		if (i < 8) {
3427 			if (!(wufc & (IGC_WUFC_FLX0 << i)))
3428 				return i;
3429 		} else {
3430 			if (!(wufc_ext & (IGC_WUFC_EXT_FLX8 << (i - 8))))
3431 				return i;
3432 		}
3433 	}
3434 
3435 	return -ENOSPC;
3436 }
3437 
igc_flex_filter_in_use(struct igc_adapter * adapter)3438 static bool igc_flex_filter_in_use(struct igc_adapter *adapter)
3439 {
3440 	struct igc_hw *hw = &adapter->hw;
3441 	u32 wufc, wufc_ext;
3442 
3443 	wufc = rd32(IGC_WUFC);
3444 	wufc_ext = rd32(IGC_WUFC_EXT);
3445 
3446 	if (wufc & IGC_WUFC_FILTER_MASK)
3447 		return true;
3448 
3449 	if (wufc_ext & IGC_WUFC_EXT_FILTER_MASK)
3450 		return true;
3451 
3452 	return false;
3453 }
3454 
igc_add_flex_filter(struct igc_adapter * adapter,struct igc_nfc_rule * rule)3455 static int igc_add_flex_filter(struct igc_adapter *adapter,
3456 			       struct igc_nfc_rule *rule)
3457 {
3458 	struct igc_flex_filter flex = { };
3459 	struct igc_nfc_filter *filter = &rule->filter;
3460 	unsigned int eth_offset, user_offset;
3461 	int ret, index;
3462 	bool vlan;
3463 
3464 	index = igc_find_avail_flex_filter_slot(adapter);
3465 	if (index < 0)
3466 		return -ENOSPC;
3467 
3468 	/* Construct the flex filter:
3469 	 *  -> dest_mac [6]
3470 	 *  -> src_mac [6]
3471 	 *  -> tpid [2]
3472 	 *  -> vlan tci [2]
3473 	 *  -> ether type [2]
3474 	 *  -> user data [8]
3475 	 *  -> = 26 bytes => 32 length
3476 	 */
3477 	flex.index    = index;
3478 	flex.length   = 32;
3479 	flex.rx_queue = rule->action;
3480 
3481 	vlan = rule->filter.vlan_tci || rule->filter.vlan_etype;
3482 	eth_offset = vlan ? 16 : 12;
3483 	user_offset = vlan ? 18 : 14;
3484 
3485 	/* Add destination MAC  */
3486 	if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR)
3487 		igc_flex_filter_add_field(&flex, &filter->dst_addr, 0,
3488 					  ETH_ALEN, NULL);
3489 
3490 	/* Add source MAC */
3491 	if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR)
3492 		igc_flex_filter_add_field(&flex, &filter->src_addr, 6,
3493 					  ETH_ALEN, NULL);
3494 
3495 	/* Add VLAN etype */
3496 	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_ETYPE)
3497 		igc_flex_filter_add_field(&flex, &filter->vlan_etype, 12,
3498 					  sizeof(filter->vlan_etype),
3499 					  NULL);
3500 
3501 	/* Add VLAN TCI */
3502 	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI)
3503 		igc_flex_filter_add_field(&flex, &filter->vlan_tci, 14,
3504 					  sizeof(filter->vlan_tci), NULL);
3505 
3506 	/* Add Ether type */
3507 	if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) {
3508 		__be16 etype = cpu_to_be16(filter->etype);
3509 
3510 		igc_flex_filter_add_field(&flex, &etype, eth_offset,
3511 					  sizeof(etype), NULL);
3512 	}
3513 
3514 	/* Add user data */
3515 	if (rule->filter.match_flags & IGC_FILTER_FLAG_USER_DATA)
3516 		igc_flex_filter_add_field(&flex, &filter->user_data,
3517 					  user_offset,
3518 					  sizeof(filter->user_data),
3519 					  filter->user_mask);
3520 
3521 	/* Add it down to the hardware and enable it. */
3522 	ret = igc_write_flex_filter_ll(adapter, &flex);
3523 	if (ret)
3524 		return ret;
3525 
3526 	filter->flex_index = index;
3527 
3528 	return 0;
3529 }
3530 
igc_del_flex_filter(struct igc_adapter * adapter,u16 reg_index)3531 static void igc_del_flex_filter(struct igc_adapter *adapter,
3532 				u16 reg_index)
3533 {
3534 	struct igc_hw *hw = &adapter->hw;
3535 	u32 wufc;
3536 
3537 	/* Just disable the filter. The filter table itself is kept
3538 	 * intact. Another flex_filter_add() should override the "old" data
3539 	 * then.
3540 	 */
3541 	if (reg_index > 8) {
3542 		u32 wufc_ext = rd32(IGC_WUFC_EXT);
3543 
3544 		wufc_ext &= ~(IGC_WUFC_EXT_FLX8 << (reg_index - 8));
3545 		wr32(IGC_WUFC_EXT, wufc_ext);
3546 	} else {
3547 		wufc = rd32(IGC_WUFC);
3548 
3549 		wufc &= ~(IGC_WUFC_FLX0 << reg_index);
3550 		wr32(IGC_WUFC, wufc);
3551 	}
3552 
3553 	if (igc_flex_filter_in_use(adapter))
3554 		return;
3555 
3556 	/* No filters are in use, we may disable flex filters */
3557 	wufc = rd32(IGC_WUFC);
3558 	wufc &= ~IGC_WUFC_FLEX_HQ;
3559 	wr32(IGC_WUFC, wufc);
3560 }
3561 
igc_enable_nfc_rule(struct igc_adapter * adapter,struct igc_nfc_rule * rule)3562 static int igc_enable_nfc_rule(struct igc_adapter *adapter,
3563 			       struct igc_nfc_rule *rule)
3564 {
3565 	int err;
3566 
3567 	if (rule->flex) {
3568 		return igc_add_flex_filter(adapter, rule);
3569 	}
3570 
3571 	if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) {
3572 		err = igc_add_etype_filter(adapter, rule->filter.etype,
3573 					   rule->action);
3574 		if (err)
3575 			return err;
3576 	}
3577 
3578 	if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) {
3579 		err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC,
3580 					 rule->filter.src_addr, rule->action);
3581 		if (err)
3582 			return err;
3583 	}
3584 
3585 	if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) {
3586 		err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST,
3587 					 rule->filter.dst_addr, rule->action);
3588 		if (err)
3589 			return err;
3590 	}
3591 
3592 	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) {
3593 		int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >>
3594 			   VLAN_PRIO_SHIFT;
3595 
3596 		err = igc_add_vlan_prio_filter(adapter, prio, rule->action);
3597 		if (err)
3598 			return err;
3599 	}
3600 
3601 	return 0;
3602 }
3603 
igc_disable_nfc_rule(struct igc_adapter * adapter,const struct igc_nfc_rule * rule)3604 static void igc_disable_nfc_rule(struct igc_adapter *adapter,
3605 				 const struct igc_nfc_rule *rule)
3606 {
3607 	if (rule->flex) {
3608 		igc_del_flex_filter(adapter, rule->filter.flex_index);
3609 		return;
3610 	}
3611 
3612 	if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE)
3613 		igc_del_etype_filter(adapter, rule->filter.etype);
3614 
3615 	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) {
3616 		int prio = (rule->filter.vlan_tci & VLAN_PRIO_MASK) >>
3617 			   VLAN_PRIO_SHIFT;
3618 
3619 		igc_del_vlan_prio_filter(adapter, prio);
3620 	}
3621 
3622 	if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR)
3623 		igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC,
3624 				   rule->filter.src_addr);
3625 
3626 	if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR)
3627 		igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST,
3628 				   rule->filter.dst_addr);
3629 }
3630 
3631 /**
3632  * igc_get_nfc_rule() - Get NFC rule
3633  * @adapter: Pointer to adapter
3634  * @location: Rule location
3635  *
3636  * Context: Expects adapter->nfc_rule_lock to be held by caller.
3637  *
3638  * Return: Pointer to NFC rule at @location. If not found, NULL.
3639  */
igc_get_nfc_rule(struct igc_adapter * adapter,u32 location)3640 struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter,
3641 				      u32 location)
3642 {
3643 	struct igc_nfc_rule *rule;
3644 
3645 	list_for_each_entry(rule, &adapter->nfc_rule_list, list) {
3646 		if (rule->location == location)
3647 			return rule;
3648 		if (rule->location > location)
3649 			break;
3650 	}
3651 
3652 	return NULL;
3653 }
3654 
3655 /**
3656  * igc_del_nfc_rule() - Delete NFC rule
3657  * @adapter: Pointer to adapter
3658  * @rule: Pointer to rule to be deleted
3659  *
3660  * Disable NFC rule in hardware and delete it from adapter.
3661  *
3662  * Context: Expects adapter->nfc_rule_lock to be held by caller.
3663  */
igc_del_nfc_rule(struct igc_adapter * adapter,struct igc_nfc_rule * rule)3664 void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule)
3665 {
3666 	igc_disable_nfc_rule(adapter, rule);
3667 
3668 	list_del(&rule->list);
3669 	adapter->nfc_rule_count--;
3670 
3671 	kfree(rule);
3672 }
3673 
igc_flush_nfc_rules(struct igc_adapter * adapter)3674 static void igc_flush_nfc_rules(struct igc_adapter *adapter)
3675 {
3676 	struct igc_nfc_rule *rule, *tmp;
3677 
3678 	mutex_lock(&adapter->nfc_rule_lock);
3679 
3680 	list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list)
3681 		igc_del_nfc_rule(adapter, rule);
3682 
3683 	mutex_unlock(&adapter->nfc_rule_lock);
3684 }
3685 
3686 /**
3687  * igc_add_nfc_rule() - Add NFC rule
3688  * @adapter: Pointer to adapter
3689  * @rule: Pointer to rule to be added
3690  *
3691  * Enable NFC rule in hardware and add it to adapter.
3692  *
3693  * Context: Expects adapter->nfc_rule_lock to be held by caller.
3694  *
3695  * Return: 0 on success, negative errno on failure.
3696  */
igc_add_nfc_rule(struct igc_adapter * adapter,struct igc_nfc_rule * rule)3697 int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule)
3698 {
3699 	struct igc_nfc_rule *pred, *cur;
3700 	int err;
3701 
3702 	err = igc_enable_nfc_rule(adapter, rule);
3703 	if (err)
3704 		return err;
3705 
3706 	pred = NULL;
3707 	list_for_each_entry(cur, &adapter->nfc_rule_list, list) {
3708 		if (cur->location >= rule->location)
3709 			break;
3710 		pred = cur;
3711 	}
3712 
3713 	list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list);
3714 	adapter->nfc_rule_count++;
3715 	return 0;
3716 }
3717 
igc_restore_nfc_rules(struct igc_adapter * adapter)3718 static void igc_restore_nfc_rules(struct igc_adapter *adapter)
3719 {
3720 	struct igc_nfc_rule *rule;
3721 
3722 	mutex_lock(&adapter->nfc_rule_lock);
3723 
3724 	list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list)
3725 		igc_enable_nfc_rule(adapter, rule);
3726 
3727 	mutex_unlock(&adapter->nfc_rule_lock);
3728 }
3729 
igc_uc_sync(struct net_device * netdev,const unsigned char * addr)3730 static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr)
3731 {
3732 	struct igc_adapter *adapter = netdev_priv(netdev);
3733 
3734 	return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1);
3735 }
3736 
igc_uc_unsync(struct net_device * netdev,const unsigned char * addr)3737 static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr)
3738 {
3739 	struct igc_adapter *adapter = netdev_priv(netdev);
3740 
3741 	igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr);
3742 	return 0;
3743 }
3744 
3745 /**
3746  * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3747  * @netdev: network interface device structure
3748  *
3749  * The set_rx_mode entry point is called whenever the unicast or multicast
3750  * address lists or the network interface flags are updated.  This routine is
3751  * responsible for configuring the hardware for proper unicast, multicast,
3752  * promiscuous mode, and all-multi behavior.
3753  */
igc_set_rx_mode(struct net_device * netdev)3754 static void igc_set_rx_mode(struct net_device *netdev)
3755 {
3756 	struct igc_adapter *adapter = netdev_priv(netdev);
3757 	struct igc_hw *hw = &adapter->hw;
3758 	u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE;
3759 	int count;
3760 
3761 	/* Check for Promiscuous and All Multicast modes */
3762 	if (netdev->flags & IFF_PROMISC) {
3763 		rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE;
3764 	} else {
3765 		if (netdev->flags & IFF_ALLMULTI) {
3766 			rctl |= IGC_RCTL_MPE;
3767 		} else {
3768 			/* Write addresses to the MTA, if the attempt fails
3769 			 * then we should just turn on promiscuous mode so
3770 			 * that we can at least receive multicast traffic
3771 			 */
3772 			count = igc_write_mc_addr_list(netdev);
3773 			if (count < 0)
3774 				rctl |= IGC_RCTL_MPE;
3775 		}
3776 	}
3777 
3778 	/* Write addresses to available RAR registers, if there is not
3779 	 * sufficient space to store all the addresses then enable
3780 	 * unicast promiscuous mode
3781 	 */
3782 	if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync))
3783 		rctl |= IGC_RCTL_UPE;
3784 
3785 	/* update state of unicast and multicast */
3786 	rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE);
3787 	wr32(IGC_RCTL, rctl);
3788 
3789 #if (PAGE_SIZE < 8192)
3790 	if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB)
3791 		rlpml = IGC_MAX_FRAME_BUILD_SKB;
3792 #endif
3793 	wr32(IGC_RLPML, rlpml);
3794 }
3795 
3796 /**
3797  * igc_configure - configure the hardware for RX and TX
3798  * @adapter: private board structure
3799  */
igc_configure(struct igc_adapter * adapter)3800 static void igc_configure(struct igc_adapter *adapter)
3801 {
3802 	struct net_device *netdev = adapter->netdev;
3803 	int i = 0;
3804 
3805 	igc_get_hw_control(adapter);
3806 	igc_set_rx_mode(netdev);
3807 
3808 	igc_restore_vlan(adapter);
3809 
3810 	igc_setup_tctl(adapter);
3811 	igc_setup_mrqc(adapter);
3812 	igc_setup_rctl(adapter);
3813 
3814 	igc_set_default_mac_filter(adapter);
3815 	igc_restore_nfc_rules(adapter);
3816 
3817 	igc_configure_tx(adapter);
3818 	igc_configure_rx(adapter);
3819 
3820 	igc_rx_fifo_flush_base(&adapter->hw);
3821 
3822 	/* call igc_desc_unused which always leaves
3823 	 * at least 1 descriptor unused to make sure
3824 	 * next_to_use != next_to_clean
3825 	 */
3826 	for (i = 0; i < adapter->num_rx_queues; i++) {
3827 		struct igc_ring *ring = adapter->rx_ring[i];
3828 
3829 		if (ring->xsk_pool)
3830 			igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring));
3831 		else
3832 			igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
3833 	}
3834 }
3835 
3836 /**
3837  * igc_write_ivar - configure ivar for given MSI-X vector
3838  * @hw: pointer to the HW structure
3839  * @msix_vector: vector number we are allocating to a given ring
3840  * @index: row index of IVAR register to write within IVAR table
3841  * @offset: column offset of in IVAR, should be multiple of 8
3842  *
3843  * The IVAR table consists of 2 columns,
3844  * each containing an cause allocation for an Rx and Tx ring, and a
3845  * variable number of rows depending on the number of queues supported.
3846  */
igc_write_ivar(struct igc_hw * hw,int msix_vector,int index,int offset)3847 static void igc_write_ivar(struct igc_hw *hw, int msix_vector,
3848 			   int index, int offset)
3849 {
3850 	u32 ivar = array_rd32(IGC_IVAR0, index);
3851 
3852 	/* clear any bits that are currently set */
3853 	ivar &= ~((u32)0xFF << offset);
3854 
3855 	/* write vector and valid bit */
3856 	ivar |= (msix_vector | IGC_IVAR_VALID) << offset;
3857 
3858 	array_wr32(IGC_IVAR0, index, ivar);
3859 }
3860 
igc_assign_vector(struct igc_q_vector * q_vector,int msix_vector)3861 static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector)
3862 {
3863 	struct igc_adapter *adapter = q_vector->adapter;
3864 	struct igc_hw *hw = &adapter->hw;
3865 	int rx_queue = IGC_N0_QUEUE;
3866 	int tx_queue = IGC_N0_QUEUE;
3867 
3868 	if (q_vector->rx.ring)
3869 		rx_queue = q_vector->rx.ring->reg_idx;
3870 	if (q_vector->tx.ring)
3871 		tx_queue = q_vector->tx.ring->reg_idx;
3872 
3873 	switch (hw->mac.type) {
3874 	case igc_i225:
3875 		if (rx_queue > IGC_N0_QUEUE)
3876 			igc_write_ivar(hw, msix_vector,
3877 				       rx_queue >> 1,
3878 				       (rx_queue & 0x1) << 4);
3879 		if (tx_queue > IGC_N0_QUEUE)
3880 			igc_write_ivar(hw, msix_vector,
3881 				       tx_queue >> 1,
3882 				       ((tx_queue & 0x1) << 4) + 8);
3883 		q_vector->eims_value = BIT(msix_vector);
3884 		break;
3885 	default:
3886 		WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n");
3887 		break;
3888 	}
3889 
3890 	/* add q_vector eims value to global eims_enable_mask */
3891 	adapter->eims_enable_mask |= q_vector->eims_value;
3892 
3893 	/* configure q_vector to set itr on first interrupt */
3894 	q_vector->set_itr = 1;
3895 }
3896 
3897 /**
3898  * igc_configure_msix - Configure MSI-X hardware
3899  * @adapter: Pointer to adapter structure
3900  *
3901  * igc_configure_msix sets up the hardware to properly
3902  * generate MSI-X interrupts.
3903  */
igc_configure_msix(struct igc_adapter * adapter)3904 static void igc_configure_msix(struct igc_adapter *adapter)
3905 {
3906 	struct igc_hw *hw = &adapter->hw;
3907 	int i, vector = 0;
3908 	u32 tmp;
3909 
3910 	adapter->eims_enable_mask = 0;
3911 
3912 	/* set vector for other causes, i.e. link changes */
3913 	switch (hw->mac.type) {
3914 	case igc_i225:
3915 		/* Turn on MSI-X capability first, or our settings
3916 		 * won't stick.  And it will take days to debug.
3917 		 */
3918 		wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE |
3919 		     IGC_GPIE_PBA | IGC_GPIE_EIAME |
3920 		     IGC_GPIE_NSICR);
3921 
3922 		/* enable msix_other interrupt */
3923 		adapter->eims_other = BIT(vector);
3924 		tmp = (vector++ | IGC_IVAR_VALID) << 8;
3925 
3926 		wr32(IGC_IVAR_MISC, tmp);
3927 		break;
3928 	default:
3929 		/* do nothing, since nothing else supports MSI-X */
3930 		break;
3931 	} /* switch (hw->mac.type) */
3932 
3933 	adapter->eims_enable_mask |= adapter->eims_other;
3934 
3935 	for (i = 0; i < adapter->num_q_vectors; i++)
3936 		igc_assign_vector(adapter->q_vector[i], vector++);
3937 
3938 	wrfl();
3939 }
3940 
3941 /**
3942  * igc_irq_enable - Enable default interrupt generation settings
3943  * @adapter: board private structure
3944  */
igc_irq_enable(struct igc_adapter * adapter)3945 static void igc_irq_enable(struct igc_adapter *adapter)
3946 {
3947 	struct igc_hw *hw = &adapter->hw;
3948 
3949 	if (adapter->msix_entries) {
3950 		u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA;
3951 		u32 regval = rd32(IGC_EIAC);
3952 
3953 		wr32(IGC_EIAC, regval | adapter->eims_enable_mask);
3954 		regval = rd32(IGC_EIAM);
3955 		wr32(IGC_EIAM, regval | adapter->eims_enable_mask);
3956 		wr32(IGC_EIMS, adapter->eims_enable_mask);
3957 		wr32(IGC_IMS, ims);
3958 	} else {
3959 		wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
3960 		wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
3961 	}
3962 }
3963 
3964 /**
3965  * igc_irq_disable - Mask off interrupt generation on the NIC
3966  * @adapter: board private structure
3967  */
igc_irq_disable(struct igc_adapter * adapter)3968 static void igc_irq_disable(struct igc_adapter *adapter)
3969 {
3970 	struct igc_hw *hw = &adapter->hw;
3971 
3972 	if (adapter->msix_entries) {
3973 		u32 regval = rd32(IGC_EIAM);
3974 
3975 		wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask);
3976 		wr32(IGC_EIMC, adapter->eims_enable_mask);
3977 		regval = rd32(IGC_EIAC);
3978 		wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask);
3979 	}
3980 
3981 	wr32(IGC_IAM, 0);
3982 	wr32(IGC_IMC, ~0);
3983 	wrfl();
3984 
3985 	if (adapter->msix_entries) {
3986 		int vector = 0, i;
3987 
3988 		synchronize_irq(adapter->msix_entries[vector++].vector);
3989 
3990 		for (i = 0; i < adapter->num_q_vectors; i++)
3991 			synchronize_irq(adapter->msix_entries[vector++].vector);
3992 	} else {
3993 		synchronize_irq(adapter->pdev->irq);
3994 	}
3995 }
3996 
igc_set_flag_queue_pairs(struct igc_adapter * adapter,const u32 max_rss_queues)3997 void igc_set_flag_queue_pairs(struct igc_adapter *adapter,
3998 			      const u32 max_rss_queues)
3999 {
4000 	/* Determine if we need to pair queues. */
4001 	/* If rss_queues > half of max_rss_queues, pair the queues in
4002 	 * order to conserve interrupts due to limited supply.
4003 	 */
4004 	if (adapter->rss_queues > (max_rss_queues / 2))
4005 		adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
4006 	else
4007 		adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS;
4008 }
4009 
igc_get_max_rss_queues(struct igc_adapter * adapter)4010 unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter)
4011 {
4012 	return IGC_MAX_RX_QUEUES;
4013 }
4014 
igc_init_queue_configuration(struct igc_adapter * adapter)4015 static void igc_init_queue_configuration(struct igc_adapter *adapter)
4016 {
4017 	u32 max_rss_queues;
4018 
4019 	max_rss_queues = igc_get_max_rss_queues(adapter);
4020 	adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
4021 
4022 	igc_set_flag_queue_pairs(adapter, max_rss_queues);
4023 }
4024 
4025 /**
4026  * igc_reset_q_vector - Reset config for interrupt vector
4027  * @adapter: board private structure to initialize
4028  * @v_idx: Index of vector to be reset
4029  *
4030  * If NAPI is enabled it will delete any references to the
4031  * NAPI struct. This is preparation for igc_free_q_vector.
4032  */
igc_reset_q_vector(struct igc_adapter * adapter,int v_idx)4033 static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx)
4034 {
4035 	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
4036 
4037 	/* if we're coming from igc_set_interrupt_capability, the vectors are
4038 	 * not yet allocated
4039 	 */
4040 	if (!q_vector)
4041 		return;
4042 
4043 	if (q_vector->tx.ring)
4044 		adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
4045 
4046 	if (q_vector->rx.ring)
4047 		adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
4048 
4049 	netif_napi_del(&q_vector->napi);
4050 }
4051 
4052 /**
4053  * igc_free_q_vector - Free memory allocated for specific interrupt vector
4054  * @adapter: board private structure to initialize
4055  * @v_idx: Index of vector to be freed
4056  *
4057  * This function frees the memory allocated to the q_vector.
4058  */
igc_free_q_vector(struct igc_adapter * adapter,int v_idx)4059 static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx)
4060 {
4061 	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
4062 
4063 	adapter->q_vector[v_idx] = NULL;
4064 
4065 	/* igc_get_stats64() might access the rings on this vector,
4066 	 * we must wait a grace period before freeing it.
4067 	 */
4068 	if (q_vector)
4069 		kfree_rcu(q_vector, rcu);
4070 }
4071 
4072 /**
4073  * igc_free_q_vectors - Free memory allocated for interrupt vectors
4074  * @adapter: board private structure to initialize
4075  *
4076  * This function frees the memory allocated to the q_vectors.  In addition if
4077  * NAPI is enabled it will delete any references to the NAPI struct prior
4078  * to freeing the q_vector.
4079  */
igc_free_q_vectors(struct igc_adapter * adapter)4080 static void igc_free_q_vectors(struct igc_adapter *adapter)
4081 {
4082 	int v_idx = adapter->num_q_vectors;
4083 
4084 	adapter->num_tx_queues = 0;
4085 	adapter->num_rx_queues = 0;
4086 	adapter->num_q_vectors = 0;
4087 
4088 	while (v_idx--) {
4089 		igc_reset_q_vector(adapter, v_idx);
4090 		igc_free_q_vector(adapter, v_idx);
4091 	}
4092 }
4093 
4094 /**
4095  * igc_update_itr - update the dynamic ITR value based on statistics
4096  * @q_vector: pointer to q_vector
4097  * @ring_container: ring info to update the itr for
4098  *
4099  * Stores a new ITR value based on packets and byte
4100  * counts during the last interrupt.  The advantage of per interrupt
4101  * computation is faster updates and more accurate ITR for the current
4102  * traffic pattern.  Constants in this function were computed
4103  * based on theoretical maximum wire speed and thresholds were set based
4104  * on testing data as well as attempting to minimize response time
4105  * while increasing bulk throughput.
4106  * NOTE: These calculations are only valid when operating in a single-
4107  * queue environment.
4108  */
igc_update_itr(struct igc_q_vector * q_vector,struct igc_ring_container * ring_container)4109 static void igc_update_itr(struct igc_q_vector *q_vector,
4110 			   struct igc_ring_container *ring_container)
4111 {
4112 	unsigned int packets = ring_container->total_packets;
4113 	unsigned int bytes = ring_container->total_bytes;
4114 	u8 itrval = ring_container->itr;
4115 
4116 	/* no packets, exit with status unchanged */
4117 	if (packets == 0)
4118 		return;
4119 
4120 	switch (itrval) {
4121 	case lowest_latency:
4122 		/* handle TSO and jumbo frames */
4123 		if (bytes / packets > 8000)
4124 			itrval = bulk_latency;
4125 		else if ((packets < 5) && (bytes > 512))
4126 			itrval = low_latency;
4127 		break;
4128 	case low_latency:  /* 50 usec aka 20000 ints/s */
4129 		if (bytes > 10000) {
4130 			/* this if handles the TSO accounting */
4131 			if (bytes / packets > 8000)
4132 				itrval = bulk_latency;
4133 			else if ((packets < 10) || ((bytes / packets) > 1200))
4134 				itrval = bulk_latency;
4135 			else if ((packets > 35))
4136 				itrval = lowest_latency;
4137 		} else if (bytes / packets > 2000) {
4138 			itrval = bulk_latency;
4139 		} else if (packets <= 2 && bytes < 512) {
4140 			itrval = lowest_latency;
4141 		}
4142 		break;
4143 	case bulk_latency: /* 250 usec aka 4000 ints/s */
4144 		if (bytes > 25000) {
4145 			if (packets > 35)
4146 				itrval = low_latency;
4147 		} else if (bytes < 1500) {
4148 			itrval = low_latency;
4149 		}
4150 		break;
4151 	}
4152 
4153 	/* clear work counters since we have the values we need */
4154 	ring_container->total_bytes = 0;
4155 	ring_container->total_packets = 0;
4156 
4157 	/* write updated itr to ring container */
4158 	ring_container->itr = itrval;
4159 }
4160 
igc_set_itr(struct igc_q_vector * q_vector)4161 static void igc_set_itr(struct igc_q_vector *q_vector)
4162 {
4163 	struct igc_adapter *adapter = q_vector->adapter;
4164 	u32 new_itr = q_vector->itr_val;
4165 	u8 current_itr = 0;
4166 
4167 	/* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4168 	switch (adapter->link_speed) {
4169 	case SPEED_10:
4170 	case SPEED_100:
4171 		current_itr = 0;
4172 		new_itr = IGC_4K_ITR;
4173 		goto set_itr_now;
4174 	default:
4175 		break;
4176 	}
4177 
4178 	igc_update_itr(q_vector, &q_vector->tx);
4179 	igc_update_itr(q_vector, &q_vector->rx);
4180 
4181 	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
4182 
4183 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
4184 	if (current_itr == lowest_latency &&
4185 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4186 	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4187 		current_itr = low_latency;
4188 
4189 	switch (current_itr) {
4190 	/* counts and packets in update_itr are dependent on these numbers */
4191 	case lowest_latency:
4192 		new_itr = IGC_70K_ITR; /* 70,000 ints/sec */
4193 		break;
4194 	case low_latency:
4195 		new_itr = IGC_20K_ITR; /* 20,000 ints/sec */
4196 		break;
4197 	case bulk_latency:
4198 		new_itr = IGC_4K_ITR;  /* 4,000 ints/sec */
4199 		break;
4200 	default:
4201 		break;
4202 	}
4203 
4204 set_itr_now:
4205 	if (new_itr != q_vector->itr_val) {
4206 		/* this attempts to bias the interrupt rate towards Bulk
4207 		 * by adding intermediate steps when interrupt rate is
4208 		 * increasing
4209 		 */
4210 		new_itr = new_itr > q_vector->itr_val ?
4211 			  max((new_itr * q_vector->itr_val) /
4212 			  (new_itr + (q_vector->itr_val >> 2)),
4213 			  new_itr) : new_itr;
4214 		/* Don't write the value here; it resets the adapter's
4215 		 * internal timer, and causes us to delay far longer than
4216 		 * we should between interrupts.  Instead, we write the ITR
4217 		 * value at the beginning of the next interrupt so the timing
4218 		 * ends up being correct.
4219 		 */
4220 		q_vector->itr_val = new_itr;
4221 		q_vector->set_itr = 1;
4222 	}
4223 }
4224 
igc_reset_interrupt_capability(struct igc_adapter * adapter)4225 static void igc_reset_interrupt_capability(struct igc_adapter *adapter)
4226 {
4227 	int v_idx = adapter->num_q_vectors;
4228 
4229 	if (adapter->msix_entries) {
4230 		pci_disable_msix(adapter->pdev);
4231 		kfree(adapter->msix_entries);
4232 		adapter->msix_entries = NULL;
4233 	} else if (adapter->flags & IGC_FLAG_HAS_MSI) {
4234 		pci_disable_msi(adapter->pdev);
4235 	}
4236 
4237 	while (v_idx--)
4238 		igc_reset_q_vector(adapter, v_idx);
4239 }
4240 
4241 /**
4242  * igc_set_interrupt_capability - set MSI or MSI-X if supported
4243  * @adapter: Pointer to adapter structure
4244  * @msix: boolean value for MSI-X capability
4245  *
4246  * Attempt to configure interrupts using the best available
4247  * capabilities of the hardware and kernel.
4248  */
igc_set_interrupt_capability(struct igc_adapter * adapter,bool msix)4249 static void igc_set_interrupt_capability(struct igc_adapter *adapter,
4250 					 bool msix)
4251 {
4252 	int numvecs, i;
4253 	int err;
4254 
4255 	if (!msix)
4256 		goto msi_only;
4257 	adapter->flags |= IGC_FLAG_HAS_MSIX;
4258 
4259 	/* Number of supported queues. */
4260 	adapter->num_rx_queues = adapter->rss_queues;
4261 
4262 	adapter->num_tx_queues = adapter->rss_queues;
4263 
4264 	/* start with one vector for every Rx queue */
4265 	numvecs = adapter->num_rx_queues;
4266 
4267 	/* if Tx handler is separate add 1 for every Tx queue */
4268 	if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS))
4269 		numvecs += adapter->num_tx_queues;
4270 
4271 	/* store the number of vectors reserved for queues */
4272 	adapter->num_q_vectors = numvecs;
4273 
4274 	/* add 1 vector for link status interrupts */
4275 	numvecs++;
4276 
4277 	adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
4278 					GFP_KERNEL);
4279 
4280 	if (!adapter->msix_entries)
4281 		return;
4282 
4283 	/* populate entry values */
4284 	for (i = 0; i < numvecs; i++)
4285 		adapter->msix_entries[i].entry = i;
4286 
4287 	err = pci_enable_msix_range(adapter->pdev,
4288 				    adapter->msix_entries,
4289 				    numvecs,
4290 				    numvecs);
4291 	if (err > 0)
4292 		return;
4293 
4294 	kfree(adapter->msix_entries);
4295 	adapter->msix_entries = NULL;
4296 
4297 	igc_reset_interrupt_capability(adapter);
4298 
4299 msi_only:
4300 	adapter->flags &= ~IGC_FLAG_HAS_MSIX;
4301 
4302 	adapter->rss_queues = 1;
4303 	adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
4304 	adapter->num_rx_queues = 1;
4305 	adapter->num_tx_queues = 1;
4306 	adapter->num_q_vectors = 1;
4307 	if (!pci_enable_msi(adapter->pdev))
4308 		adapter->flags |= IGC_FLAG_HAS_MSI;
4309 }
4310 
4311 /**
4312  * igc_update_ring_itr - update the dynamic ITR value based on packet size
4313  * @q_vector: pointer to q_vector
4314  *
4315  * Stores a new ITR value based on strictly on packet size.  This
4316  * algorithm is less sophisticated than that used in igc_update_itr,
4317  * due to the difficulty of synchronizing statistics across multiple
4318  * receive rings.  The divisors and thresholds used by this function
4319  * were determined based on theoretical maximum wire speed and testing
4320  * data, in order to minimize response time while increasing bulk
4321  * throughput.
4322  * NOTE: This function is called only when operating in a multiqueue
4323  * receive environment.
4324  */
igc_update_ring_itr(struct igc_q_vector * q_vector)4325 static void igc_update_ring_itr(struct igc_q_vector *q_vector)
4326 {
4327 	struct igc_adapter *adapter = q_vector->adapter;
4328 	int new_val = q_vector->itr_val;
4329 	int avg_wire_size = 0;
4330 	unsigned int packets;
4331 
4332 	/* For non-gigabit speeds, just fix the interrupt rate at 4000
4333 	 * ints/sec - ITR timer value of 120 ticks.
4334 	 */
4335 	switch (adapter->link_speed) {
4336 	case SPEED_10:
4337 	case SPEED_100:
4338 		new_val = IGC_4K_ITR;
4339 		goto set_itr_val;
4340 	default:
4341 		break;
4342 	}
4343 
4344 	packets = q_vector->rx.total_packets;
4345 	if (packets)
4346 		avg_wire_size = q_vector->rx.total_bytes / packets;
4347 
4348 	packets = q_vector->tx.total_packets;
4349 	if (packets)
4350 		avg_wire_size = max_t(u32, avg_wire_size,
4351 				      q_vector->tx.total_bytes / packets);
4352 
4353 	/* if avg_wire_size isn't set no work was done */
4354 	if (!avg_wire_size)
4355 		goto clear_counts;
4356 
4357 	/* Add 24 bytes to size to account for CRC, preamble, and gap */
4358 	avg_wire_size += 24;
4359 
4360 	/* Don't starve jumbo frames */
4361 	avg_wire_size = min(avg_wire_size, 3000);
4362 
4363 	/* Give a little boost to mid-size frames */
4364 	if (avg_wire_size > 300 && avg_wire_size < 1200)
4365 		new_val = avg_wire_size / 3;
4366 	else
4367 		new_val = avg_wire_size / 2;
4368 
4369 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
4370 	if (new_val < IGC_20K_ITR &&
4371 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4372 	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4373 		new_val = IGC_20K_ITR;
4374 
4375 set_itr_val:
4376 	if (new_val != q_vector->itr_val) {
4377 		q_vector->itr_val = new_val;
4378 		q_vector->set_itr = 1;
4379 	}
4380 clear_counts:
4381 	q_vector->rx.total_bytes = 0;
4382 	q_vector->rx.total_packets = 0;
4383 	q_vector->tx.total_bytes = 0;
4384 	q_vector->tx.total_packets = 0;
4385 }
4386 
igc_ring_irq_enable(struct igc_q_vector * q_vector)4387 static void igc_ring_irq_enable(struct igc_q_vector *q_vector)
4388 {
4389 	struct igc_adapter *adapter = q_vector->adapter;
4390 	struct igc_hw *hw = &adapter->hw;
4391 
4392 	if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
4393 	    (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
4394 		if (adapter->num_q_vectors == 1)
4395 			igc_set_itr(q_vector);
4396 		else
4397 			igc_update_ring_itr(q_vector);
4398 	}
4399 
4400 	if (!test_bit(__IGC_DOWN, &adapter->state)) {
4401 		if (adapter->msix_entries)
4402 			wr32(IGC_EIMS, q_vector->eims_value);
4403 		else
4404 			igc_irq_enable(adapter);
4405 	}
4406 }
4407 
igc_add_ring(struct igc_ring * ring,struct igc_ring_container * head)4408 static void igc_add_ring(struct igc_ring *ring,
4409 			 struct igc_ring_container *head)
4410 {
4411 	head->ring = ring;
4412 	head->count++;
4413 }
4414 
4415 /**
4416  * igc_cache_ring_register - Descriptor ring to register mapping
4417  * @adapter: board private structure to initialize
4418  *
4419  * Once we know the feature-set enabled for the device, we'll cache
4420  * the register offset the descriptor ring is assigned to.
4421  */
igc_cache_ring_register(struct igc_adapter * adapter)4422 static void igc_cache_ring_register(struct igc_adapter *adapter)
4423 {
4424 	int i = 0, j = 0;
4425 
4426 	switch (adapter->hw.mac.type) {
4427 	case igc_i225:
4428 	default:
4429 		for (; i < adapter->num_rx_queues; i++)
4430 			adapter->rx_ring[i]->reg_idx = i;
4431 		for (; j < adapter->num_tx_queues; j++)
4432 			adapter->tx_ring[j]->reg_idx = j;
4433 		break;
4434 	}
4435 }
4436 
4437 /**
4438  * igc_poll - NAPI Rx polling callback
4439  * @napi: napi polling structure
4440  * @budget: count of how many packets we should handle
4441  */
igc_poll(struct napi_struct * napi,int budget)4442 static int igc_poll(struct napi_struct *napi, int budget)
4443 {
4444 	struct igc_q_vector *q_vector = container_of(napi,
4445 						     struct igc_q_vector,
4446 						     napi);
4447 	struct igc_ring *rx_ring = q_vector->rx.ring;
4448 	bool clean_complete = true;
4449 	int work_done = 0;
4450 
4451 	if (q_vector->tx.ring)
4452 		clean_complete = igc_clean_tx_irq(q_vector, budget);
4453 
4454 	if (rx_ring) {
4455 		int cleaned = rx_ring->xsk_pool ?
4456 			      igc_clean_rx_irq_zc(q_vector, budget) :
4457 			      igc_clean_rx_irq(q_vector, budget);
4458 
4459 		work_done += cleaned;
4460 		if (cleaned >= budget)
4461 			clean_complete = false;
4462 	}
4463 
4464 	/* If all work not completed, return budget and keep polling */
4465 	if (!clean_complete)
4466 		return budget;
4467 
4468 	/* Exit the polling mode, but don't re-enable interrupts if stack might
4469 	 * poll us due to busy-polling
4470 	 */
4471 	if (likely(napi_complete_done(napi, work_done)))
4472 		igc_ring_irq_enable(q_vector);
4473 
4474 	return min(work_done, budget - 1);
4475 }
4476 
4477 /**
4478  * igc_alloc_q_vector - Allocate memory for a single interrupt vector
4479  * @adapter: board private structure to initialize
4480  * @v_count: q_vectors allocated on adapter, used for ring interleaving
4481  * @v_idx: index of vector in adapter struct
4482  * @txr_count: total number of Tx rings to allocate
4483  * @txr_idx: index of first Tx ring to allocate
4484  * @rxr_count: total number of Rx rings to allocate
4485  * @rxr_idx: index of first Rx ring to allocate
4486  *
4487  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
4488  */
igc_alloc_q_vector(struct igc_adapter * adapter,unsigned int v_count,unsigned int v_idx,unsigned int txr_count,unsigned int txr_idx,unsigned int rxr_count,unsigned int rxr_idx)4489 static int igc_alloc_q_vector(struct igc_adapter *adapter,
4490 			      unsigned int v_count, unsigned int v_idx,
4491 			      unsigned int txr_count, unsigned int txr_idx,
4492 			      unsigned int rxr_count, unsigned int rxr_idx)
4493 {
4494 	struct igc_q_vector *q_vector;
4495 	struct igc_ring *ring;
4496 	int ring_count;
4497 
4498 	/* igc only supports 1 Tx and/or 1 Rx queue per vector */
4499 	if (txr_count > 1 || rxr_count > 1)
4500 		return -ENOMEM;
4501 
4502 	ring_count = txr_count + rxr_count;
4503 
4504 	/* allocate q_vector and rings */
4505 	q_vector = adapter->q_vector[v_idx];
4506 	if (!q_vector)
4507 		q_vector = kzalloc(struct_size(q_vector, ring, ring_count),
4508 				   GFP_KERNEL);
4509 	else
4510 		memset(q_vector, 0, struct_size(q_vector, ring, ring_count));
4511 	if (!q_vector)
4512 		return -ENOMEM;
4513 
4514 	/* initialize NAPI */
4515 	netif_napi_add(adapter->netdev, &q_vector->napi, igc_poll);
4516 
4517 	/* tie q_vector and adapter together */
4518 	adapter->q_vector[v_idx] = q_vector;
4519 	q_vector->adapter = adapter;
4520 
4521 	/* initialize work limits */
4522 	q_vector->tx.work_limit = adapter->tx_work_limit;
4523 
4524 	/* initialize ITR configuration */
4525 	q_vector->itr_register = adapter->io_addr + IGC_EITR(0);
4526 	q_vector->itr_val = IGC_START_ITR;
4527 
4528 	/* initialize pointer to rings */
4529 	ring = q_vector->ring;
4530 
4531 	/* initialize ITR */
4532 	if (rxr_count) {
4533 		/* rx or rx/tx vector */
4534 		if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
4535 			q_vector->itr_val = adapter->rx_itr_setting;
4536 	} else {
4537 		/* tx only vector */
4538 		if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
4539 			q_vector->itr_val = adapter->tx_itr_setting;
4540 	}
4541 
4542 	if (txr_count) {
4543 		/* assign generic ring traits */
4544 		ring->dev = &adapter->pdev->dev;
4545 		ring->netdev = adapter->netdev;
4546 
4547 		/* configure backlink on ring */
4548 		ring->q_vector = q_vector;
4549 
4550 		/* update q_vector Tx values */
4551 		igc_add_ring(ring, &q_vector->tx);
4552 
4553 		/* apply Tx specific ring traits */
4554 		ring->count = adapter->tx_ring_count;
4555 		ring->queue_index = txr_idx;
4556 
4557 		/* assign ring to adapter */
4558 		adapter->tx_ring[txr_idx] = ring;
4559 
4560 		/* push pointer to next ring */
4561 		ring++;
4562 	}
4563 
4564 	if (rxr_count) {
4565 		/* assign generic ring traits */
4566 		ring->dev = &adapter->pdev->dev;
4567 		ring->netdev = adapter->netdev;
4568 
4569 		/* configure backlink on ring */
4570 		ring->q_vector = q_vector;
4571 
4572 		/* update q_vector Rx values */
4573 		igc_add_ring(ring, &q_vector->rx);
4574 
4575 		/* apply Rx specific ring traits */
4576 		ring->count = adapter->rx_ring_count;
4577 		ring->queue_index = rxr_idx;
4578 
4579 		/* assign ring to adapter */
4580 		adapter->rx_ring[rxr_idx] = ring;
4581 	}
4582 
4583 	return 0;
4584 }
4585 
4586 /**
4587  * igc_alloc_q_vectors - Allocate memory for interrupt vectors
4588  * @adapter: board private structure to initialize
4589  *
4590  * We allocate one q_vector per queue interrupt.  If allocation fails we
4591  * return -ENOMEM.
4592  */
igc_alloc_q_vectors(struct igc_adapter * adapter)4593 static int igc_alloc_q_vectors(struct igc_adapter *adapter)
4594 {
4595 	int rxr_remaining = adapter->num_rx_queues;
4596 	int txr_remaining = adapter->num_tx_queues;
4597 	int rxr_idx = 0, txr_idx = 0, v_idx = 0;
4598 	int q_vectors = adapter->num_q_vectors;
4599 	int err;
4600 
4601 	if (q_vectors >= (rxr_remaining + txr_remaining)) {
4602 		for (; rxr_remaining; v_idx++) {
4603 			err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
4604 						 0, 0, 1, rxr_idx);
4605 
4606 			if (err)
4607 				goto err_out;
4608 
4609 			/* update counts and index */
4610 			rxr_remaining--;
4611 			rxr_idx++;
4612 		}
4613 	}
4614 
4615 	for (; v_idx < q_vectors; v_idx++) {
4616 		int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
4617 		int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
4618 
4619 		err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
4620 					 tqpv, txr_idx, rqpv, rxr_idx);
4621 
4622 		if (err)
4623 			goto err_out;
4624 
4625 		/* update counts and index */
4626 		rxr_remaining -= rqpv;
4627 		txr_remaining -= tqpv;
4628 		rxr_idx++;
4629 		txr_idx++;
4630 	}
4631 
4632 	return 0;
4633 
4634 err_out:
4635 	adapter->num_tx_queues = 0;
4636 	adapter->num_rx_queues = 0;
4637 	adapter->num_q_vectors = 0;
4638 
4639 	while (v_idx--)
4640 		igc_free_q_vector(adapter, v_idx);
4641 
4642 	return -ENOMEM;
4643 }
4644 
4645 /**
4646  * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
4647  * @adapter: Pointer to adapter structure
4648  * @msix: boolean for MSI-X capability
4649  *
4650  * This function initializes the interrupts and allocates all of the queues.
4651  */
igc_init_interrupt_scheme(struct igc_adapter * adapter,bool msix)4652 static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix)
4653 {
4654 	struct net_device *dev = adapter->netdev;
4655 	int err = 0;
4656 
4657 	igc_set_interrupt_capability(adapter, msix);
4658 
4659 	err = igc_alloc_q_vectors(adapter);
4660 	if (err) {
4661 		netdev_err(dev, "Unable to allocate memory for vectors\n");
4662 		goto err_alloc_q_vectors;
4663 	}
4664 
4665 	igc_cache_ring_register(adapter);
4666 
4667 	return 0;
4668 
4669 err_alloc_q_vectors:
4670 	igc_reset_interrupt_capability(adapter);
4671 	return err;
4672 }
4673 
4674 /**
4675  * igc_sw_init - Initialize general software structures (struct igc_adapter)
4676  * @adapter: board private structure to initialize
4677  *
4678  * igc_sw_init initializes the Adapter private data structure.
4679  * Fields are initialized based on PCI device information and
4680  * OS network device settings (MTU size).
4681  */
igc_sw_init(struct igc_adapter * adapter)4682 static int igc_sw_init(struct igc_adapter *adapter)
4683 {
4684 	struct net_device *netdev = adapter->netdev;
4685 	struct pci_dev *pdev = adapter->pdev;
4686 	struct igc_hw *hw = &adapter->hw;
4687 
4688 	pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
4689 
4690 	/* set default ring sizes */
4691 	adapter->tx_ring_count = IGC_DEFAULT_TXD;
4692 	adapter->rx_ring_count = IGC_DEFAULT_RXD;
4693 
4694 	/* set default ITR values */
4695 	adapter->rx_itr_setting = IGC_DEFAULT_ITR;
4696 	adapter->tx_itr_setting = IGC_DEFAULT_ITR;
4697 
4698 	/* set default work limits */
4699 	adapter->tx_work_limit = IGC_DEFAULT_TX_WORK;
4700 
4701 	/* adjust max frame to be at least the size of a standard frame */
4702 	adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
4703 				VLAN_HLEN;
4704 	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
4705 
4706 	mutex_init(&adapter->nfc_rule_lock);
4707 	INIT_LIST_HEAD(&adapter->nfc_rule_list);
4708 	adapter->nfc_rule_count = 0;
4709 
4710 	spin_lock_init(&adapter->stats64_lock);
4711 	/* Assume MSI-X interrupts, will be checked during IRQ allocation */
4712 	adapter->flags |= IGC_FLAG_HAS_MSIX;
4713 
4714 	igc_init_queue_configuration(adapter);
4715 
4716 	/* This call may decrease the number of queues */
4717 	if (igc_init_interrupt_scheme(adapter, true)) {
4718 		netdev_err(netdev, "Unable to allocate memory for queues\n");
4719 		return -ENOMEM;
4720 	}
4721 
4722 	/* Explicitly disable IRQ since the NIC can be in any state. */
4723 	igc_irq_disable(adapter);
4724 
4725 	set_bit(__IGC_DOWN, &adapter->state);
4726 
4727 	return 0;
4728 }
4729 
4730 /**
4731  * igc_up - Open the interface and prepare it to handle traffic
4732  * @adapter: board private structure
4733  */
igc_up(struct igc_adapter * adapter)4734 void igc_up(struct igc_adapter *adapter)
4735 {
4736 	struct igc_hw *hw = &adapter->hw;
4737 	int i = 0;
4738 
4739 	/* hardware has been reset, we need to reload some things */
4740 	igc_configure(adapter);
4741 
4742 	clear_bit(__IGC_DOWN, &adapter->state);
4743 
4744 	for (i = 0; i < adapter->num_q_vectors; i++)
4745 		napi_enable(&adapter->q_vector[i]->napi);
4746 
4747 	if (adapter->msix_entries)
4748 		igc_configure_msix(adapter);
4749 	else
4750 		igc_assign_vector(adapter->q_vector[0], 0);
4751 
4752 	/* Clear any pending interrupts. */
4753 	rd32(IGC_ICR);
4754 	igc_irq_enable(adapter);
4755 
4756 	netif_tx_start_all_queues(adapter->netdev);
4757 
4758 	/* start the watchdog. */
4759 	hw->mac.get_link_status = true;
4760 	schedule_work(&adapter->watchdog_task);
4761 }
4762 
4763 /**
4764  * igc_update_stats - Update the board statistics counters
4765  * @adapter: board private structure
4766  */
igc_update_stats(struct igc_adapter * adapter)4767 void igc_update_stats(struct igc_adapter *adapter)
4768 {
4769 	struct rtnl_link_stats64 *net_stats = &adapter->stats64;
4770 	struct pci_dev *pdev = adapter->pdev;
4771 	struct igc_hw *hw = &adapter->hw;
4772 	u64 _bytes, _packets;
4773 	u64 bytes, packets;
4774 	unsigned int start;
4775 	u32 mpc;
4776 	int i;
4777 
4778 	/* Prevent stats update while adapter is being reset, or if the pci
4779 	 * connection is down.
4780 	 */
4781 	if (adapter->link_speed == 0)
4782 		return;
4783 	if (pci_channel_offline(pdev))
4784 		return;
4785 
4786 	packets = 0;
4787 	bytes = 0;
4788 
4789 	rcu_read_lock();
4790 	for (i = 0; i < adapter->num_rx_queues; i++) {
4791 		struct igc_ring *ring = adapter->rx_ring[i];
4792 		u32 rqdpc = rd32(IGC_RQDPC(i));
4793 
4794 		if (hw->mac.type >= igc_i225)
4795 			wr32(IGC_RQDPC(i), 0);
4796 
4797 		if (rqdpc) {
4798 			ring->rx_stats.drops += rqdpc;
4799 			net_stats->rx_fifo_errors += rqdpc;
4800 		}
4801 
4802 		do {
4803 			start = u64_stats_fetch_begin_irq(&ring->rx_syncp);
4804 			_bytes = ring->rx_stats.bytes;
4805 			_packets = ring->rx_stats.packets;
4806 		} while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start));
4807 		bytes += _bytes;
4808 		packets += _packets;
4809 	}
4810 
4811 	net_stats->rx_bytes = bytes;
4812 	net_stats->rx_packets = packets;
4813 
4814 	packets = 0;
4815 	bytes = 0;
4816 	for (i = 0; i < adapter->num_tx_queues; i++) {
4817 		struct igc_ring *ring = adapter->tx_ring[i];
4818 
4819 		do {
4820 			start = u64_stats_fetch_begin_irq(&ring->tx_syncp);
4821 			_bytes = ring->tx_stats.bytes;
4822 			_packets = ring->tx_stats.packets;
4823 		} while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start));
4824 		bytes += _bytes;
4825 		packets += _packets;
4826 	}
4827 	net_stats->tx_bytes = bytes;
4828 	net_stats->tx_packets = packets;
4829 	rcu_read_unlock();
4830 
4831 	/* read stats registers */
4832 	adapter->stats.crcerrs += rd32(IGC_CRCERRS);
4833 	adapter->stats.gprc += rd32(IGC_GPRC);
4834 	adapter->stats.gorc += rd32(IGC_GORCL);
4835 	rd32(IGC_GORCH); /* clear GORCL */
4836 	adapter->stats.bprc += rd32(IGC_BPRC);
4837 	adapter->stats.mprc += rd32(IGC_MPRC);
4838 	adapter->stats.roc += rd32(IGC_ROC);
4839 
4840 	adapter->stats.prc64 += rd32(IGC_PRC64);
4841 	adapter->stats.prc127 += rd32(IGC_PRC127);
4842 	adapter->stats.prc255 += rd32(IGC_PRC255);
4843 	adapter->stats.prc511 += rd32(IGC_PRC511);
4844 	adapter->stats.prc1023 += rd32(IGC_PRC1023);
4845 	adapter->stats.prc1522 += rd32(IGC_PRC1522);
4846 	adapter->stats.tlpic += rd32(IGC_TLPIC);
4847 	adapter->stats.rlpic += rd32(IGC_RLPIC);
4848 	adapter->stats.hgptc += rd32(IGC_HGPTC);
4849 
4850 	mpc = rd32(IGC_MPC);
4851 	adapter->stats.mpc += mpc;
4852 	net_stats->rx_fifo_errors += mpc;
4853 	adapter->stats.scc += rd32(IGC_SCC);
4854 	adapter->stats.ecol += rd32(IGC_ECOL);
4855 	adapter->stats.mcc += rd32(IGC_MCC);
4856 	adapter->stats.latecol += rd32(IGC_LATECOL);
4857 	adapter->stats.dc += rd32(IGC_DC);
4858 	adapter->stats.rlec += rd32(IGC_RLEC);
4859 	adapter->stats.xonrxc += rd32(IGC_XONRXC);
4860 	adapter->stats.xontxc += rd32(IGC_XONTXC);
4861 	adapter->stats.xoffrxc += rd32(IGC_XOFFRXC);
4862 	adapter->stats.xofftxc += rd32(IGC_XOFFTXC);
4863 	adapter->stats.fcruc += rd32(IGC_FCRUC);
4864 	adapter->stats.gptc += rd32(IGC_GPTC);
4865 	adapter->stats.gotc += rd32(IGC_GOTCL);
4866 	rd32(IGC_GOTCH); /* clear GOTCL */
4867 	adapter->stats.rnbc += rd32(IGC_RNBC);
4868 	adapter->stats.ruc += rd32(IGC_RUC);
4869 	adapter->stats.rfc += rd32(IGC_RFC);
4870 	adapter->stats.rjc += rd32(IGC_RJC);
4871 	adapter->stats.tor += rd32(IGC_TORH);
4872 	adapter->stats.tot += rd32(IGC_TOTH);
4873 	adapter->stats.tpr += rd32(IGC_TPR);
4874 
4875 	adapter->stats.ptc64 += rd32(IGC_PTC64);
4876 	adapter->stats.ptc127 += rd32(IGC_PTC127);
4877 	adapter->stats.ptc255 += rd32(IGC_PTC255);
4878 	adapter->stats.ptc511 += rd32(IGC_PTC511);
4879 	adapter->stats.ptc1023 += rd32(IGC_PTC1023);
4880 	adapter->stats.ptc1522 += rd32(IGC_PTC1522);
4881 
4882 	adapter->stats.mptc += rd32(IGC_MPTC);
4883 	adapter->stats.bptc += rd32(IGC_BPTC);
4884 
4885 	adapter->stats.tpt += rd32(IGC_TPT);
4886 	adapter->stats.colc += rd32(IGC_COLC);
4887 	adapter->stats.colc += rd32(IGC_RERC);
4888 
4889 	adapter->stats.algnerrc += rd32(IGC_ALGNERRC);
4890 
4891 	adapter->stats.tsctc += rd32(IGC_TSCTC);
4892 
4893 	adapter->stats.iac += rd32(IGC_IAC);
4894 
4895 	/* Fill out the OS statistics structure */
4896 	net_stats->multicast = adapter->stats.mprc;
4897 	net_stats->collisions = adapter->stats.colc;
4898 
4899 	/* Rx Errors */
4900 
4901 	/* RLEC on some newer hardware can be incorrect so build
4902 	 * our own version based on RUC and ROC
4903 	 */
4904 	net_stats->rx_errors = adapter->stats.rxerrc +
4905 		adapter->stats.crcerrs + adapter->stats.algnerrc +
4906 		adapter->stats.ruc + adapter->stats.roc +
4907 		adapter->stats.cexterr;
4908 	net_stats->rx_length_errors = adapter->stats.ruc +
4909 				      adapter->stats.roc;
4910 	net_stats->rx_crc_errors = adapter->stats.crcerrs;
4911 	net_stats->rx_frame_errors = adapter->stats.algnerrc;
4912 	net_stats->rx_missed_errors = adapter->stats.mpc;
4913 
4914 	/* Tx Errors */
4915 	net_stats->tx_errors = adapter->stats.ecol +
4916 			       adapter->stats.latecol;
4917 	net_stats->tx_aborted_errors = adapter->stats.ecol;
4918 	net_stats->tx_window_errors = adapter->stats.latecol;
4919 	net_stats->tx_carrier_errors = adapter->stats.tncrs;
4920 
4921 	/* Tx Dropped needs to be maintained elsewhere */
4922 
4923 	/* Management Stats */
4924 	adapter->stats.mgptc += rd32(IGC_MGTPTC);
4925 	adapter->stats.mgprc += rd32(IGC_MGTPRC);
4926 	adapter->stats.mgpdc += rd32(IGC_MGTPDC);
4927 }
4928 
4929 /**
4930  * igc_down - Close the interface
4931  * @adapter: board private structure
4932  */
igc_down(struct igc_adapter * adapter)4933 void igc_down(struct igc_adapter *adapter)
4934 {
4935 	struct net_device *netdev = adapter->netdev;
4936 	struct igc_hw *hw = &adapter->hw;
4937 	u32 tctl, rctl;
4938 	int i = 0;
4939 
4940 	set_bit(__IGC_DOWN, &adapter->state);
4941 
4942 	igc_ptp_suspend(adapter);
4943 
4944 	if (pci_device_is_present(adapter->pdev)) {
4945 		/* disable receives in the hardware */
4946 		rctl = rd32(IGC_RCTL);
4947 		wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN);
4948 		/* flush and sleep below */
4949 	}
4950 	/* set trans_start so we don't get spurious watchdogs during reset */
4951 	netif_trans_update(netdev);
4952 
4953 	netif_carrier_off(netdev);
4954 	netif_tx_stop_all_queues(netdev);
4955 
4956 	if (pci_device_is_present(adapter->pdev)) {
4957 		/* disable transmits in the hardware */
4958 		tctl = rd32(IGC_TCTL);
4959 		tctl &= ~IGC_TCTL_EN;
4960 		wr32(IGC_TCTL, tctl);
4961 		/* flush both disables and wait for them to finish */
4962 		wrfl();
4963 		usleep_range(10000, 20000);
4964 
4965 		igc_irq_disable(adapter);
4966 	}
4967 
4968 	adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
4969 
4970 	for (i = 0; i < adapter->num_q_vectors; i++) {
4971 		if (adapter->q_vector[i]) {
4972 			napi_synchronize(&adapter->q_vector[i]->napi);
4973 			napi_disable(&adapter->q_vector[i]->napi);
4974 		}
4975 	}
4976 
4977 	del_timer_sync(&adapter->watchdog_timer);
4978 	del_timer_sync(&adapter->phy_info_timer);
4979 
4980 	/* record the stats before reset*/
4981 	spin_lock(&adapter->stats64_lock);
4982 	igc_update_stats(adapter);
4983 	spin_unlock(&adapter->stats64_lock);
4984 
4985 	adapter->link_speed = 0;
4986 	adapter->link_duplex = 0;
4987 
4988 	if (!pci_channel_offline(adapter->pdev))
4989 		igc_reset(adapter);
4990 
4991 	/* clear VLAN promisc flag so VFTA will be updated if necessary */
4992 	adapter->flags &= ~IGC_FLAG_VLAN_PROMISC;
4993 
4994 	igc_clean_all_tx_rings(adapter);
4995 	igc_clean_all_rx_rings(adapter);
4996 }
4997 
igc_reinit_locked(struct igc_adapter * adapter)4998 void igc_reinit_locked(struct igc_adapter *adapter)
4999 {
5000 	while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
5001 		usleep_range(1000, 2000);
5002 	igc_down(adapter);
5003 	igc_up(adapter);
5004 	clear_bit(__IGC_RESETTING, &adapter->state);
5005 }
5006 
igc_reset_task(struct work_struct * work)5007 static void igc_reset_task(struct work_struct *work)
5008 {
5009 	struct igc_adapter *adapter;
5010 
5011 	adapter = container_of(work, struct igc_adapter, reset_task);
5012 
5013 	rtnl_lock();
5014 	/* If we're already down or resetting, just bail */
5015 	if (test_bit(__IGC_DOWN, &adapter->state) ||
5016 	    test_bit(__IGC_RESETTING, &adapter->state)) {
5017 		rtnl_unlock();
5018 		return;
5019 	}
5020 
5021 	igc_rings_dump(adapter);
5022 	igc_regs_dump(adapter);
5023 	netdev_err(adapter->netdev, "Reset adapter\n");
5024 	igc_reinit_locked(adapter);
5025 	rtnl_unlock();
5026 }
5027 
5028 /**
5029  * igc_change_mtu - Change the Maximum Transfer Unit
5030  * @netdev: network interface device structure
5031  * @new_mtu: new value for maximum frame size
5032  *
5033  * Returns 0 on success, negative on failure
5034  */
igc_change_mtu(struct net_device * netdev,int new_mtu)5035 static int igc_change_mtu(struct net_device *netdev, int new_mtu)
5036 {
5037 	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
5038 	struct igc_adapter *adapter = netdev_priv(netdev);
5039 
5040 	if (igc_xdp_is_enabled(adapter) && new_mtu > ETH_DATA_LEN) {
5041 		netdev_dbg(netdev, "Jumbo frames not supported with XDP");
5042 		return -EINVAL;
5043 	}
5044 
5045 	/* adjust max frame to be at least the size of a standard frame */
5046 	if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
5047 		max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
5048 
5049 	while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
5050 		usleep_range(1000, 2000);
5051 
5052 	/* igc_down has a dependency on max_frame_size */
5053 	adapter->max_frame_size = max_frame;
5054 
5055 	if (netif_running(netdev))
5056 		igc_down(adapter);
5057 
5058 	netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
5059 	netdev->mtu = new_mtu;
5060 
5061 	if (netif_running(netdev))
5062 		igc_up(adapter);
5063 	else
5064 		igc_reset(adapter);
5065 
5066 	clear_bit(__IGC_RESETTING, &adapter->state);
5067 
5068 	return 0;
5069 }
5070 
5071 /**
5072  * igc_get_stats64 - Get System Network Statistics
5073  * @netdev: network interface device structure
5074  * @stats: rtnl_link_stats64 pointer
5075  *
5076  * Returns the address of the device statistics structure.
5077  * The statistics are updated here and also from the timer callback.
5078  */
igc_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * stats)5079 static void igc_get_stats64(struct net_device *netdev,
5080 			    struct rtnl_link_stats64 *stats)
5081 {
5082 	struct igc_adapter *adapter = netdev_priv(netdev);
5083 
5084 	spin_lock(&adapter->stats64_lock);
5085 	if (!test_bit(__IGC_RESETTING, &adapter->state))
5086 		igc_update_stats(adapter);
5087 	memcpy(stats, &adapter->stats64, sizeof(*stats));
5088 	spin_unlock(&adapter->stats64_lock);
5089 }
5090 
igc_fix_features(struct net_device * netdev,netdev_features_t features)5091 static netdev_features_t igc_fix_features(struct net_device *netdev,
5092 					  netdev_features_t features)
5093 {
5094 	/* Since there is no support for separate Rx/Tx vlan accel
5095 	 * enable/disable make sure Tx flag is always in same state as Rx.
5096 	 */
5097 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
5098 		features |= NETIF_F_HW_VLAN_CTAG_TX;
5099 	else
5100 		features &= ~NETIF_F_HW_VLAN_CTAG_TX;
5101 
5102 	return features;
5103 }
5104 
igc_set_features(struct net_device * netdev,netdev_features_t features)5105 static int igc_set_features(struct net_device *netdev,
5106 			    netdev_features_t features)
5107 {
5108 	netdev_features_t changed = netdev->features ^ features;
5109 	struct igc_adapter *adapter = netdev_priv(netdev);
5110 
5111 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
5112 		igc_vlan_mode(netdev, features);
5113 
5114 	/* Add VLAN support */
5115 	if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
5116 		return 0;
5117 
5118 	if (!(features & NETIF_F_NTUPLE))
5119 		igc_flush_nfc_rules(adapter);
5120 
5121 	netdev->features = features;
5122 
5123 	if (netif_running(netdev))
5124 		igc_reinit_locked(adapter);
5125 	else
5126 		igc_reset(adapter);
5127 
5128 	return 1;
5129 }
5130 
5131 static netdev_features_t
igc_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)5132 igc_features_check(struct sk_buff *skb, struct net_device *dev,
5133 		   netdev_features_t features)
5134 {
5135 	unsigned int network_hdr_len, mac_hdr_len;
5136 
5137 	/* Make certain the headers can be described by a context descriptor */
5138 	mac_hdr_len = skb_network_header(skb) - skb->data;
5139 	if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN))
5140 		return features & ~(NETIF_F_HW_CSUM |
5141 				    NETIF_F_SCTP_CRC |
5142 				    NETIF_F_HW_VLAN_CTAG_TX |
5143 				    NETIF_F_TSO |
5144 				    NETIF_F_TSO6);
5145 
5146 	network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
5147 	if (unlikely(network_hdr_len >  IGC_MAX_NETWORK_HDR_LEN))
5148 		return features & ~(NETIF_F_HW_CSUM |
5149 				    NETIF_F_SCTP_CRC |
5150 				    NETIF_F_TSO |
5151 				    NETIF_F_TSO6);
5152 
5153 	/* We can only support IPv4 TSO in tunnels if we can mangle the
5154 	 * inner IP ID field, so strip TSO if MANGLEID is not supported.
5155 	 */
5156 	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
5157 		features &= ~NETIF_F_TSO;
5158 
5159 	return features;
5160 }
5161 
igc_tsync_interrupt(struct igc_adapter * adapter)5162 static void igc_tsync_interrupt(struct igc_adapter *adapter)
5163 {
5164 	u32 ack, tsauxc, sec, nsec, tsicr;
5165 	struct igc_hw *hw = &adapter->hw;
5166 	struct ptp_clock_event event;
5167 	struct timespec64 ts;
5168 
5169 	tsicr = rd32(IGC_TSICR);
5170 	ack = 0;
5171 
5172 	if (tsicr & IGC_TSICR_SYS_WRAP) {
5173 		event.type = PTP_CLOCK_PPS;
5174 		if (adapter->ptp_caps.pps)
5175 			ptp_clock_event(adapter->ptp_clock, &event);
5176 		ack |= IGC_TSICR_SYS_WRAP;
5177 	}
5178 
5179 	if (tsicr & IGC_TSICR_TXTS) {
5180 		/* retrieve hardware timestamp */
5181 		schedule_work(&adapter->ptp_tx_work);
5182 		ack |= IGC_TSICR_TXTS;
5183 	}
5184 
5185 	if (tsicr & IGC_TSICR_TT0) {
5186 		spin_lock(&adapter->tmreg_lock);
5187 		ts = timespec64_add(adapter->perout[0].start,
5188 				    adapter->perout[0].period);
5189 		wr32(IGC_TRGTTIML0, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0);
5190 		wr32(IGC_TRGTTIMH0, (u32)ts.tv_sec);
5191 		tsauxc = rd32(IGC_TSAUXC);
5192 		tsauxc |= IGC_TSAUXC_EN_TT0;
5193 		wr32(IGC_TSAUXC, tsauxc);
5194 		adapter->perout[0].start = ts;
5195 		spin_unlock(&adapter->tmreg_lock);
5196 		ack |= IGC_TSICR_TT0;
5197 	}
5198 
5199 	if (tsicr & IGC_TSICR_TT1) {
5200 		spin_lock(&adapter->tmreg_lock);
5201 		ts = timespec64_add(adapter->perout[1].start,
5202 				    adapter->perout[1].period);
5203 		wr32(IGC_TRGTTIML1, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0);
5204 		wr32(IGC_TRGTTIMH1, (u32)ts.tv_sec);
5205 		tsauxc = rd32(IGC_TSAUXC);
5206 		tsauxc |= IGC_TSAUXC_EN_TT1;
5207 		wr32(IGC_TSAUXC, tsauxc);
5208 		adapter->perout[1].start = ts;
5209 		spin_unlock(&adapter->tmreg_lock);
5210 		ack |= IGC_TSICR_TT1;
5211 	}
5212 
5213 	if (tsicr & IGC_TSICR_AUTT0) {
5214 		nsec = rd32(IGC_AUXSTMPL0);
5215 		sec  = rd32(IGC_AUXSTMPH0);
5216 		event.type = PTP_CLOCK_EXTTS;
5217 		event.index = 0;
5218 		event.timestamp = sec * NSEC_PER_SEC + nsec;
5219 		ptp_clock_event(adapter->ptp_clock, &event);
5220 		ack |= IGC_TSICR_AUTT0;
5221 	}
5222 
5223 	if (tsicr & IGC_TSICR_AUTT1) {
5224 		nsec = rd32(IGC_AUXSTMPL1);
5225 		sec  = rd32(IGC_AUXSTMPH1);
5226 		event.type = PTP_CLOCK_EXTTS;
5227 		event.index = 1;
5228 		event.timestamp = sec * NSEC_PER_SEC + nsec;
5229 		ptp_clock_event(adapter->ptp_clock, &event);
5230 		ack |= IGC_TSICR_AUTT1;
5231 	}
5232 
5233 	/* acknowledge the interrupts */
5234 	wr32(IGC_TSICR, ack);
5235 }
5236 
5237 /**
5238  * igc_msix_other - msix other interrupt handler
5239  * @irq: interrupt number
5240  * @data: pointer to a q_vector
5241  */
igc_msix_other(int irq,void * data)5242 static irqreturn_t igc_msix_other(int irq, void *data)
5243 {
5244 	struct igc_adapter *adapter = data;
5245 	struct igc_hw *hw = &adapter->hw;
5246 	u32 icr = rd32(IGC_ICR);
5247 
5248 	/* reading ICR causes bit 31 of EICR to be cleared */
5249 	if (icr & IGC_ICR_DRSTA)
5250 		schedule_work(&adapter->reset_task);
5251 
5252 	if (icr & IGC_ICR_DOUTSYNC) {
5253 		/* HW is reporting DMA is out of sync */
5254 		adapter->stats.doosync++;
5255 	}
5256 
5257 	if (icr & IGC_ICR_LSC) {
5258 		hw->mac.get_link_status = true;
5259 		/* guard against interrupt when we're going down */
5260 		if (!test_bit(__IGC_DOWN, &adapter->state))
5261 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
5262 	}
5263 
5264 	if (icr & IGC_ICR_TS)
5265 		igc_tsync_interrupt(adapter);
5266 
5267 	wr32(IGC_EIMS, adapter->eims_other);
5268 
5269 	return IRQ_HANDLED;
5270 }
5271 
igc_write_itr(struct igc_q_vector * q_vector)5272 static void igc_write_itr(struct igc_q_vector *q_vector)
5273 {
5274 	u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK;
5275 
5276 	if (!q_vector->set_itr)
5277 		return;
5278 
5279 	if (!itr_val)
5280 		itr_val = IGC_ITR_VAL_MASK;
5281 
5282 	itr_val |= IGC_EITR_CNT_IGNR;
5283 
5284 	writel(itr_val, q_vector->itr_register);
5285 	q_vector->set_itr = 0;
5286 }
5287 
igc_msix_ring(int irq,void * data)5288 static irqreturn_t igc_msix_ring(int irq, void *data)
5289 {
5290 	struct igc_q_vector *q_vector = data;
5291 
5292 	/* Write the ITR value calculated from the previous interrupt. */
5293 	igc_write_itr(q_vector);
5294 
5295 	napi_schedule(&q_vector->napi);
5296 
5297 	return IRQ_HANDLED;
5298 }
5299 
5300 /**
5301  * igc_request_msix - Initialize MSI-X interrupts
5302  * @adapter: Pointer to adapter structure
5303  *
5304  * igc_request_msix allocates MSI-X vectors and requests interrupts from the
5305  * kernel.
5306  */
igc_request_msix(struct igc_adapter * adapter)5307 static int igc_request_msix(struct igc_adapter *adapter)
5308 {
5309 	unsigned int num_q_vectors = adapter->num_q_vectors;
5310 	int i = 0, err = 0, vector = 0, free_vector = 0;
5311 	struct net_device *netdev = adapter->netdev;
5312 
5313 	err = request_irq(adapter->msix_entries[vector].vector,
5314 			  &igc_msix_other, 0, netdev->name, adapter);
5315 	if (err)
5316 		goto err_out;
5317 
5318 	if (num_q_vectors > MAX_Q_VECTORS) {
5319 		num_q_vectors = MAX_Q_VECTORS;
5320 		dev_warn(&adapter->pdev->dev,
5321 			 "The number of queue vectors (%d) is higher than max allowed (%d)\n",
5322 			 adapter->num_q_vectors, MAX_Q_VECTORS);
5323 	}
5324 	for (i = 0; i < num_q_vectors; i++) {
5325 		struct igc_q_vector *q_vector = adapter->q_vector[i];
5326 
5327 		vector++;
5328 
5329 		q_vector->itr_register = adapter->io_addr + IGC_EITR(vector);
5330 
5331 		if (q_vector->rx.ring && q_vector->tx.ring)
5332 			sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
5333 				q_vector->rx.ring->queue_index);
5334 		else if (q_vector->tx.ring)
5335 			sprintf(q_vector->name, "%s-tx-%u", netdev->name,
5336 				q_vector->tx.ring->queue_index);
5337 		else if (q_vector->rx.ring)
5338 			sprintf(q_vector->name, "%s-rx-%u", netdev->name,
5339 				q_vector->rx.ring->queue_index);
5340 		else
5341 			sprintf(q_vector->name, "%s-unused", netdev->name);
5342 
5343 		err = request_irq(adapter->msix_entries[vector].vector,
5344 				  igc_msix_ring, 0, q_vector->name,
5345 				  q_vector);
5346 		if (err)
5347 			goto err_free;
5348 	}
5349 
5350 	igc_configure_msix(adapter);
5351 	return 0;
5352 
5353 err_free:
5354 	/* free already assigned IRQs */
5355 	free_irq(adapter->msix_entries[free_vector++].vector, adapter);
5356 
5357 	vector--;
5358 	for (i = 0; i < vector; i++) {
5359 		free_irq(adapter->msix_entries[free_vector++].vector,
5360 			 adapter->q_vector[i]);
5361 	}
5362 err_out:
5363 	return err;
5364 }
5365 
5366 /**
5367  * igc_clear_interrupt_scheme - reset the device to a state of no interrupts
5368  * @adapter: Pointer to adapter structure
5369  *
5370  * This function resets the device so that it has 0 rx queues, tx queues, and
5371  * MSI-X interrupts allocated.
5372  */
igc_clear_interrupt_scheme(struct igc_adapter * adapter)5373 static void igc_clear_interrupt_scheme(struct igc_adapter *adapter)
5374 {
5375 	igc_free_q_vectors(adapter);
5376 	igc_reset_interrupt_capability(adapter);
5377 }
5378 
5379 /* Need to wait a few seconds after link up to get diagnostic information from
5380  * the phy
5381  */
igc_update_phy_info(struct timer_list * t)5382 static void igc_update_phy_info(struct timer_list *t)
5383 {
5384 	struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer);
5385 
5386 	igc_get_phy_info(&adapter->hw);
5387 }
5388 
5389 /**
5390  * igc_has_link - check shared code for link and determine up/down
5391  * @adapter: pointer to driver private info
5392  */
igc_has_link(struct igc_adapter * adapter)5393 bool igc_has_link(struct igc_adapter *adapter)
5394 {
5395 	struct igc_hw *hw = &adapter->hw;
5396 	bool link_active = false;
5397 
5398 	/* get_link_status is set on LSC (link status) interrupt or
5399 	 * rx sequence error interrupt.  get_link_status will stay
5400 	 * false until the igc_check_for_link establishes link
5401 	 * for copper adapters ONLY
5402 	 */
5403 	if (!hw->mac.get_link_status)
5404 		return true;
5405 	hw->mac.ops.check_for_link(hw);
5406 	link_active = !hw->mac.get_link_status;
5407 
5408 	if (hw->mac.type == igc_i225) {
5409 		if (!netif_carrier_ok(adapter->netdev)) {
5410 			adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
5411 		} else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) {
5412 			adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE;
5413 			adapter->link_check_timeout = jiffies;
5414 		}
5415 	}
5416 
5417 	return link_active;
5418 }
5419 
5420 /**
5421  * igc_watchdog - Timer Call-back
5422  * @t: timer for the watchdog
5423  */
igc_watchdog(struct timer_list * t)5424 static void igc_watchdog(struct timer_list *t)
5425 {
5426 	struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer);
5427 	/* Do the rest outside of interrupt context */
5428 	schedule_work(&adapter->watchdog_task);
5429 }
5430 
igc_watchdog_task(struct work_struct * work)5431 static void igc_watchdog_task(struct work_struct *work)
5432 {
5433 	struct igc_adapter *adapter = container_of(work,
5434 						   struct igc_adapter,
5435 						   watchdog_task);
5436 	struct net_device *netdev = adapter->netdev;
5437 	struct igc_hw *hw = &adapter->hw;
5438 	struct igc_phy_info *phy = &hw->phy;
5439 	u16 phy_data, retry_count = 20;
5440 	u32 link;
5441 	int i;
5442 
5443 	link = igc_has_link(adapter);
5444 
5445 	if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) {
5446 		if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
5447 			adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
5448 		else
5449 			link = false;
5450 	}
5451 
5452 	if (link) {
5453 		/* Cancel scheduled suspend requests. */
5454 		pm_runtime_resume(netdev->dev.parent);
5455 
5456 		if (!netif_carrier_ok(netdev)) {
5457 			u32 ctrl;
5458 
5459 			hw->mac.ops.get_speed_and_duplex(hw,
5460 							 &adapter->link_speed,
5461 							 &adapter->link_duplex);
5462 
5463 			ctrl = rd32(IGC_CTRL);
5464 			/* Link status message must follow this format */
5465 			netdev_info(netdev,
5466 				    "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
5467 				    adapter->link_speed,
5468 				    adapter->link_duplex == FULL_DUPLEX ?
5469 				    "Full" : "Half",
5470 				    (ctrl & IGC_CTRL_TFCE) &&
5471 				    (ctrl & IGC_CTRL_RFCE) ? "RX/TX" :
5472 				    (ctrl & IGC_CTRL_RFCE) ?  "RX" :
5473 				    (ctrl & IGC_CTRL_TFCE) ?  "TX" : "None");
5474 
5475 			/* disable EEE if enabled */
5476 			if ((adapter->flags & IGC_FLAG_EEE) &&
5477 			    adapter->link_duplex == HALF_DUPLEX) {
5478 				netdev_info(netdev,
5479 					    "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n");
5480 				adapter->hw.dev_spec._base.eee_enable = false;
5481 				adapter->flags &= ~IGC_FLAG_EEE;
5482 			}
5483 
5484 			/* check if SmartSpeed worked */
5485 			igc_check_downshift(hw);
5486 			if (phy->speed_downgraded)
5487 				netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
5488 
5489 			/* adjust timeout factor according to speed/duplex */
5490 			adapter->tx_timeout_factor = 1;
5491 			switch (adapter->link_speed) {
5492 			case SPEED_10:
5493 				adapter->tx_timeout_factor = 14;
5494 				break;
5495 			case SPEED_100:
5496 			case SPEED_1000:
5497 			case SPEED_2500:
5498 				adapter->tx_timeout_factor = 7;
5499 				break;
5500 			}
5501 
5502 			if (adapter->link_speed != SPEED_1000)
5503 				goto no_wait;
5504 
5505 			/* wait for Remote receiver status OK */
5506 retry_read_status:
5507 			if (!igc_read_phy_reg(hw, PHY_1000T_STATUS,
5508 					      &phy_data)) {
5509 				if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
5510 				    retry_count) {
5511 					msleep(100);
5512 					retry_count--;
5513 					goto retry_read_status;
5514 				} else if (!retry_count) {
5515 					netdev_err(netdev, "exceed max 2 second\n");
5516 				}
5517 			} else {
5518 				netdev_err(netdev, "read 1000Base-T Status Reg\n");
5519 			}
5520 no_wait:
5521 			netif_carrier_on(netdev);
5522 
5523 			/* link state has changed, schedule phy info update */
5524 			if (!test_bit(__IGC_DOWN, &adapter->state))
5525 				mod_timer(&adapter->phy_info_timer,
5526 					  round_jiffies(jiffies + 2 * HZ));
5527 		}
5528 	} else {
5529 		if (netif_carrier_ok(netdev)) {
5530 			adapter->link_speed = 0;
5531 			adapter->link_duplex = 0;
5532 
5533 			/* Links status message must follow this format */
5534 			netdev_info(netdev, "NIC Link is Down\n");
5535 			netif_carrier_off(netdev);
5536 
5537 			/* link state has changed, schedule phy info update */
5538 			if (!test_bit(__IGC_DOWN, &adapter->state))
5539 				mod_timer(&adapter->phy_info_timer,
5540 					  round_jiffies(jiffies + 2 * HZ));
5541 
5542 			/* link is down, time to check for alternate media */
5543 			if (adapter->flags & IGC_FLAG_MAS_ENABLE) {
5544 				if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
5545 					schedule_work(&adapter->reset_task);
5546 					/* return immediately */
5547 					return;
5548 				}
5549 			}
5550 			pm_schedule_suspend(netdev->dev.parent,
5551 					    MSEC_PER_SEC * 5);
5552 
5553 		/* also check for alternate media here */
5554 		} else if (!netif_carrier_ok(netdev) &&
5555 			   (adapter->flags & IGC_FLAG_MAS_ENABLE)) {
5556 			if (adapter->flags & IGC_FLAG_MEDIA_RESET) {
5557 				schedule_work(&adapter->reset_task);
5558 				/* return immediately */
5559 				return;
5560 			}
5561 		}
5562 	}
5563 
5564 	spin_lock(&adapter->stats64_lock);
5565 	igc_update_stats(adapter);
5566 	spin_unlock(&adapter->stats64_lock);
5567 
5568 	for (i = 0; i < adapter->num_tx_queues; i++) {
5569 		struct igc_ring *tx_ring = adapter->tx_ring[i];
5570 
5571 		if (!netif_carrier_ok(netdev)) {
5572 			/* We've lost link, so the controller stops DMA,
5573 			 * but we've got queued Tx work that's never going
5574 			 * to get done, so reset controller to flush Tx.
5575 			 * (Do the reset outside of interrupt context).
5576 			 */
5577 			if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) {
5578 				adapter->tx_timeout_count++;
5579 				schedule_work(&adapter->reset_task);
5580 				/* return immediately since reset is imminent */
5581 				return;
5582 			}
5583 		}
5584 
5585 		/* Force detection of hung controller every watchdog period */
5586 		set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5587 	}
5588 
5589 	/* Cause software interrupt to ensure Rx ring is cleaned */
5590 	if (adapter->flags & IGC_FLAG_HAS_MSIX) {
5591 		u32 eics = 0;
5592 
5593 		for (i = 0; i < adapter->num_q_vectors; i++)
5594 			eics |= adapter->q_vector[i]->eims_value;
5595 		wr32(IGC_EICS, eics);
5596 	} else {
5597 		wr32(IGC_ICS, IGC_ICS_RXDMT0);
5598 	}
5599 
5600 	igc_ptp_tx_hang(adapter);
5601 
5602 	/* Reset the timer */
5603 	if (!test_bit(__IGC_DOWN, &adapter->state)) {
5604 		if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)
5605 			mod_timer(&adapter->watchdog_timer,
5606 				  round_jiffies(jiffies +  HZ));
5607 		else
5608 			mod_timer(&adapter->watchdog_timer,
5609 				  round_jiffies(jiffies + 2 * HZ));
5610 	}
5611 }
5612 
5613 /**
5614  * igc_intr_msi - Interrupt Handler
5615  * @irq: interrupt number
5616  * @data: pointer to a network interface device structure
5617  */
igc_intr_msi(int irq,void * data)5618 static irqreturn_t igc_intr_msi(int irq, void *data)
5619 {
5620 	struct igc_adapter *adapter = data;
5621 	struct igc_q_vector *q_vector = adapter->q_vector[0];
5622 	struct igc_hw *hw = &adapter->hw;
5623 	/* read ICR disables interrupts using IAM */
5624 	u32 icr = rd32(IGC_ICR);
5625 
5626 	igc_write_itr(q_vector);
5627 
5628 	if (icr & IGC_ICR_DRSTA)
5629 		schedule_work(&adapter->reset_task);
5630 
5631 	if (icr & IGC_ICR_DOUTSYNC) {
5632 		/* HW is reporting DMA is out of sync */
5633 		adapter->stats.doosync++;
5634 	}
5635 
5636 	if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
5637 		hw->mac.get_link_status = true;
5638 		if (!test_bit(__IGC_DOWN, &adapter->state))
5639 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
5640 	}
5641 
5642 	if (icr & IGC_ICR_TS)
5643 		igc_tsync_interrupt(adapter);
5644 
5645 	napi_schedule(&q_vector->napi);
5646 
5647 	return IRQ_HANDLED;
5648 }
5649 
5650 /**
5651  * igc_intr - Legacy Interrupt Handler
5652  * @irq: interrupt number
5653  * @data: pointer to a network interface device structure
5654  */
igc_intr(int irq,void * data)5655 static irqreturn_t igc_intr(int irq, void *data)
5656 {
5657 	struct igc_adapter *adapter = data;
5658 	struct igc_q_vector *q_vector = adapter->q_vector[0];
5659 	struct igc_hw *hw = &adapter->hw;
5660 	/* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
5661 	 * need for the IMC write
5662 	 */
5663 	u32 icr = rd32(IGC_ICR);
5664 
5665 	/* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5666 	 * not set, then the adapter didn't send an interrupt
5667 	 */
5668 	if (!(icr & IGC_ICR_INT_ASSERTED))
5669 		return IRQ_NONE;
5670 
5671 	igc_write_itr(q_vector);
5672 
5673 	if (icr & IGC_ICR_DRSTA)
5674 		schedule_work(&adapter->reset_task);
5675 
5676 	if (icr & IGC_ICR_DOUTSYNC) {
5677 		/* HW is reporting DMA is out of sync */
5678 		adapter->stats.doosync++;
5679 	}
5680 
5681 	if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
5682 		hw->mac.get_link_status = true;
5683 		/* guard against interrupt when we're going down */
5684 		if (!test_bit(__IGC_DOWN, &adapter->state))
5685 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
5686 	}
5687 
5688 	if (icr & IGC_ICR_TS)
5689 		igc_tsync_interrupt(adapter);
5690 
5691 	napi_schedule(&q_vector->napi);
5692 
5693 	return IRQ_HANDLED;
5694 }
5695 
igc_free_irq(struct igc_adapter * adapter)5696 static void igc_free_irq(struct igc_adapter *adapter)
5697 {
5698 	if (adapter->msix_entries) {
5699 		int vector = 0, i;
5700 
5701 		free_irq(adapter->msix_entries[vector++].vector, adapter);
5702 
5703 		for (i = 0; i < adapter->num_q_vectors; i++)
5704 			free_irq(adapter->msix_entries[vector++].vector,
5705 				 adapter->q_vector[i]);
5706 	} else {
5707 		free_irq(adapter->pdev->irq, adapter);
5708 	}
5709 }
5710 
5711 /**
5712  * igc_request_irq - initialize interrupts
5713  * @adapter: Pointer to adapter structure
5714  *
5715  * Attempts to configure interrupts using the best available
5716  * capabilities of the hardware and kernel.
5717  */
igc_request_irq(struct igc_adapter * adapter)5718 static int igc_request_irq(struct igc_adapter *adapter)
5719 {
5720 	struct net_device *netdev = adapter->netdev;
5721 	struct pci_dev *pdev = adapter->pdev;
5722 	int err = 0;
5723 
5724 	if (adapter->flags & IGC_FLAG_HAS_MSIX) {
5725 		err = igc_request_msix(adapter);
5726 		if (!err)
5727 			goto request_done;
5728 		/* fall back to MSI */
5729 		igc_free_all_tx_resources(adapter);
5730 		igc_free_all_rx_resources(adapter);
5731 
5732 		igc_clear_interrupt_scheme(adapter);
5733 		err = igc_init_interrupt_scheme(adapter, false);
5734 		if (err)
5735 			goto request_done;
5736 		igc_setup_all_tx_resources(adapter);
5737 		igc_setup_all_rx_resources(adapter);
5738 		igc_configure(adapter);
5739 	}
5740 
5741 	igc_assign_vector(adapter->q_vector[0], 0);
5742 
5743 	if (adapter->flags & IGC_FLAG_HAS_MSI) {
5744 		err = request_irq(pdev->irq, &igc_intr_msi, 0,
5745 				  netdev->name, adapter);
5746 		if (!err)
5747 			goto request_done;
5748 
5749 		/* fall back to legacy interrupts */
5750 		igc_reset_interrupt_capability(adapter);
5751 		adapter->flags &= ~IGC_FLAG_HAS_MSI;
5752 	}
5753 
5754 	err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED,
5755 			  netdev->name, adapter);
5756 
5757 	if (err)
5758 		netdev_err(netdev, "Error %d getting interrupt\n", err);
5759 
5760 request_done:
5761 	return err;
5762 }
5763 
5764 /**
5765  * __igc_open - Called when a network interface is made active
5766  * @netdev: network interface device structure
5767  * @resuming: boolean indicating if the device is resuming
5768  *
5769  * Returns 0 on success, negative value on failure
5770  *
5771  * The open entry point is called when a network interface is made
5772  * active by the system (IFF_UP).  At this point all resources needed
5773  * for transmit and receive operations are allocated, the interrupt
5774  * handler is registered with the OS, the watchdog timer is started,
5775  * and the stack is notified that the interface is ready.
5776  */
__igc_open(struct net_device * netdev,bool resuming)5777 static int __igc_open(struct net_device *netdev, bool resuming)
5778 {
5779 	struct igc_adapter *adapter = netdev_priv(netdev);
5780 	struct pci_dev *pdev = adapter->pdev;
5781 	struct igc_hw *hw = &adapter->hw;
5782 	int err = 0;
5783 	int i = 0;
5784 
5785 	/* disallow open during test */
5786 
5787 	if (test_bit(__IGC_TESTING, &adapter->state)) {
5788 		WARN_ON(resuming);
5789 		return -EBUSY;
5790 	}
5791 
5792 	if (!resuming)
5793 		pm_runtime_get_sync(&pdev->dev);
5794 
5795 	netif_carrier_off(netdev);
5796 
5797 	/* allocate transmit descriptors */
5798 	err = igc_setup_all_tx_resources(adapter);
5799 	if (err)
5800 		goto err_setup_tx;
5801 
5802 	/* allocate receive descriptors */
5803 	err = igc_setup_all_rx_resources(adapter);
5804 	if (err)
5805 		goto err_setup_rx;
5806 
5807 	igc_power_up_link(adapter);
5808 
5809 	igc_configure(adapter);
5810 
5811 	err = igc_request_irq(adapter);
5812 	if (err)
5813 		goto err_req_irq;
5814 
5815 	/* Notify the stack of the actual queue counts. */
5816 	err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
5817 	if (err)
5818 		goto err_set_queues;
5819 
5820 	err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
5821 	if (err)
5822 		goto err_set_queues;
5823 
5824 	clear_bit(__IGC_DOWN, &adapter->state);
5825 
5826 	for (i = 0; i < adapter->num_q_vectors; i++)
5827 		napi_enable(&adapter->q_vector[i]->napi);
5828 
5829 	/* Clear any pending interrupts. */
5830 	rd32(IGC_ICR);
5831 	igc_irq_enable(adapter);
5832 
5833 	if (!resuming)
5834 		pm_runtime_put(&pdev->dev);
5835 
5836 	netif_tx_start_all_queues(netdev);
5837 
5838 	/* start the watchdog. */
5839 	hw->mac.get_link_status = true;
5840 	schedule_work(&adapter->watchdog_task);
5841 
5842 	return IGC_SUCCESS;
5843 
5844 err_set_queues:
5845 	igc_free_irq(adapter);
5846 err_req_irq:
5847 	igc_release_hw_control(adapter);
5848 	igc_power_down_phy_copper_base(&adapter->hw);
5849 	igc_free_all_rx_resources(adapter);
5850 err_setup_rx:
5851 	igc_free_all_tx_resources(adapter);
5852 err_setup_tx:
5853 	igc_reset(adapter);
5854 	if (!resuming)
5855 		pm_runtime_put(&pdev->dev);
5856 
5857 	return err;
5858 }
5859 
igc_open(struct net_device * netdev)5860 int igc_open(struct net_device *netdev)
5861 {
5862 	return __igc_open(netdev, false);
5863 }
5864 
5865 /**
5866  * __igc_close - Disables a network interface
5867  * @netdev: network interface device structure
5868  * @suspending: boolean indicating the device is suspending
5869  *
5870  * Returns 0, this is not allowed to fail
5871  *
5872  * The close entry point is called when an interface is de-activated
5873  * by the OS.  The hardware is still under the driver's control, but
5874  * needs to be disabled.  A global MAC reset is issued to stop the
5875  * hardware, and all transmit and receive resources are freed.
5876  */
__igc_close(struct net_device * netdev,bool suspending)5877 static int __igc_close(struct net_device *netdev, bool suspending)
5878 {
5879 	struct igc_adapter *adapter = netdev_priv(netdev);
5880 	struct pci_dev *pdev = adapter->pdev;
5881 
5882 	WARN_ON(test_bit(__IGC_RESETTING, &adapter->state));
5883 
5884 	if (!suspending)
5885 		pm_runtime_get_sync(&pdev->dev);
5886 
5887 	igc_down(adapter);
5888 
5889 	igc_release_hw_control(adapter);
5890 
5891 	igc_free_irq(adapter);
5892 
5893 	igc_free_all_tx_resources(adapter);
5894 	igc_free_all_rx_resources(adapter);
5895 
5896 	if (!suspending)
5897 		pm_runtime_put_sync(&pdev->dev);
5898 
5899 	return 0;
5900 }
5901 
igc_close(struct net_device * netdev)5902 int igc_close(struct net_device *netdev)
5903 {
5904 	if (netif_device_present(netdev) || netdev->dismantle)
5905 		return __igc_close(netdev, false);
5906 	return 0;
5907 }
5908 
5909 /**
5910  * igc_ioctl - Access the hwtstamp interface
5911  * @netdev: network interface device structure
5912  * @ifr: interface request data
5913  * @cmd: ioctl command
5914  **/
igc_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)5915 static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5916 {
5917 	switch (cmd) {
5918 	case SIOCGHWTSTAMP:
5919 		return igc_ptp_get_ts_config(netdev, ifr);
5920 	case SIOCSHWTSTAMP:
5921 		return igc_ptp_set_ts_config(netdev, ifr);
5922 	default:
5923 		return -EOPNOTSUPP;
5924 	}
5925 }
5926 
igc_save_launchtime_params(struct igc_adapter * adapter,int queue,bool enable)5927 static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue,
5928 				      bool enable)
5929 {
5930 	struct igc_ring *ring;
5931 
5932 	if (queue < 0 || queue >= adapter->num_tx_queues)
5933 		return -EINVAL;
5934 
5935 	ring = adapter->tx_ring[queue];
5936 	ring->launchtime_enable = enable;
5937 
5938 	return 0;
5939 }
5940 
is_base_time_past(ktime_t base_time,const struct timespec64 * now)5941 static bool is_base_time_past(ktime_t base_time, const struct timespec64 *now)
5942 {
5943 	struct timespec64 b;
5944 
5945 	b = ktime_to_timespec64(base_time);
5946 
5947 	return timespec64_compare(now, &b) > 0;
5948 }
5949 
validate_schedule(struct igc_adapter * adapter,const struct tc_taprio_qopt_offload * qopt)5950 static bool validate_schedule(struct igc_adapter *adapter,
5951 			      const struct tc_taprio_qopt_offload *qopt)
5952 {
5953 	int queue_uses[IGC_MAX_TX_QUEUES] = { };
5954 	struct timespec64 now;
5955 	size_t n;
5956 
5957 	if (qopt->cycle_time_extension)
5958 		return false;
5959 
5960 	igc_ptp_read(adapter, &now);
5961 
5962 	/* If we program the controller's BASET registers with a time
5963 	 * in the future, it will hold all the packets until that
5964 	 * time, causing a lot of TX Hangs, so to avoid that, we
5965 	 * reject schedules that would start in the future.
5966 	 */
5967 	if (!is_base_time_past(qopt->base_time, &now))
5968 		return false;
5969 
5970 	for (n = 0; n < qopt->num_entries; n++) {
5971 		const struct tc_taprio_sched_entry *e, *prev;
5972 		int i;
5973 
5974 		prev = n ? &qopt->entries[n - 1] : NULL;
5975 		e = &qopt->entries[n];
5976 
5977 		/* i225 only supports "global" frame preemption
5978 		 * settings.
5979 		 */
5980 		if (e->command != TC_TAPRIO_CMD_SET_GATES)
5981 			return false;
5982 
5983 		for (i = 0; i < adapter->num_tx_queues; i++) {
5984 			if (e->gate_mask & BIT(i))
5985 				queue_uses[i]++;
5986 
5987 			/* There are limitations: A single queue cannot be
5988 			 * opened and closed multiple times per cycle unless the
5989 			 * gate stays open. Check for it.
5990 			 */
5991 			if (queue_uses[i] > 1 &&
5992 			    !(prev->gate_mask & BIT(i)))
5993 				return false;
5994 		}
5995 	}
5996 
5997 	return true;
5998 }
5999 
igc_tsn_enable_launchtime(struct igc_adapter * adapter,struct tc_etf_qopt_offload * qopt)6000 static int igc_tsn_enable_launchtime(struct igc_adapter *adapter,
6001 				     struct tc_etf_qopt_offload *qopt)
6002 {
6003 	struct igc_hw *hw = &adapter->hw;
6004 	int err;
6005 
6006 	if (hw->mac.type != igc_i225)
6007 		return -EOPNOTSUPP;
6008 
6009 	err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable);
6010 	if (err)
6011 		return err;
6012 
6013 	return igc_tsn_offload_apply(adapter);
6014 }
6015 
igc_tsn_clear_schedule(struct igc_adapter * adapter)6016 static int igc_tsn_clear_schedule(struct igc_adapter *adapter)
6017 {
6018 	int i;
6019 
6020 	adapter->base_time = 0;
6021 	adapter->cycle_time = NSEC_PER_SEC;
6022 
6023 	for (i = 0; i < adapter->num_tx_queues; i++) {
6024 		struct igc_ring *ring = adapter->tx_ring[i];
6025 
6026 		ring->start_time = 0;
6027 		ring->end_time = NSEC_PER_SEC;
6028 	}
6029 
6030 	return 0;
6031 }
6032 
igc_save_qbv_schedule(struct igc_adapter * adapter,struct tc_taprio_qopt_offload * qopt)6033 static int igc_save_qbv_schedule(struct igc_adapter *adapter,
6034 				 struct tc_taprio_qopt_offload *qopt)
6035 {
6036 	bool queue_configured[IGC_MAX_TX_QUEUES] = { };
6037 	u32 start_time = 0, end_time = 0;
6038 	size_t n;
6039 	int i;
6040 
6041 	adapter->qbv_enable = qopt->enable;
6042 
6043 	if (!qopt->enable)
6044 		return igc_tsn_clear_schedule(adapter);
6045 
6046 	if (qopt->base_time < 0)
6047 		return -ERANGE;
6048 
6049 	if (adapter->base_time)
6050 		return -EALREADY;
6051 
6052 	if (!validate_schedule(adapter, qopt))
6053 		return -EINVAL;
6054 
6055 	adapter->cycle_time = qopt->cycle_time;
6056 	adapter->base_time = qopt->base_time;
6057 
6058 	for (n = 0; n < qopt->num_entries; n++) {
6059 		struct tc_taprio_sched_entry *e = &qopt->entries[n];
6060 
6061 		end_time += e->interval;
6062 
6063 		/* If any of the conditions below are true, we need to manually
6064 		 * control the end time of the cycle.
6065 		 * 1. Qbv users can specify a cycle time that is not equal
6066 		 * to the total GCL intervals. Hence, recalculation is
6067 		 * necessary here to exclude the time interval that
6068 		 * exceeds the cycle time.
6069 		 * 2. According to IEEE Std. 802.1Q-2018 section 8.6.9.2,
6070 		 * once the end of the list is reached, it will switch
6071 		 * to the END_OF_CYCLE state and leave the gates in the
6072 		 * same state until the next cycle is started.
6073 		 */
6074 		if (end_time > adapter->cycle_time ||
6075 		    n + 1 == qopt->num_entries)
6076 			end_time = adapter->cycle_time;
6077 
6078 		for (i = 0; i < adapter->num_tx_queues; i++) {
6079 			struct igc_ring *ring = adapter->tx_ring[i];
6080 
6081 			if (!(e->gate_mask & BIT(i)))
6082 				continue;
6083 
6084 			/* Check whether a queue stays open for more than one
6085 			 * entry. If so, keep the start and advance the end
6086 			 * time.
6087 			 */
6088 			if (!queue_configured[i])
6089 				ring->start_time = start_time;
6090 			ring->end_time = end_time;
6091 
6092 			queue_configured[i] = true;
6093 		}
6094 
6095 		start_time += e->interval;
6096 	}
6097 
6098 	/* Check whether a queue gets configured.
6099 	 * If not, set the start and end time to be end time.
6100 	 */
6101 	for (i = 0; i < adapter->num_tx_queues; i++) {
6102 		if (!queue_configured[i]) {
6103 			struct igc_ring *ring = adapter->tx_ring[i];
6104 
6105 			ring->start_time = end_time;
6106 			ring->end_time = end_time;
6107 		}
6108 	}
6109 
6110 	return 0;
6111 }
6112 
igc_tsn_enable_qbv_scheduling(struct igc_adapter * adapter,struct tc_taprio_qopt_offload * qopt)6113 static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter,
6114 					 struct tc_taprio_qopt_offload *qopt)
6115 {
6116 	struct igc_hw *hw = &adapter->hw;
6117 	int err;
6118 
6119 	if (hw->mac.type != igc_i225)
6120 		return -EOPNOTSUPP;
6121 
6122 	err = igc_save_qbv_schedule(adapter, qopt);
6123 	if (err)
6124 		return err;
6125 
6126 	return igc_tsn_offload_apply(adapter);
6127 }
6128 
igc_save_cbs_params(struct igc_adapter * adapter,int queue,bool enable,int idleslope,int sendslope,int hicredit,int locredit)6129 static int igc_save_cbs_params(struct igc_adapter *adapter, int queue,
6130 			       bool enable, int idleslope, int sendslope,
6131 			       int hicredit, int locredit)
6132 {
6133 	bool cbs_status[IGC_MAX_SR_QUEUES] = { false };
6134 	struct net_device *netdev = adapter->netdev;
6135 	struct igc_ring *ring;
6136 	int i;
6137 
6138 	/* i225 has two sets of credit-based shaper logic.
6139 	 * Supporting it only on the top two priority queues
6140 	 */
6141 	if (queue < 0 || queue > 1)
6142 		return -EINVAL;
6143 
6144 	ring = adapter->tx_ring[queue];
6145 
6146 	for (i = 0; i < IGC_MAX_SR_QUEUES; i++)
6147 		if (adapter->tx_ring[i])
6148 			cbs_status[i] = adapter->tx_ring[i]->cbs_enable;
6149 
6150 	/* CBS should be enabled on the highest priority queue first in order
6151 	 * for the CBS algorithm to operate as intended.
6152 	 */
6153 	if (enable) {
6154 		if (queue == 1 && !cbs_status[0]) {
6155 			netdev_err(netdev,
6156 				   "Enabling CBS on queue1 before queue0\n");
6157 			return -EINVAL;
6158 		}
6159 	} else {
6160 		if (queue == 0 && cbs_status[1]) {
6161 			netdev_err(netdev,
6162 				   "Disabling CBS on queue0 before queue1\n");
6163 			return -EINVAL;
6164 		}
6165 	}
6166 
6167 	ring->cbs_enable = enable;
6168 	ring->idleslope = idleslope;
6169 	ring->sendslope = sendslope;
6170 	ring->hicredit = hicredit;
6171 	ring->locredit = locredit;
6172 
6173 	return 0;
6174 }
6175 
igc_tsn_enable_cbs(struct igc_adapter * adapter,struct tc_cbs_qopt_offload * qopt)6176 static int igc_tsn_enable_cbs(struct igc_adapter *adapter,
6177 			      struct tc_cbs_qopt_offload *qopt)
6178 {
6179 	struct igc_hw *hw = &adapter->hw;
6180 	int err;
6181 
6182 	if (hw->mac.type != igc_i225)
6183 		return -EOPNOTSUPP;
6184 
6185 	if (qopt->queue < 0 || qopt->queue > 1)
6186 		return -EINVAL;
6187 
6188 	err = igc_save_cbs_params(adapter, qopt->queue, qopt->enable,
6189 				  qopt->idleslope, qopt->sendslope,
6190 				  qopt->hicredit, qopt->locredit);
6191 	if (err)
6192 		return err;
6193 
6194 	return igc_tsn_offload_apply(adapter);
6195 }
6196 
igc_setup_tc(struct net_device * dev,enum tc_setup_type type,void * type_data)6197 static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type,
6198 			void *type_data)
6199 {
6200 	struct igc_adapter *adapter = netdev_priv(dev);
6201 
6202 	switch (type) {
6203 	case TC_SETUP_QDISC_TAPRIO:
6204 		return igc_tsn_enable_qbv_scheduling(adapter, type_data);
6205 
6206 	case TC_SETUP_QDISC_ETF:
6207 		return igc_tsn_enable_launchtime(adapter, type_data);
6208 
6209 	case TC_SETUP_QDISC_CBS:
6210 		return igc_tsn_enable_cbs(adapter, type_data);
6211 
6212 	default:
6213 		return -EOPNOTSUPP;
6214 	}
6215 }
6216 
igc_bpf(struct net_device * dev,struct netdev_bpf * bpf)6217 static int igc_bpf(struct net_device *dev, struct netdev_bpf *bpf)
6218 {
6219 	struct igc_adapter *adapter = netdev_priv(dev);
6220 
6221 	switch (bpf->command) {
6222 	case XDP_SETUP_PROG:
6223 		return igc_xdp_set_prog(adapter, bpf->prog, bpf->extack);
6224 	case XDP_SETUP_XSK_POOL:
6225 		return igc_xdp_setup_pool(adapter, bpf->xsk.pool,
6226 					  bpf->xsk.queue_id);
6227 	default:
6228 		return -EOPNOTSUPP;
6229 	}
6230 }
6231 
igc_xdp_xmit(struct net_device * dev,int num_frames,struct xdp_frame ** frames,u32 flags)6232 static int igc_xdp_xmit(struct net_device *dev, int num_frames,
6233 			struct xdp_frame **frames, u32 flags)
6234 {
6235 	struct igc_adapter *adapter = netdev_priv(dev);
6236 	int cpu = smp_processor_id();
6237 	struct netdev_queue *nq;
6238 	struct igc_ring *ring;
6239 	int i, drops;
6240 
6241 	if (unlikely(test_bit(__IGC_DOWN, &adapter->state)))
6242 		return -ENETDOWN;
6243 
6244 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
6245 		return -EINVAL;
6246 
6247 	ring = igc_xdp_get_tx_ring(adapter, cpu);
6248 	nq = txring_txq(ring);
6249 
6250 	__netif_tx_lock(nq, cpu);
6251 
6252 	drops = 0;
6253 	for (i = 0; i < num_frames; i++) {
6254 		int err;
6255 		struct xdp_frame *xdpf = frames[i];
6256 
6257 		err = igc_xdp_init_tx_descriptor(ring, xdpf);
6258 		if (err) {
6259 			xdp_return_frame_rx_napi(xdpf);
6260 			drops++;
6261 		}
6262 	}
6263 
6264 	if (flags & XDP_XMIT_FLUSH)
6265 		igc_flush_tx_descriptors(ring);
6266 
6267 	__netif_tx_unlock(nq);
6268 
6269 	return num_frames - drops;
6270 }
6271 
igc_trigger_rxtxq_interrupt(struct igc_adapter * adapter,struct igc_q_vector * q_vector)6272 static void igc_trigger_rxtxq_interrupt(struct igc_adapter *adapter,
6273 					struct igc_q_vector *q_vector)
6274 {
6275 	struct igc_hw *hw = &adapter->hw;
6276 	u32 eics = 0;
6277 
6278 	eics |= q_vector->eims_value;
6279 	wr32(IGC_EICS, eics);
6280 }
6281 
igc_xsk_wakeup(struct net_device * dev,u32 queue_id,u32 flags)6282 int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
6283 {
6284 	struct igc_adapter *adapter = netdev_priv(dev);
6285 	struct igc_q_vector *q_vector;
6286 	struct igc_ring *ring;
6287 
6288 	if (test_bit(__IGC_DOWN, &adapter->state))
6289 		return -ENETDOWN;
6290 
6291 	if (!igc_xdp_is_enabled(adapter))
6292 		return -ENXIO;
6293 
6294 	if (queue_id >= adapter->num_rx_queues)
6295 		return -EINVAL;
6296 
6297 	ring = adapter->rx_ring[queue_id];
6298 
6299 	if (!ring->xsk_pool)
6300 		return -ENXIO;
6301 
6302 	q_vector = adapter->q_vector[queue_id];
6303 	if (!napi_if_scheduled_mark_missed(&q_vector->napi))
6304 		igc_trigger_rxtxq_interrupt(adapter, q_vector);
6305 
6306 	return 0;
6307 }
6308 
6309 static const struct net_device_ops igc_netdev_ops = {
6310 	.ndo_open		= igc_open,
6311 	.ndo_stop		= igc_close,
6312 	.ndo_start_xmit		= igc_xmit_frame,
6313 	.ndo_set_rx_mode	= igc_set_rx_mode,
6314 	.ndo_set_mac_address	= igc_set_mac,
6315 	.ndo_change_mtu		= igc_change_mtu,
6316 	.ndo_get_stats64	= igc_get_stats64,
6317 	.ndo_fix_features	= igc_fix_features,
6318 	.ndo_set_features	= igc_set_features,
6319 	.ndo_features_check	= igc_features_check,
6320 	.ndo_eth_ioctl		= igc_ioctl,
6321 	.ndo_setup_tc		= igc_setup_tc,
6322 	.ndo_bpf		= igc_bpf,
6323 	.ndo_xdp_xmit		= igc_xdp_xmit,
6324 	.ndo_xsk_wakeup		= igc_xsk_wakeup,
6325 };
6326 
6327 /* PCIe configuration access */
igc_read_pci_cfg(struct igc_hw * hw,u32 reg,u16 * value)6328 void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
6329 {
6330 	struct igc_adapter *adapter = hw->back;
6331 
6332 	pci_read_config_word(adapter->pdev, reg, value);
6333 }
6334 
igc_write_pci_cfg(struct igc_hw * hw,u32 reg,u16 * value)6335 void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
6336 {
6337 	struct igc_adapter *adapter = hw->back;
6338 
6339 	pci_write_config_word(adapter->pdev, reg, *value);
6340 }
6341 
igc_read_pcie_cap_reg(struct igc_hw * hw,u32 reg,u16 * value)6342 s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
6343 {
6344 	struct igc_adapter *adapter = hw->back;
6345 
6346 	if (!pci_is_pcie(adapter->pdev))
6347 		return -IGC_ERR_CONFIG;
6348 
6349 	pcie_capability_read_word(adapter->pdev, reg, value);
6350 
6351 	return IGC_SUCCESS;
6352 }
6353 
igc_write_pcie_cap_reg(struct igc_hw * hw,u32 reg,u16 * value)6354 s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
6355 {
6356 	struct igc_adapter *adapter = hw->back;
6357 
6358 	if (!pci_is_pcie(adapter->pdev))
6359 		return -IGC_ERR_CONFIG;
6360 
6361 	pcie_capability_write_word(adapter->pdev, reg, *value);
6362 
6363 	return IGC_SUCCESS;
6364 }
6365 
igc_rd32(struct igc_hw * hw,u32 reg)6366 u32 igc_rd32(struct igc_hw *hw, u32 reg)
6367 {
6368 	struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw);
6369 	u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
6370 	u32 value = 0;
6371 
6372 	if (IGC_REMOVED(hw_addr))
6373 		return ~value;
6374 
6375 	value = readl(&hw_addr[reg]);
6376 
6377 	/* reads should not return all F's */
6378 	if (!(~value) && (!reg || !(~readl(hw_addr)))) {
6379 		struct net_device *netdev = igc->netdev;
6380 
6381 		hw->hw_addr = NULL;
6382 		netif_device_detach(netdev);
6383 		netdev_err(netdev, "PCIe link lost, device now detached\n");
6384 		WARN(pci_device_is_present(igc->pdev),
6385 		     "igc: Failed to read reg 0x%x!\n", reg);
6386 	}
6387 
6388 	return value;
6389 }
6390 
6391 /**
6392  * igc_probe - Device Initialization Routine
6393  * @pdev: PCI device information struct
6394  * @ent: entry in igc_pci_tbl
6395  *
6396  * Returns 0 on success, negative on failure
6397  *
6398  * igc_probe initializes an adapter identified by a pci_dev structure.
6399  * The OS initialization, configuring the adapter private structure,
6400  * and a hardware reset occur.
6401  */
igc_probe(struct pci_dev * pdev,const struct pci_device_id * ent)6402 static int igc_probe(struct pci_dev *pdev,
6403 		     const struct pci_device_id *ent)
6404 {
6405 	struct igc_adapter *adapter;
6406 	struct net_device *netdev;
6407 	struct igc_hw *hw;
6408 	const struct igc_info *ei = igc_info_tbl[ent->driver_data];
6409 	int err;
6410 
6411 	err = pci_enable_device_mem(pdev);
6412 	if (err)
6413 		return err;
6414 
6415 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
6416 	if (err) {
6417 		dev_err(&pdev->dev,
6418 			"No usable DMA configuration, aborting\n");
6419 		goto err_dma;
6420 	}
6421 
6422 	err = pci_request_mem_regions(pdev, igc_driver_name);
6423 	if (err)
6424 		goto err_pci_reg;
6425 
6426 	pci_enable_pcie_error_reporting(pdev);
6427 
6428 	err = pci_enable_ptm(pdev, NULL);
6429 	if (err < 0)
6430 		dev_info(&pdev->dev, "PCIe PTM not supported by PCIe bus/controller\n");
6431 
6432 	pci_set_master(pdev);
6433 
6434 	err = -ENOMEM;
6435 	netdev = alloc_etherdev_mq(sizeof(struct igc_adapter),
6436 				   IGC_MAX_TX_QUEUES);
6437 
6438 	if (!netdev)
6439 		goto err_alloc_etherdev;
6440 
6441 	SET_NETDEV_DEV(netdev, &pdev->dev);
6442 
6443 	pci_set_drvdata(pdev, netdev);
6444 	adapter = netdev_priv(netdev);
6445 	adapter->netdev = netdev;
6446 	adapter->pdev = pdev;
6447 	hw = &adapter->hw;
6448 	hw->back = adapter;
6449 	adapter->port_num = hw->bus.func;
6450 	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
6451 
6452 	err = pci_save_state(pdev);
6453 	if (err)
6454 		goto err_ioremap;
6455 
6456 	err = -EIO;
6457 	adapter->io_addr = ioremap(pci_resource_start(pdev, 0),
6458 				   pci_resource_len(pdev, 0));
6459 	if (!adapter->io_addr)
6460 		goto err_ioremap;
6461 
6462 	/* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */
6463 	hw->hw_addr = adapter->io_addr;
6464 
6465 	netdev->netdev_ops = &igc_netdev_ops;
6466 	igc_ethtool_set_ops(netdev);
6467 	netdev->watchdog_timeo = 5 * HZ;
6468 
6469 	netdev->mem_start = pci_resource_start(pdev, 0);
6470 	netdev->mem_end = pci_resource_end(pdev, 0);
6471 
6472 	/* PCI config space info */
6473 	hw->vendor_id = pdev->vendor;
6474 	hw->device_id = pdev->device;
6475 	hw->revision_id = pdev->revision;
6476 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
6477 	hw->subsystem_device_id = pdev->subsystem_device;
6478 
6479 	/* Copy the default MAC and PHY function pointers */
6480 	memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
6481 	memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
6482 
6483 	/* Initialize skew-specific constants */
6484 	err = ei->get_invariants(hw);
6485 	if (err)
6486 		goto err_sw_init;
6487 
6488 	/* Add supported features to the features list*/
6489 	netdev->features |= NETIF_F_SG;
6490 	netdev->features |= NETIF_F_TSO;
6491 	netdev->features |= NETIF_F_TSO6;
6492 	netdev->features |= NETIF_F_TSO_ECN;
6493 	netdev->features |= NETIF_F_RXCSUM;
6494 	netdev->features |= NETIF_F_HW_CSUM;
6495 	netdev->features |= NETIF_F_SCTP_CRC;
6496 	netdev->features |= NETIF_F_HW_TC;
6497 
6498 #define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
6499 				  NETIF_F_GSO_GRE_CSUM | \
6500 				  NETIF_F_GSO_IPXIP4 | \
6501 				  NETIF_F_GSO_IPXIP6 | \
6502 				  NETIF_F_GSO_UDP_TUNNEL | \
6503 				  NETIF_F_GSO_UDP_TUNNEL_CSUM)
6504 
6505 	netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES;
6506 	netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES;
6507 
6508 	/* setup the private structure */
6509 	err = igc_sw_init(adapter);
6510 	if (err)
6511 		goto err_sw_init;
6512 
6513 	/* copy netdev features into list of user selectable features */
6514 	netdev->hw_features |= NETIF_F_NTUPLE;
6515 	netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
6516 	netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
6517 	netdev->hw_features |= netdev->features;
6518 
6519 	netdev->features |= NETIF_F_HIGHDMA;
6520 
6521 	netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
6522 	netdev->mpls_features |= NETIF_F_HW_CSUM;
6523 	netdev->hw_enc_features |= netdev->vlan_features;
6524 
6525 	/* MTU range: 68 - 9216 */
6526 	netdev->min_mtu = ETH_MIN_MTU;
6527 	netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
6528 
6529 	/* before reading the NVM, reset the controller to put the device in a
6530 	 * known good starting state
6531 	 */
6532 	hw->mac.ops.reset_hw(hw);
6533 
6534 	if (igc_get_flash_presence_i225(hw)) {
6535 		if (hw->nvm.ops.validate(hw) < 0) {
6536 			dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
6537 			err = -EIO;
6538 			goto err_eeprom;
6539 		}
6540 	}
6541 
6542 	if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
6543 		/* copy the MAC address out of the NVM */
6544 		if (hw->mac.ops.read_mac_addr(hw))
6545 			dev_err(&pdev->dev, "NVM Read Error\n");
6546 	}
6547 
6548 	eth_hw_addr_set(netdev, hw->mac.addr);
6549 
6550 	if (!is_valid_ether_addr(netdev->dev_addr)) {
6551 		dev_err(&pdev->dev, "Invalid MAC Address\n");
6552 		err = -EIO;
6553 		goto err_eeprom;
6554 	}
6555 
6556 	/* configure RXPBSIZE and TXPBSIZE */
6557 	wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT);
6558 	wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT);
6559 
6560 	timer_setup(&adapter->watchdog_timer, igc_watchdog, 0);
6561 	timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0);
6562 
6563 	INIT_WORK(&adapter->reset_task, igc_reset_task);
6564 	INIT_WORK(&adapter->watchdog_task, igc_watchdog_task);
6565 
6566 	/* Initialize link properties that are user-changeable */
6567 	adapter->fc_autoneg = true;
6568 	hw->mac.autoneg = true;
6569 	hw->phy.autoneg_advertised = 0xaf;
6570 
6571 	hw->fc.requested_mode = igc_fc_default;
6572 	hw->fc.current_mode = igc_fc_default;
6573 
6574 	/* By default, support wake on port A */
6575 	adapter->flags |= IGC_FLAG_WOL_SUPPORTED;
6576 
6577 	/* initialize the wol settings based on the eeprom settings */
6578 	if (adapter->flags & IGC_FLAG_WOL_SUPPORTED)
6579 		adapter->wol |= IGC_WUFC_MAG;
6580 
6581 	device_set_wakeup_enable(&adapter->pdev->dev,
6582 				 adapter->flags & IGC_FLAG_WOL_SUPPORTED);
6583 
6584 	igc_ptp_init(adapter);
6585 
6586 	igc_tsn_clear_schedule(adapter);
6587 
6588 	/* reset the hardware with the new settings */
6589 	igc_reset(adapter);
6590 
6591 	/* let the f/w know that the h/w is now under the control of the
6592 	 * driver.
6593 	 */
6594 	igc_get_hw_control(adapter);
6595 
6596 	strncpy(netdev->name, "eth%d", IFNAMSIZ);
6597 	err = register_netdev(netdev);
6598 	if (err)
6599 		goto err_register;
6600 
6601 	 /* carrier off reporting is important to ethtool even BEFORE open */
6602 	netif_carrier_off(netdev);
6603 
6604 	/* Check if Media Autosense is enabled */
6605 	adapter->ei = *ei;
6606 
6607 	/* print pcie link status and MAC address */
6608 	pcie_print_link_status(pdev);
6609 	netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr);
6610 
6611 	dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE);
6612 	/* Disable EEE for internal PHY devices */
6613 	hw->dev_spec._base.eee_enable = false;
6614 	adapter->flags &= ~IGC_FLAG_EEE;
6615 	igc_set_eee_i225(hw, false, false, false);
6616 
6617 	pm_runtime_put_noidle(&pdev->dev);
6618 
6619 	return 0;
6620 
6621 err_register:
6622 	igc_release_hw_control(adapter);
6623 err_eeprom:
6624 	if (!igc_check_reset_block(hw))
6625 		igc_reset_phy(hw);
6626 err_sw_init:
6627 	igc_clear_interrupt_scheme(adapter);
6628 	iounmap(adapter->io_addr);
6629 err_ioremap:
6630 	free_netdev(netdev);
6631 err_alloc_etherdev:
6632 	pci_disable_pcie_error_reporting(pdev);
6633 	pci_release_mem_regions(pdev);
6634 err_pci_reg:
6635 err_dma:
6636 	pci_disable_device(pdev);
6637 	return err;
6638 }
6639 
6640 /**
6641  * igc_remove - Device Removal Routine
6642  * @pdev: PCI device information struct
6643  *
6644  * igc_remove is called by the PCI subsystem to alert the driver
6645  * that it should release a PCI device.  This could be caused by a
6646  * Hot-Plug event, or because the driver is going to be removed from
6647  * memory.
6648  */
igc_remove(struct pci_dev * pdev)6649 static void igc_remove(struct pci_dev *pdev)
6650 {
6651 	struct net_device *netdev = pci_get_drvdata(pdev);
6652 	struct igc_adapter *adapter = netdev_priv(netdev);
6653 
6654 	pm_runtime_get_noresume(&pdev->dev);
6655 
6656 	igc_flush_nfc_rules(adapter);
6657 
6658 	igc_ptp_stop(adapter);
6659 
6660 	set_bit(__IGC_DOWN, &adapter->state);
6661 
6662 	del_timer_sync(&adapter->watchdog_timer);
6663 	del_timer_sync(&adapter->phy_info_timer);
6664 
6665 	cancel_work_sync(&adapter->reset_task);
6666 	cancel_work_sync(&adapter->watchdog_task);
6667 
6668 	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
6669 	 * would have already happened in close and is redundant.
6670 	 */
6671 	igc_release_hw_control(adapter);
6672 	unregister_netdev(netdev);
6673 
6674 	igc_clear_interrupt_scheme(adapter);
6675 	pci_iounmap(pdev, adapter->io_addr);
6676 	pci_release_mem_regions(pdev);
6677 
6678 	free_netdev(netdev);
6679 
6680 	pci_disable_pcie_error_reporting(pdev);
6681 
6682 	pci_disable_device(pdev);
6683 }
6684 
__igc_shutdown(struct pci_dev * pdev,bool * enable_wake,bool runtime)6685 static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake,
6686 			  bool runtime)
6687 {
6688 	struct net_device *netdev = pci_get_drvdata(pdev);
6689 	struct igc_adapter *adapter = netdev_priv(netdev);
6690 	u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol;
6691 	struct igc_hw *hw = &adapter->hw;
6692 	u32 ctrl, rctl, status;
6693 	bool wake;
6694 
6695 	rtnl_lock();
6696 	netif_device_detach(netdev);
6697 
6698 	if (netif_running(netdev))
6699 		__igc_close(netdev, true);
6700 
6701 	igc_ptp_suspend(adapter);
6702 
6703 	igc_clear_interrupt_scheme(adapter);
6704 	rtnl_unlock();
6705 
6706 	status = rd32(IGC_STATUS);
6707 	if (status & IGC_STATUS_LU)
6708 		wufc &= ~IGC_WUFC_LNKC;
6709 
6710 	if (wufc) {
6711 		igc_setup_rctl(adapter);
6712 		igc_set_rx_mode(netdev);
6713 
6714 		/* turn on all-multi mode if wake on multicast is enabled */
6715 		if (wufc & IGC_WUFC_MC) {
6716 			rctl = rd32(IGC_RCTL);
6717 			rctl |= IGC_RCTL_MPE;
6718 			wr32(IGC_RCTL, rctl);
6719 		}
6720 
6721 		ctrl = rd32(IGC_CTRL);
6722 		ctrl |= IGC_CTRL_ADVD3WUC;
6723 		wr32(IGC_CTRL, ctrl);
6724 
6725 		/* Allow time for pending master requests to run */
6726 		igc_disable_pcie_master(hw);
6727 
6728 		wr32(IGC_WUC, IGC_WUC_PME_EN);
6729 		wr32(IGC_WUFC, wufc);
6730 	} else {
6731 		wr32(IGC_WUC, 0);
6732 		wr32(IGC_WUFC, 0);
6733 	}
6734 
6735 	wake = wufc || adapter->en_mng_pt;
6736 	if (!wake)
6737 		igc_power_down_phy_copper_base(&adapter->hw);
6738 	else
6739 		igc_power_up_link(adapter);
6740 
6741 	if (enable_wake)
6742 		*enable_wake = wake;
6743 
6744 	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
6745 	 * would have already happened in close and is redundant.
6746 	 */
6747 	igc_release_hw_control(adapter);
6748 
6749 	pci_disable_device(pdev);
6750 
6751 	return 0;
6752 }
6753 
6754 #ifdef CONFIG_PM
igc_runtime_suspend(struct device * dev)6755 static int __maybe_unused igc_runtime_suspend(struct device *dev)
6756 {
6757 	return __igc_shutdown(to_pci_dev(dev), NULL, 1);
6758 }
6759 
igc_deliver_wake_packet(struct net_device * netdev)6760 static void igc_deliver_wake_packet(struct net_device *netdev)
6761 {
6762 	struct igc_adapter *adapter = netdev_priv(netdev);
6763 	struct igc_hw *hw = &adapter->hw;
6764 	struct sk_buff *skb;
6765 	u32 wupl;
6766 
6767 	wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK;
6768 
6769 	/* WUPM stores only the first 128 bytes of the wake packet.
6770 	 * Read the packet only if we have the whole thing.
6771 	 */
6772 	if (wupl == 0 || wupl > IGC_WUPM_BYTES)
6773 		return;
6774 
6775 	skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES);
6776 	if (!skb)
6777 		return;
6778 
6779 	skb_put(skb, wupl);
6780 
6781 	/* Ensure reads are 32-bit aligned */
6782 	wupl = roundup(wupl, 4);
6783 
6784 	memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl);
6785 
6786 	skb->protocol = eth_type_trans(skb, netdev);
6787 	netif_rx(skb);
6788 }
6789 
igc_resume(struct device * dev)6790 static int __maybe_unused igc_resume(struct device *dev)
6791 {
6792 	struct pci_dev *pdev = to_pci_dev(dev);
6793 	struct net_device *netdev = pci_get_drvdata(pdev);
6794 	struct igc_adapter *adapter = netdev_priv(netdev);
6795 	struct igc_hw *hw = &adapter->hw;
6796 	u32 err, val;
6797 
6798 	pci_set_power_state(pdev, PCI_D0);
6799 	pci_restore_state(pdev);
6800 	pci_save_state(pdev);
6801 
6802 	if (!pci_device_is_present(pdev))
6803 		return -ENODEV;
6804 	err = pci_enable_device_mem(pdev);
6805 	if (err) {
6806 		netdev_err(netdev, "Cannot enable PCI device from suspend\n");
6807 		return err;
6808 	}
6809 	pci_set_master(pdev);
6810 
6811 	pci_enable_wake(pdev, PCI_D3hot, 0);
6812 	pci_enable_wake(pdev, PCI_D3cold, 0);
6813 
6814 	if (igc_init_interrupt_scheme(adapter, true)) {
6815 		netdev_err(netdev, "Unable to allocate memory for queues\n");
6816 		return -ENOMEM;
6817 	}
6818 
6819 	igc_reset(adapter);
6820 
6821 	/* let the f/w know that the h/w is now under the control of the
6822 	 * driver.
6823 	 */
6824 	igc_get_hw_control(adapter);
6825 
6826 	val = rd32(IGC_WUS);
6827 	if (val & WAKE_PKT_WUS)
6828 		igc_deliver_wake_packet(netdev);
6829 
6830 	wr32(IGC_WUS, ~0);
6831 
6832 	rtnl_lock();
6833 	if (!err && netif_running(netdev))
6834 		err = __igc_open(netdev, true);
6835 
6836 	if (!err)
6837 		netif_device_attach(netdev);
6838 	rtnl_unlock();
6839 
6840 	return err;
6841 }
6842 
igc_runtime_resume(struct device * dev)6843 static int __maybe_unused igc_runtime_resume(struct device *dev)
6844 {
6845 	return igc_resume(dev);
6846 }
6847 
igc_suspend(struct device * dev)6848 static int __maybe_unused igc_suspend(struct device *dev)
6849 {
6850 	return __igc_shutdown(to_pci_dev(dev), NULL, 0);
6851 }
6852 
igc_runtime_idle(struct device * dev)6853 static int __maybe_unused igc_runtime_idle(struct device *dev)
6854 {
6855 	struct net_device *netdev = dev_get_drvdata(dev);
6856 	struct igc_adapter *adapter = netdev_priv(netdev);
6857 
6858 	if (!igc_has_link(adapter))
6859 		pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
6860 
6861 	return -EBUSY;
6862 }
6863 #endif /* CONFIG_PM */
6864 
igc_shutdown(struct pci_dev * pdev)6865 static void igc_shutdown(struct pci_dev *pdev)
6866 {
6867 	bool wake;
6868 
6869 	__igc_shutdown(pdev, &wake, 0);
6870 
6871 	if (system_state == SYSTEM_POWER_OFF) {
6872 		pci_wake_from_d3(pdev, wake);
6873 		pci_set_power_state(pdev, PCI_D3hot);
6874 	}
6875 }
6876 
6877 /**
6878  *  igc_io_error_detected - called when PCI error is detected
6879  *  @pdev: Pointer to PCI device
6880  *  @state: The current PCI connection state
6881  *
6882  *  This function is called after a PCI bus error affecting
6883  *  this device has been detected.
6884  **/
igc_io_error_detected(struct pci_dev * pdev,pci_channel_state_t state)6885 static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev,
6886 					      pci_channel_state_t state)
6887 {
6888 	struct net_device *netdev = pci_get_drvdata(pdev);
6889 	struct igc_adapter *adapter = netdev_priv(netdev);
6890 
6891 	netif_device_detach(netdev);
6892 
6893 	if (state == pci_channel_io_perm_failure)
6894 		return PCI_ERS_RESULT_DISCONNECT;
6895 
6896 	if (netif_running(netdev))
6897 		igc_down(adapter);
6898 	pci_disable_device(pdev);
6899 
6900 	/* Request a slot reset. */
6901 	return PCI_ERS_RESULT_NEED_RESET;
6902 }
6903 
6904 /**
6905  *  igc_io_slot_reset - called after the PCI bus has been reset.
6906  *  @pdev: Pointer to PCI device
6907  *
6908  *  Restart the card from scratch, as if from a cold-boot. Implementation
6909  *  resembles the first-half of the igc_resume routine.
6910  **/
igc_io_slot_reset(struct pci_dev * pdev)6911 static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev)
6912 {
6913 	struct net_device *netdev = pci_get_drvdata(pdev);
6914 	struct igc_adapter *adapter = netdev_priv(netdev);
6915 	struct igc_hw *hw = &adapter->hw;
6916 	pci_ers_result_t result;
6917 
6918 	if (pci_enable_device_mem(pdev)) {
6919 		netdev_err(netdev, "Could not re-enable PCI device after reset\n");
6920 		result = PCI_ERS_RESULT_DISCONNECT;
6921 	} else {
6922 		pci_set_master(pdev);
6923 		pci_restore_state(pdev);
6924 		pci_save_state(pdev);
6925 
6926 		pci_enable_wake(pdev, PCI_D3hot, 0);
6927 		pci_enable_wake(pdev, PCI_D3cold, 0);
6928 
6929 		/* In case of PCI error, adapter loses its HW address
6930 		 * so we should re-assign it here.
6931 		 */
6932 		hw->hw_addr = adapter->io_addr;
6933 
6934 		igc_reset(adapter);
6935 		wr32(IGC_WUS, ~0);
6936 		result = PCI_ERS_RESULT_RECOVERED;
6937 	}
6938 
6939 	return result;
6940 }
6941 
6942 /**
6943  *  igc_io_resume - called when traffic can start to flow again.
6944  *  @pdev: Pointer to PCI device
6945  *
6946  *  This callback is called when the error recovery driver tells us that
6947  *  its OK to resume normal operation. Implementation resembles the
6948  *  second-half of the igc_resume routine.
6949  */
igc_io_resume(struct pci_dev * pdev)6950 static void igc_io_resume(struct pci_dev *pdev)
6951 {
6952 	struct net_device *netdev = pci_get_drvdata(pdev);
6953 	struct igc_adapter *adapter = netdev_priv(netdev);
6954 
6955 	rtnl_lock();
6956 	if (netif_running(netdev)) {
6957 		if (igc_open(netdev)) {
6958 			netdev_err(netdev, "igc_open failed after reset\n");
6959 			return;
6960 		}
6961 	}
6962 
6963 	netif_device_attach(netdev);
6964 
6965 	/* let the f/w know that the h/w is now under the control of the
6966 	 * driver.
6967 	 */
6968 	igc_get_hw_control(adapter);
6969 	rtnl_unlock();
6970 }
6971 
6972 static const struct pci_error_handlers igc_err_handler = {
6973 	.error_detected = igc_io_error_detected,
6974 	.slot_reset = igc_io_slot_reset,
6975 	.resume = igc_io_resume,
6976 };
6977 
6978 #ifdef CONFIG_PM
6979 static const struct dev_pm_ops igc_pm_ops = {
6980 	SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume)
6981 	SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume,
6982 			   igc_runtime_idle)
6983 };
6984 #endif
6985 
6986 static struct pci_driver igc_driver = {
6987 	.name     = igc_driver_name,
6988 	.id_table = igc_pci_tbl,
6989 	.probe    = igc_probe,
6990 	.remove   = igc_remove,
6991 #ifdef CONFIG_PM
6992 	.driver.pm = &igc_pm_ops,
6993 #endif
6994 	.shutdown = igc_shutdown,
6995 	.err_handler = &igc_err_handler,
6996 };
6997 
6998 /**
6999  * igc_reinit_queues - return error
7000  * @adapter: pointer to adapter structure
7001  */
igc_reinit_queues(struct igc_adapter * adapter)7002 int igc_reinit_queues(struct igc_adapter *adapter)
7003 {
7004 	struct net_device *netdev = adapter->netdev;
7005 	int err = 0;
7006 
7007 	if (netif_running(netdev))
7008 		igc_close(netdev);
7009 
7010 	igc_reset_interrupt_capability(adapter);
7011 
7012 	if (igc_init_interrupt_scheme(adapter, true)) {
7013 		netdev_err(netdev, "Unable to allocate memory for queues\n");
7014 		return -ENOMEM;
7015 	}
7016 
7017 	if (netif_running(netdev))
7018 		err = igc_open(netdev);
7019 
7020 	return err;
7021 }
7022 
7023 /**
7024  * igc_get_hw_dev - return device
7025  * @hw: pointer to hardware structure
7026  *
7027  * used by hardware layer to print debugging information
7028  */
igc_get_hw_dev(struct igc_hw * hw)7029 struct net_device *igc_get_hw_dev(struct igc_hw *hw)
7030 {
7031 	struct igc_adapter *adapter = hw->back;
7032 
7033 	return adapter->netdev;
7034 }
7035 
igc_disable_rx_ring_hw(struct igc_ring * ring)7036 static void igc_disable_rx_ring_hw(struct igc_ring *ring)
7037 {
7038 	struct igc_hw *hw = &ring->q_vector->adapter->hw;
7039 	u8 idx = ring->reg_idx;
7040 	u32 rxdctl;
7041 
7042 	rxdctl = rd32(IGC_RXDCTL(idx));
7043 	rxdctl &= ~IGC_RXDCTL_QUEUE_ENABLE;
7044 	rxdctl |= IGC_RXDCTL_SWFLUSH;
7045 	wr32(IGC_RXDCTL(idx), rxdctl);
7046 }
7047 
igc_disable_rx_ring(struct igc_ring * ring)7048 void igc_disable_rx_ring(struct igc_ring *ring)
7049 {
7050 	igc_disable_rx_ring_hw(ring);
7051 	igc_clean_rx_ring(ring);
7052 }
7053 
igc_enable_rx_ring(struct igc_ring * ring)7054 void igc_enable_rx_ring(struct igc_ring *ring)
7055 {
7056 	struct igc_adapter *adapter = ring->q_vector->adapter;
7057 
7058 	igc_configure_rx_ring(adapter, ring);
7059 
7060 	if (ring->xsk_pool)
7061 		igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring));
7062 	else
7063 		igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
7064 }
7065 
igc_disable_tx_ring_hw(struct igc_ring * ring)7066 static void igc_disable_tx_ring_hw(struct igc_ring *ring)
7067 {
7068 	struct igc_hw *hw = &ring->q_vector->adapter->hw;
7069 	u8 idx = ring->reg_idx;
7070 	u32 txdctl;
7071 
7072 	txdctl = rd32(IGC_TXDCTL(idx));
7073 	txdctl &= ~IGC_TXDCTL_QUEUE_ENABLE;
7074 	txdctl |= IGC_TXDCTL_SWFLUSH;
7075 	wr32(IGC_TXDCTL(idx), txdctl);
7076 }
7077 
igc_disable_tx_ring(struct igc_ring * ring)7078 void igc_disable_tx_ring(struct igc_ring *ring)
7079 {
7080 	igc_disable_tx_ring_hw(ring);
7081 	igc_clean_tx_ring(ring);
7082 }
7083 
igc_enable_tx_ring(struct igc_ring * ring)7084 void igc_enable_tx_ring(struct igc_ring *ring)
7085 {
7086 	struct igc_adapter *adapter = ring->q_vector->adapter;
7087 
7088 	igc_configure_tx_ring(adapter, ring);
7089 }
7090 
7091 /**
7092  * igc_init_module - Driver Registration Routine
7093  *
7094  * igc_init_module is the first routine called when the driver is
7095  * loaded. All it does is register with the PCI subsystem.
7096  */
igc_init_module(void)7097 static int __init igc_init_module(void)
7098 {
7099 	int ret;
7100 
7101 	pr_info("%s\n", igc_driver_string);
7102 	pr_info("%s\n", igc_copyright);
7103 
7104 	ret = pci_register_driver(&igc_driver);
7105 	return ret;
7106 }
7107 
7108 module_init(igc_init_module);
7109 
7110 /**
7111  * igc_exit_module - Driver Exit Cleanup Routine
7112  *
7113  * igc_exit_module is called just before the driver is removed
7114  * from memory.
7115  */
igc_exit_module(void)7116 static void __exit igc_exit_module(void)
7117 {
7118 	pci_unregister_driver(&igc_driver);
7119 }
7120 
7121 module_exit(igc_exit_module);
7122 /* igc_main.c */
7123