581 		ethertype = ring->q_vector->wx->tpid[idx];  in wx_rx_vlan()
706 	unsigned int budget = q_vector->wx->tx_work_limit;  in wx_clean_tx_irq()
831 	struct wx *wx = q_vector->wx;  in wx_poll()  local
866 		if (netif_running(wx->netdev))  in wx_poll()
867 			wx_intr_enable(wx, WX_INTR_Q(q_vector->v_idx));  in wx_poll()
1508 	struct wx *wx = netdev_priv(netdev);  in wx_xmit_frame()  local
1522 	if (r_idx >= wx->num_tx_queues)  in wx_xmit_frame()
1523 		r_idx = r_idx % wx->num_tx_queues;  in wx_xmit_frame()
1524 	tx_ring = wx->tx_ring[r_idx];  in wx_xmit_frame()
1530 void wx_napi_enable_all(struct wx *wx)  in wx_napi_enable_all()  argument
1535 	for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {  in wx_napi_enable_all()
1536 		q_vector = wx->q_vector[q_idx];  in wx_napi_enable_all()
1542 void wx_napi_disable_all(struct wx *wx)  in wx_napi_disable_all()  argument
1547 	for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {  in wx_napi_disable_all()
1548 		q_vector = wx->q_vector[q_idx];  in wx_napi_disable_all()
1562 static void wx_set_rss_queues(struct wx *wx)  in wx_set_rss_queues()  argument
1564 	wx->num_rx_queues = wx->mac.max_rx_queues;  in wx_set_rss_queues()
1565 	wx->num_tx_queues = wx->mac.max_tx_queues;  in wx_set_rss_queues()
1568 static void wx_set_num_queues(struct wx *wx)  in wx_set_num_queues()  argument
1571 	wx->num_rx_queues = 1;  in wx_set_num_queues()
1572 	wx->num_tx_queues = 1;  in wx_set_num_queues()
1573 	wx->queues_per_pool = 1;  in wx_set_num_queues()
1575 	wx_set_rss_queues(wx);  in wx_set_num_queues()
1586 static int wx_acquire_msix_vectors(struct wx *wx)  in wx_acquire_msix_vectors()  argument
1591 	nvecs = min_t(int, num_online_cpus(), wx->mac.max_msix_vectors);  in wx_acquire_msix_vectors()
1593 	wx->msix_entries = kcalloc(nvecs,  in wx_acquire_msix_vectors()
1596 	if (!wx->msix_entries)  in wx_acquire_msix_vectors()
1599 	nvecs = pci_alloc_irq_vectors_affinity(wx->pdev, nvecs,  in wx_acquire_msix_vectors()
1604 		wx_err(wx, "Failed to allocate MSI-X interrupts. Err: %d\n", nvecs);  in wx_acquire_msix_vectors()
1605 		kfree(wx->msix_entries);  in wx_acquire_msix_vectors()
1606 		wx->msix_entries = NULL;  in wx_acquire_msix_vectors()
1611 		wx->msix_entries[i].entry = i;  in wx_acquire_msix_vectors()
1612 		wx->msix_entries[i].vector = pci_irq_vector(wx->pdev, i);  in wx_acquire_msix_vectors()
1617 	wx->num_q_vectors = nvecs;  in wx_acquire_msix_vectors()
1618 	wx->num_rx_queues = nvecs;  in wx_acquire_msix_vectors()
1619 	wx->num_tx_queues = nvecs;  in wx_acquire_msix_vectors()
1631 static int wx_set_interrupt_capability(struct wx *wx)  in wx_set_interrupt_capability()  argument
1633 	struct pci_dev *pdev = wx->pdev;  in wx_set_interrupt_capability()
1637 	ret = wx_acquire_msix_vectors(wx);  in wx_set_interrupt_capability()
1641 	wx->num_rx_queues = 1;  in wx_set_interrupt_capability()
1642 	wx->num_tx_queues = 1;  in wx_set_interrupt_capability()
1643 	wx->num_q_vectors = 1;  in wx_set_interrupt_capability()
1651 			wx_err(wx, "Fallback to MSI.\n");  in wx_set_interrupt_capability()
1653 			wx_err(wx, "Fallback to LEGACY.\n");  in wx_set_interrupt_capability()
1655 		wx_err(wx, "Failed to allocate MSI/LEGACY interrupts. Error: %d\n", nvecs);  in wx_set_interrupt_capability()
1671 static void wx_cache_ring_rss(struct wx *wx)  in wx_cache_ring_rss()  argument
1675 	for (i = 0; i < wx->num_rx_queues; i++)  in wx_cache_ring_rss()
1676 		wx->rx_ring[i]->reg_idx = i;  in wx_cache_ring_rss()
1678 	for (i = 0; i < wx->num_tx_queues; i++)  in wx_cache_ring_rss()
1679 		wx->tx_ring[i]->reg_idx = i;  in wx_cache_ring_rss()
1701 static int wx_alloc_q_vector(struct wx *wx,  in wx_alloc_q_vector()  argument
1719 	netif_napi_add(wx->netdev, &q_vector->napi,  in wx_alloc_q_vector()
1723 	wx->q_vector[v_idx] = q_vector;  in wx_alloc_q_vector()
1724 	q_vector->wx = wx;  in wx_alloc_q_vector()
1732 	if (wx->mac.type == wx_mac_sp)  in wx_alloc_q_vector()
1739 		q_vector->itr = wx->tx_itr_setting ?  in wx_alloc_q_vector()
1740 				default_itr : wx->tx_itr_setting;  in wx_alloc_q_vector()
1743 		q_vector->itr = wx->rx_itr_setting ?  in wx_alloc_q_vector()
1744 				default_itr : wx->rx_itr_setting;  in wx_alloc_q_vector()
1748 		ring->dev = &wx->pdev->dev;  in wx_alloc_q_vector()
1749 		ring->netdev = wx->netdev;  in wx_alloc_q_vector()
1758 		ring->count = wx->tx_ring_count;  in wx_alloc_q_vector()
1763 		wx->tx_ring[txr_idx] = ring;  in wx_alloc_q_vector()
1775 		ring->dev = &wx->pdev->dev;  in wx_alloc_q_vector()
1776 		ring->netdev = wx->netdev;  in wx_alloc_q_vector()
1785 		ring->count = wx->rx_ring_count;  in wx_alloc_q_vector()
1789 		wx->rx_ring[rxr_idx] = ring;  in wx_alloc_q_vector()
1811 static void wx_free_q_vector(struct wx *wx, int v_idx)  in wx_free_q_vector()  argument
1813 	struct wx_q_vector *q_vector = wx->q_vector[v_idx];  in wx_free_q_vector()
1817 		wx->tx_ring[ring->queue_index] = NULL;  in wx_free_q_vector()
1820 		wx->rx_ring[ring->queue_index] = NULL;  in wx_free_q_vector()
1822 	wx->q_vector[v_idx] = NULL;  in wx_free_q_vector()
1834 static int wx_alloc_q_vectors(struct wx *wx)  in wx_alloc_q_vectors()  argument
1837 	unsigned int rxr_remaining = wx->num_rx_queues;  in wx_alloc_q_vectors()
1838 	unsigned int txr_remaining = wx->num_tx_queues;  in wx_alloc_q_vectors()
1839 	unsigned int q_vectors = wx->num_q_vectors;  in wx_alloc_q_vectors()
1846 		err = wx_alloc_q_vector(wx, q_vectors, v_idx,  in wx_alloc_q_vectors()
1863 	wx->num_tx_queues = 0;  in wx_alloc_q_vectors()
1864 	wx->num_rx_queues = 0;  in wx_alloc_q_vectors()
1865 	wx->num_q_vectors = 0;  in wx_alloc_q_vectors()
1868 		wx_free_q_vector(wx, v_idx);  in wx_alloc_q_vectors()
1881 static void wx_free_q_vectors(struct wx *wx)  in wx_free_q_vectors()  argument
1883 	int v_idx = wx->num_q_vectors;  in wx_free_q_vectors()
1885 	wx->num_tx_queues = 0;  in wx_free_q_vectors()
1886 	wx->num_rx_queues = 0;  in wx_free_q_vectors()
1887 	wx->num_q_vectors = 0;  in wx_free_q_vectors()
1890 		wx_free_q_vector(wx, v_idx);  in wx_free_q_vectors()
1893 void wx_reset_interrupt_capability(struct wx *wx)  in wx_reset_interrupt_capability()  argument
1895 	struct pci_dev *pdev = wx->pdev;  in wx_reset_interrupt_capability()
1901 		kfree(wx->msix_entries);  in wx_reset_interrupt_capability()
1902 		wx->msix_entries = NULL;  in wx_reset_interrupt_capability()
1904 	pci_free_irq_vectors(wx->pdev);  in wx_reset_interrupt_capability()
1915 void wx_clear_interrupt_scheme(struct wx *wx)  in wx_clear_interrupt_scheme()  argument
1917 	wx_free_q_vectors(wx);  in wx_clear_interrupt_scheme()
1918 	wx_reset_interrupt_capability(wx);  in wx_clear_interrupt_scheme()
1922 int wx_init_interrupt_scheme(struct wx *wx)  in wx_init_interrupt_scheme()  argument
1927 	wx_set_num_queues(wx);  in wx_init_interrupt_scheme()
1930 	ret = wx_set_interrupt_capability(wx);  in wx_init_interrupt_scheme()
1932 		wx_err(wx, "Allocate irq vectors for failed.\n");  in wx_init_interrupt_scheme()
1937 	ret = wx_alloc_q_vectors(wx);  in wx_init_interrupt_scheme()
1939 		wx_err(wx, "Unable to allocate memory for queue vectors.\n");  in wx_init_interrupt_scheme()
1940 		wx_reset_interrupt_capability(wx);  in wx_init_interrupt_scheme()
1944 	wx_cache_ring_rss(wx);  in wx_init_interrupt_scheme()
1962 void wx_free_irq(struct wx *wx)  in wx_free_irq()  argument
1964 	struct pci_dev *pdev = wx->pdev;  in wx_free_irq()
1968 		free_irq(pdev->irq, wx);  in wx_free_irq()
1972 	for (vector = 0; vector < wx->num_q_vectors; vector++) {  in wx_free_irq()
1973 		struct wx_q_vector *q_vector = wx->q_vector[vector];  in wx_free_irq()
1974 		struct msix_entry *entry = &wx->msix_entries[vector];  in wx_free_irq()
1983 	if (wx->mac.type == wx_mac_em)  in wx_free_irq()
1984 		free_irq(wx->msix_entries[vector].vector, wx);  in wx_free_irq()
1994 int wx_setup_isb_resources(struct wx *wx)  in wx_setup_isb_resources()  argument
1996 	struct pci_dev *pdev = wx->pdev;  in wx_setup_isb_resources()
1998 	wx->isb_mem = dma_alloc_coherent(&pdev->dev,  in wx_setup_isb_resources()
2000 					 &wx->isb_dma,  in wx_setup_isb_resources()
2002 	if (!wx->isb_mem) {  in wx_setup_isb_resources()
2003 		wx_err(wx, "Alloc isb_mem failed\n");  in wx_setup_isb_resources()
2017 void wx_free_isb_resources(struct wx *wx)  in wx_free_isb_resources()  argument
2019 	struct pci_dev *pdev = wx->pdev;  in wx_free_isb_resources()
2022 			  wx->isb_mem, wx->isb_dma);  in wx_free_isb_resources()
2023 	wx->isb_mem = NULL;  in wx_free_isb_resources()
2027 u32 wx_misc_isb(struct wx *wx, enum wx_isb_idx idx)  in wx_misc_isb()  argument
2031 	cur_tag = wx->isb_mem[WX_ISB_HEADER];  in wx_misc_isb()
2032 	wx->isb_tag[idx] = cur_tag;  in wx_misc_isb()
2034 	return (__force u32)cpu_to_le32(wx->isb_mem[idx]);  in wx_misc_isb()
2046 static void wx_set_ivar(struct wx *wx, s8 direction,  in wx_set_ivar()  argument
2055 		ivar = rd32(wx, WX_PX_MISC_IVAR);  in wx_set_ivar()
2058 		wr32(wx, WX_PX_MISC_IVAR, ivar);  in wx_set_ivar()
2063 		ivar = rd32(wx, WX_PX_IVAR(queue >> 1));  in wx_set_ivar()
2066 		wr32(wx, WX_PX_IVAR(queue >> 1), ivar);  in wx_set_ivar()
2080 	struct wx *wx = q_vector->wx;  in wx_write_eitr()  local
2084 	if (wx->mac.type == wx_mac_sp)  in wx_write_eitr()
2091 	wr32(wx, WX_PX_ITR(v_idx), itr_reg);  in wx_write_eitr()
2101 void wx_configure_vectors(struct wx *wx)  in wx_configure_vectors()  argument
2103 	struct pci_dev *pdev = wx->pdev;  in wx_configure_vectors()
2109 		wr32(wx, WX_PX_ITRSEL, eitrsel);  in wx_configure_vectors()
2113 		wr32(wx, WX_PX_GPIE, WX_PX_GPIE_MODEL);  in wx_configure_vectors()
2118 		wr32(wx, WX_PX_GPIE, 0);  in wx_configure_vectors()
2124 	for (v_idx = 0; v_idx < wx->num_q_vectors; v_idx++) {  in wx_configure_vectors()
2125 		struct wx_q_vector *q_vector = wx->q_vector[v_idx];  in wx_configure_vectors()
2129 			wx_set_ivar(wx, 0, ring->reg_idx, v_idx);  in wx_configure_vectors()
2132 			wx_set_ivar(wx, 1, ring->reg_idx, v_idx);  in wx_configure_vectors()
2137 	wx_set_ivar(wx, -1, 0, v_idx);  in wx_configure_vectors()
2139 		wr32(wx, WX_PX_ITR(v_idx), 1950);  in wx_configure_vectors()
2194 void wx_clean_all_rx_rings(struct wx *wx)  in wx_clean_all_rx_rings()  argument
2198 	for (i = 0; i < wx->num_rx_queues; i++)  in wx_clean_all_rx_rings()
2199 		wx_clean_rx_ring(wx->rx_ring[i]);  in wx_clean_all_rx_rings()
2236 static void wx_free_all_rx_resources(struct wx *wx)  in wx_free_all_rx_resources()  argument
2240 	for (i = 0; i < wx->num_rx_queues; i++)  in wx_free_all_rx_resources()
2241 		wx_free_rx_resources(wx->rx_ring[i]);  in wx_free_all_rx_resources()
2310 void wx_clean_all_tx_rings(struct wx *wx)  in wx_clean_all_tx_rings()  argument
2314 	for (i = 0; i < wx->num_tx_queues; i++)  in wx_clean_all_tx_rings()
2315 		wx_clean_tx_ring(wx->tx_ring[i]);  in wx_clean_all_tx_rings()
2346 static void wx_free_all_tx_resources(struct wx *wx)  in wx_free_all_tx_resources()  argument
2350 	for (i = 0; i < wx->num_tx_queues; i++)  in wx_free_all_tx_resources()
2351 		wx_free_tx_resources(wx->tx_ring[i]);  in wx_free_all_tx_resources()
2354 void wx_free_resources(struct wx *wx)  in wx_free_resources()  argument
2356 	wx_free_isb_resources(wx);  in wx_free_resources()
2357 	wx_free_all_rx_resources(wx);  in wx_free_resources()
2358 	wx_free_all_tx_resources(wx);  in wx_free_resources()
2456 static int wx_setup_all_rx_resources(struct wx *wx)  in wx_setup_all_rx_resources()  argument
2460 	for (i = 0; i < wx->num_rx_queues; i++) {  in wx_setup_all_rx_resources()
2461 		err = wx_setup_rx_resources(wx->rx_ring[i]);  in wx_setup_all_rx_resources()
2465 		wx_err(wx, "Allocation for Rx Queue %u failed\n", i);  in wx_setup_all_rx_resources()
2473 		wx_free_rx_resources(wx->rx_ring[i]);  in wx_setup_all_rx_resources()
2539 static int wx_setup_all_tx_resources(struct wx *wx)  in wx_setup_all_tx_resources()  argument
2543 	for (i = 0; i < wx->num_tx_queues; i++) {  in wx_setup_all_tx_resources()
2544 		err = wx_setup_tx_resources(wx->tx_ring[i]);  in wx_setup_all_tx_resources()
2548 		wx_err(wx, "Allocation for Tx Queue %u failed\n", i);  in wx_setup_all_tx_resources()
2556 		wx_free_tx_resources(wx->tx_ring[i]);  in wx_setup_all_tx_resources()
2560 int wx_setup_resources(struct wx *wx)  in wx_setup_resources()  argument
2565 	err = wx_setup_all_tx_resources(wx);  in wx_setup_resources()
2570 	err = wx_setup_all_rx_resources(wx);  in wx_setup_resources()
2574 	err = wx_setup_isb_resources(wx);  in wx_setup_resources()
2581 	wx_free_all_rx_resources(wx);  in wx_setup_resources()
2583 	wx_free_all_tx_resources(wx);  in wx_setup_resources()
2597 	struct wx *wx = netdev_priv(netdev);  in wx_get_stats64()  local
2601 	for (i = 0; i < wx->num_rx_queues; i++) {  in wx_get_stats64()
2602 		struct wx_ring *ring = READ_ONCE(wx->rx_ring[i]);  in wx_get_stats64()
2617 	for (i = 0; i < wx->num_tx_queues; i++) {  in wx_get_stats64()
2618 		struct wx_ring *ring = READ_ONCE(wx->tx_ring[i]);  in wx_get_stats64()
2641 	struct wx *wx = netdev_priv(netdev);  in wx_set_features()  local
2644 		wr32m(wx, WX_RDB_RA_CTL, WX_RDB_RA_CTL_RSS_EN,  in wx_set_features()
2647 		wr32m(wx, WX_RDB_RA_CTL, WX_RDB_RA_CTL_RSS_EN, 0);  in wx_set_features()