64 bool wil_is_tx_idle(struct wil6210_priv *wil)  in wil_is_tx_idle()  argument
68 	int min_ring_id = wil_get_min_tx_ring_id(wil);  in wil_is_tx_idle()
71 		struct wil_ring *vring = &wil->ring_tx[i];  in wil_is_tx_idle()
72 		int vring_index = vring - wil->ring_tx;  in wil_is_tx_idle()
74 			&wil->ring_tx_data[vring_index];  in wil_is_tx_idle()
85 		if (test_bit(wil_status_napi_en, wil->status)) {  in wil_is_tx_idle()
88 					wil_dbg_pm(wil,  in wil_is_tx_idle()
93 				wil_dbg_ratelimited(wil,  in wil_is_tx_idle()
96 				napi_synchronize(&wil->napi_tx);  in wil_is_tx_idle()
110 static int wil_vring_alloc(struct wil6210_priv *wil, struct wil_ring *vring)  in wil_vring_alloc()  argument
112 	struct device *dev = wil_to_dev(wil);  in wil_vring_alloc()
116 	wil_dbg_misc(wil, "vring_alloc:\n");  in wil_vring_alloc()
140 	if (wil->dma_addr_size > 32)  in wil_vring_alloc()
150 	if (wil->dma_addr_size > 32)  in wil_vring_alloc()
152 					  DMA_BIT_MASK(wil->dma_addr_size));  in wil_vring_alloc()
165 	wil_dbg_misc(wil, "vring[%d] 0x%p:%pad 0x%p\n", vring->size,  in wil_vring_alloc()
190 static void wil_vring_free(struct wil6210_priv *wil, struct wil_ring *vring)  in wil_vring_free()  argument
192 	struct device *dev = wil_to_dev(wil);  in wil_vring_free()
195 	lockdep_assert_held(&wil->mutex);  in wil_vring_free()
197 		int vring_index = vring - wil->ring_tx;  in wil_vring_free()
199 		wil_dbg_misc(wil, "free Tx vring %d [%d] 0x%p:%pad 0x%p\n",  in wil_vring_free()
203 		wil_dbg_misc(wil, "free Rx vring [%d] 0x%p:%pad 0x%p\n",  in wil_vring_free()
220 				wil_dbg_txrx(wil,  in wil_vring_free()
256 static int wil_vring_alloc_skb(struct wil6210_priv *wil, struct wil_ring *vring,  in wil_vring_alloc_skb()  argument
259 	struct device *dev = wil_to_dev(wil);  in wil_vring_alloc_skb()
260 	unsigned int sz = wil->rx_buf_len + ETH_HLEN + wil_rx_snaplen();  in wil_vring_alloc_skb()
305 static void wil_rx_add_radiotap_header(struct wil6210_priv *wil,  in wil_rx_add_radiotap_header()  argument
324 	struct ieee80211_channel *ch = wil->monitor_chandef.chan;  in wil_rx_add_radiotap_header()
328 		wil_err(wil, "Unable to expand headroom to %d\n", rtap_len);  in wil_rx_add_radiotap_header()
351 static bool wil_is_rx_idle(struct wil6210_priv *wil)  in wil_is_rx_idle()  argument
354 	struct wil_ring *ring = &wil->ring_rx;  in wil_is_rx_idle()
363 static int wil_rx_get_cid_by_skb(struct wil6210_priv *wil, struct sk_buff *skb)  in wil_rx_get_cid_by_skb()  argument
367 	struct wil6210_vif *vif = wil->vifs[mid];  in wil_rx_get_cid_by_skb()
386 			wil_err_ratelimited(wil,  in wil_rx_get_cid_by_skb()
394 			wil_err_ratelimited(wil, "Short frame, len = %d\n",  in wil_rx_get_cid_by_skb()
402 	if (wil->max_assoc_sta <= WIL6210_RX_DESC_MAX_CID)  in wil_rx_get_cid_by_skb()
417 	for (i = cid; i < wil->max_assoc_sta; i += WIL6210_RX_DESC_MAX_CID) {  in wil_rx_get_cid_by_skb()
418 		if (wil->sta[i].status != wil_sta_unused &&  in wil_rx_get_cid_by_skb()
419 		    ether_addr_equal(wil->sta[i].addr, ta)) {  in wil_rx_get_cid_by_skb()
424 	if (i >= wil->max_assoc_sta) {  in wil_rx_get_cid_by_skb()
425 …wil_err_ratelimited(wil, "Could not find cid for frame with transmit addr = %pM, iftype = %d, fram…  in wil_rx_get_cid_by_skb()
439 static struct sk_buff *wil_vring_reap_rx(struct wil6210_priv *wil,  in wil_vring_reap_rx()  argument
442 	struct device *dev = wil_to_dev(wil);  in wil_vring_reap_rx()
450 	unsigned int sz = wil->rx_buf_len + ETH_HLEN + snaplen;  in wil_vring_reap_rx()
474 		wil_err(wil, "No Rx skb at [%d]\n", i);  in wil_vring_reap_rx()
485 	wil_dbg_txrx(wil, "Rx[%3d] : %d bytes\n", i, dmalen);  in wil_vring_reap_rx()
490 	vif = wil->vifs[mid];  in wil_vring_reap_rx()
493 		wil_dbg_txrx(wil, "skipped RX descriptor with invalid mid %d",  in wil_vring_reap_rx()
500 		wil_err_ratelimited(wil, "Rx size too large: %d bytes!\n",  in wil_vring_reap_rx()
512 	cid = wil_rx_get_cid_by_skb(wil, skb);  in wil_vring_reap_rx()
518 	stats = &wil->sta[cid].stats;  in wil_vring_reap_rx()
526 		wil_rx_add_radiotap_header(wil, skb);  in wil_vring_reap_rx()
541 		wil_dbg_txrx(wil,  in wil_vring_reap_rx()
546 			wil_dbg_txrx(wil,  in wil_vring_reap_rx()
549 			wil_rx_bar(wil, vif, cid, tid, seq);  in wil_vring_reap_rx()
554 			wil_dbg_txrx(wil,  in wil_vring_reap_rx()
605 static int wil_rx_refill(struct wil6210_priv *wil, int count)  in wil_rx_refill()  argument
607 	struct net_device *ndev = wil->main_ndev;  in wil_rx_refill()
608 	struct wil_ring *v = &wil->ring_rx;  in wil_rx_refill()
617 		rc = wil_vring_alloc_skb(wil, v, v->swtail, headroom);  in wil_rx_refill()
619 			wil_err_ratelimited(wil, "Error %d in rx refill[%d]\n",  in wil_rx_refill()
630 	wil_w(wil, v->hwtail, v->swtail);  in wil_rx_refill()
658 static int wil_rx_crypto_check(struct wil6210_priv *wil, struct sk_buff *skb)  in wil_rx_crypto_check()  argument
665 	struct wil_sta_info *s = &wil->sta[cid];  in wil_rx_crypto_check()
672 		wil_err_ratelimited(wil,  in wil_rx_crypto_check()
679 		wil_err_ratelimited(wil,  in wil_rx_crypto_check()
689 static int wil_rx_error_check(struct wil6210_priv *wil, struct sk_buff *skb,  in wil_rx_error_check()  argument
697 		wil_dbg_txrx(wil, "MIC error, dropping packet\n");  in wil_rx_error_check()
719 static struct wil_eapol_key *wil_is_ptk_eapol_key(struct wil6210_priv *wil,  in wil_is_ptk_eapol_key()  argument
729 		wil_err(wil, "mac header was not set\n");  in wil_is_ptk_eapol_key()
757 static bool wil_skb_is_eap_3(struct wil6210_priv *wil, struct sk_buff *skb)  in wil_skb_is_eap_3()  argument
762 	key = wil_is_ptk_eapol_key(wil, skb);  in wil_skb_is_eap_3()
770 		wil_dbg_misc(wil, "EAPOL key message 3\n");  in wil_skb_is_eap_3()
774 	wil_dbg_misc(wil, "EAPOL key message 1\n");  in wil_skb_is_eap_3()
779 static bool wil_skb_is_eap_4(struct wil6210_priv *wil, struct sk_buff *skb)  in wil_skb_is_eap_4()  argument
784 	key = wil_is_ptk_eapol_key(wil, skb);  in wil_skb_is_eap_4()
792 			wil_dbg_misc(wil, "EAPOL key message 2\n");  in wil_skb_is_eap_4()
796 	wil_dbg_misc(wil, "EAPOL key message 4\n");  in wil_skb_is_eap_4()
805 	struct wil6210_priv *wil = vif_to_wil(vif);  in wil_enable_tx_key_worker()  local
810 		wil_dbg_misc(wil, "Invalid rekey state = %d\n",  in wil_enable_tx_key_worker()
816 	cid =  wil_find_cid_by_idx(wil, vif->mid, 0);  in wil_enable_tx_key_worker()
817 	if (!wil_cid_valid(wil, cid)) {  in wil_enable_tx_key_worker()
818 		wil_err(wil, "Invalid cid = %d\n", cid);  in wil_enable_tx_key_worker()
823 	wil_dbg_misc(wil, "Apply PTK key after eapol was sent out\n");  in wil_enable_tx_key_worker()
824 	rc = wmi_add_cipher_key(vif, 0, wil->sta[cid].addr, 0, NULL,  in wil_enable_tx_key_worker()
831 		wil_err(wil, "Apply PTK key failed %d\n", rc);  in wil_enable_tx_key_worker()
836 	struct wil6210_priv *wil = vif_to_wil(vif);  in wil_tx_complete_handle_eapol()  local
841 	    !test_bit(WMI_FW_CAPABILITY_SPLIT_REKEY, wil->fw_capabilities))  in wil_tx_complete_handle_eapol()
845 	if (!wil_skb_is_eap_4(wil, skb))  in wil_tx_complete_handle_eapol()
848 	spin_lock_bh(&wil->eap_lock);  in wil_tx_complete_handle_eapol()
860 		wil_err(wil, "Unknown rekey state = %d",  in wil_tx_complete_handle_eapol()
863 	spin_unlock_bh(&wil->eap_lock);  in wil_tx_complete_handle_eapol()
866 		q = queue_work(wil->wmi_wq, &vif->enable_tx_key_worker);  in wil_tx_complete_handle_eapol()
867 		wil_dbg_misc(wil, "queue_work of enable_tx_key_worker -> %d\n",  in wil_tx_complete_handle_eapol()
874 	struct wil6210_priv *wil = vif_to_wil(vif);  in wil_rx_handle_eapol()  local
878 	    !test_bit(WMI_FW_CAPABILITY_SPLIT_REKEY, wil->fw_capabilities))  in wil_rx_handle_eapol()
882 	if (!wil_skb_is_eap_3(wil, skb))  in wil_rx_handle_eapol()
897 	struct wil6210_priv *wil = ndev_to_wil(ndev);  in wil_netif_rx()  local
914 			wil_dbg_txrx(wil, "Rx drop %d bytes\n", len);  in wil_netif_rx()
924 			int xmit_cid = wil_find_cid(wil, vif->mid, da);  in wil_netif_rx()
947 		wil_dbg_txrx(wil, "Rx -> Tx %d bytes\n", len);  in wil_netif_rx()
959 			napi_gro_receive(&wil->napi_rx, skb);  in wil_netif_rx()
974 	struct wil6210_priv *wil = ndev_to_wil(ndev);  in wil_netif_rx_any()  local
977 	wil->txrx_ops.get_netif_rx_params(skb, &cid, &security);  in wil_netif_rx_any()
979 	stats = &wil->sta[cid].stats;  in wil_netif_rx_any()
983 	if (security && (wil->txrx_ops.rx_crypto_check(wil, skb) != 0)) {  in wil_netif_rx_any()
984 		wil_dbg_txrx(wil, "Rx drop %d bytes\n", skb->len);  in wil_netif_rx_any()
993 	if (unlikely(wil->txrx_ops.rx_error_check(wil, skb, stats))) {  in wil_netif_rx_any()
1005 void wil_rx_handle(struct wil6210_priv *wil, int *quota)  in wil_rx_handle()  argument
1007 	struct net_device *ndev = wil->main_ndev;  in wil_rx_handle()
1009 	struct wil_ring *v = &wil->ring_rx;  in wil_rx_handle()
1013 		wil_err(wil, "Rx IRQ while Rx not yet initialized\n");  in wil_rx_handle()
1016 	wil_dbg_txrx(wil, "rx_handle\n");  in wil_rx_handle()
1017 	while ((*quota > 0) && (NULL != (skb = wil_vring_reap_rx(wil, v)))) {  in wil_rx_handle()
1029 			wil_rx_reorder(wil, skb);  in wil_rx_handle()
1032 	wil_rx_refill(wil, v->size);  in wil_rx_handle()
1035 static void wil_rx_buf_len_init(struct wil6210_priv *wil)  in wil_rx_buf_len_init()  argument
1037 	wil->rx_buf_len = rx_large_buf ?  in wil_rx_buf_len_init()
1039 	if (mtu_max > wil->rx_buf_len) {  in wil_rx_buf_len_init()
1044 		wil_info(wil, "Override RX buffer to mtu_max(%d)\n", mtu_max);  in wil_rx_buf_len_init()
1045 		wil->rx_buf_len = mtu_max;  in wil_rx_buf_len_init()
1049 static int wil_rx_init(struct wil6210_priv *wil, uint order)  in wil_rx_init()  argument
1051 	struct wil_ring *vring = &wil->ring_rx;  in wil_rx_init()
1054 	wil_dbg_misc(wil, "rx_init\n");  in wil_rx_init()
1057 		wil_err(wil, "Rx ring already allocated\n");  in wil_rx_init()
1061 	wil_rx_buf_len_init(wil);  in wil_rx_init()
1065 	rc = wil_vring_alloc(wil, vring);  in wil_rx_init()
1069 	rc = wmi_rx_chain_add(wil, vring);  in wil_rx_init()
1073 	rc = wil_rx_refill(wil, vring->size);  in wil_rx_init()
1079 	wil_vring_free(wil, vring);  in wil_rx_init()
1084 static void wil_rx_fini(struct wil6210_priv *wil)  in wil_rx_fini()  argument
1086 	struct wil_ring *vring = &wil->ring_rx;  in wil_rx_fini()
1088 	wil_dbg_misc(wil, "rx_fini\n");  in wil_rx_fini()
1091 		wil_vring_free(wil, vring);  in wil_rx_fini()
1137 	struct wil6210_priv *wil = vif_to_wil(vif);  in wil_vring_init_tx()  local
1164 	struct wil_ring *vring = &wil->ring_tx[id];  in wil_vring_init_tx()
1165 	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[id];  in wil_vring_init_tx()
1175 	wil_dbg_misc(wil, "vring_init_tx: max_mpdu_size %d\n",  in wil_vring_init_tx()
1177 	lockdep_assert_held(&wil->mutex);  in wil_vring_init_tx()
1180 		wil_err(wil, "Tx ring [%d] already allocated\n", id);  in wil_vring_init_tx()
1188 	rc = wil_vring_alloc(wil, vring);  in wil_vring_init_tx()
1192 	wil->ring2cid_tid[id][0] = cid;  in wil_vring_init_tx()
1193 	wil->ring2cid_tid[id][1] = tid;  in wil_vring_init_tx()
1199 	rc = wmi_call(wil, WMI_VRING_CFG_CMDID, vif->mid, &cmd, sizeof(cmd),  in wil_vring_init_tx()
1206 		wil_err(wil, "Tx config failed, status 0x%02x\n",  in wil_vring_init_tx()
1219 		wil_addba_tx_request(wil, id, agg_wsize);  in wil_vring_init_tx()
1227 	wil_vring_free(wil, vring);  in wil_vring_init_tx()
1228 	wil->ring2cid_tid[id][0] = wil->max_assoc_sta;  in wil_vring_init_tx()
1229 	wil->ring2cid_tid[id][1] = 0;  in wil_vring_init_tx()
1239 	struct wil6210_priv *wil = vif_to_wil(vif);  in wil_tx_vring_modify()  local
1267 	struct wil_ring *vring = &wil->ring_tx[ring_id];  in wil_tx_vring_modify()
1268 	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_id];  in wil_tx_vring_modify()
1270 	wil_dbg_misc(wil, "vring_modify: ring %d cid %d tid %d\n", ring_id,  in wil_tx_vring_modify()
1272 	lockdep_assert_held(&wil->mutex);  in wil_tx_vring_modify()
1275 		wil_err(wil, "Tx ring [%d] not allocated\n", ring_id);  in wil_tx_vring_modify()
1279 	if (wil->ring2cid_tid[ring_id][0] != cid ||  in wil_tx_vring_modify()
1280 	    wil->ring2cid_tid[ring_id][1] != tid) {  in wil_tx_vring_modify()
1281 		wil_err(wil, "ring info does not match cid=%u tid=%u\n",  in wil_tx_vring_modify()
1282 			wil->ring2cid_tid[ring_id][0],  in wil_tx_vring_modify()
1283 			wil->ring2cid_tid[ring_id][1]);  in wil_tx_vring_modify()
1288 	rc = wmi_call(wil, WMI_VRING_CFG_CMDID, vif->mid, &cmd, sizeof(cmd),  in wil_tx_vring_modify()
1295 		wil_err(wil, "Tx modify failed, status 0x%02x\n",  in wil_tx_vring_modify()
1306 		wil_addba_tx_request(wil, ring_id, agg_wsize);  in wil_tx_vring_modify()
1314 	wil->ring2cid_tid[ring_id][0] = wil->max_assoc_sta;  in wil_tx_vring_modify()
1315 	wil->ring2cid_tid[ring_id][1] = 0;  in wil_tx_vring_modify()
1321 	struct wil6210_priv *wil = vif_to_wil(vif);  in wil_vring_init_bcast()  local
1341 	struct wil_ring *vring = &wil->ring_tx[id];  in wil_vring_init_bcast()
1342 	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[id];  in wil_vring_init_bcast()
1344 	wil_dbg_misc(wil, "vring_init_bcast: max_mpdu_size %d\n",  in wil_vring_init_bcast()
1346 	lockdep_assert_held(&wil->mutex);  in wil_vring_init_bcast()
1349 		wil_err(wil, "Tx ring [%d] already allocated\n", id);  in wil_vring_init_bcast()
1357 	rc = wil_vring_alloc(wil, vring);  in wil_vring_init_bcast()
1361 	wil->ring2cid_tid[id][0] = wil->max_assoc_sta; /* CID */  in wil_vring_init_bcast()
1362 	wil->ring2cid_tid[id][1] = 0; /* TID */  in wil_vring_init_bcast()
1368 	rc = wmi_call(wil, WMI_BCAST_VRING_CFG_CMDID, vif->mid,  in wil_vring_init_bcast()
1376 		wil_err(wil, "Tx config failed, status 0x%02x\n",  in wil_vring_init_bcast()
1394 	wil_vring_free(wil, vring);  in wil_vring_init_bcast()
1400 static struct wil_ring *wil_find_tx_ucast(struct wil6210_priv *wil,  in wil_find_tx_ucast()  argument
1406 	int min_ring_id = wil_get_min_tx_ring_id(wil);  in wil_find_tx_ucast()
1408 	cid = wil_find_cid(wil, vif->mid, da);  in wil_find_tx_ucast()
1410 	if (cid < 0 || cid >= wil->max_assoc_sta)  in wil_find_tx_ucast()
1414 	for (i = min_ring_id; i < ARRAY_SIZE(wil->ring2cid_tid); i++) {  in wil_find_tx_ucast()
1415 		if (!wil->ring_tx_data[i].dot1x_open &&  in wil_find_tx_ucast()
1418 		if (wil->ring2cid_tid[i][0] == cid) {  in wil_find_tx_ucast()
1419 			struct wil_ring *v = &wil->ring_tx[i];  in wil_find_tx_ucast()
1420 			struct wil_ring_tx_data *txdata = &wil->ring_tx_data[i];  in wil_find_tx_ucast()
1422 			wil_dbg_txrx(wil, "find_tx_ucast: (%pM) -> [%d]\n",  in wil_find_tx_ucast()
1427 				wil_dbg_txrx(wil,  in wil_find_tx_ucast()
1438 static int wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif,
1441 static struct wil_ring *wil_find_tx_ring_sta(struct wil6210_priv *wil,  in wil_find_tx_ring_sta()  argument
1449 	int min_ring_id = wil_get_min_tx_ring_id(wil);  in wil_find_tx_ring_sta()
1456 		ring = &wil->ring_tx[i];  in wil_find_tx_ring_sta()
1457 		txdata = &wil->ring_tx_data[i];  in wil_find_tx_ring_sta()
1461 		cid = wil->ring2cid_tid[i][0];  in wil_find_tx_ring_sta()
1462 		if (cid >= wil->max_assoc_sta) /* skip BCAST */  in wil_find_tx_ring_sta()
1465 		if (!wil->ring_tx_data[i].dot1x_open &&  in wil_find_tx_ring_sta()
1469 		wil_dbg_txrx(wil, "Tx -> ring %d\n", i);  in wil_find_tx_ring_sta()
1474 	wil_dbg_txrx(wil, "Tx while no rings active?\n");  in wil_find_tx_ring_sta()
1490 static struct wil_ring *wil_find_tx_bcast_1(struct wil6210_priv *wil,  in wil_find_tx_bcast_1()  argument
1500 	v = &wil->ring_tx[i];  in wil_find_tx_bcast_1()
1501 	txdata = &wil->ring_tx_data[i];  in wil_find_tx_bcast_1()
1504 	if (!wil->ring_tx_data[i].dot1x_open &&  in wil_find_tx_bcast_1()
1514 static bool wil_check_multicast_to_unicast(struct wil6210_priv *wil,  in wil_check_multicast_to_unicast()  argument
1521 	if (!wil->multicast_to_unicast)  in wil_check_multicast_to_unicast()
1540 static void wil_set_da_for_vring(struct wil6210_priv *wil,  in wil_set_da_for_vring()  argument
1544 	int cid = wil->ring2cid_tid[vring_index][0];  in wil_set_da_for_vring()
1546 	ether_addr_copy(da, wil->sta[cid].addr);  in wil_set_da_for_vring()
1549 static struct wil_ring *wil_find_tx_bcast_2(struct wil6210_priv *wil,  in wil_find_tx_bcast_2()  argument
1559 	int min_ring_id = wil_get_min_tx_ring_id(wil);  in wil_find_tx_bcast_2()
1563 		v = &wil->ring_tx[i];  in wil_find_tx_bcast_2()
1564 		txdata = &wil->ring_tx_data[i];  in wil_find_tx_bcast_2()
1568 		cid = wil->ring2cid_tid[i][0];  in wil_find_tx_bcast_2()
1569 		if (cid >= wil->max_assoc_sta) /* skip BCAST */  in wil_find_tx_bcast_2()
1571 		if (!wil->ring_tx_data[i].dot1x_open &&  in wil_find_tx_bcast_2()
1576 		if (0 == memcmp(wil->sta[cid].addr, src, ETH_ALEN))  in wil_find_tx_bcast_2()
1582 	wil_dbg_txrx(wil, "Tx while no vrings active?\n");  in wil_find_tx_bcast_2()
1587 	wil_dbg_txrx(wil, "BCAST -> ring %d\n", i);  in wil_find_tx_bcast_2()
1588 	wil_set_da_for_vring(wil, skb, i);  in wil_find_tx_bcast_2()
1592 		v2 = &wil->ring_tx[i];  in wil_find_tx_bcast_2()
1593 		txdata2 = &wil->ring_tx_data[i];  in wil_find_tx_bcast_2()
1596 		cid = wil->ring2cid_tid[i][0];  in wil_find_tx_bcast_2()
1597 		if (cid >= wil->max_assoc_sta) /* skip BCAST */  in wil_find_tx_bcast_2()
1599 		if (!wil->ring_tx_data[i].dot1x_open &&  in wil_find_tx_bcast_2()
1603 		if (0 == memcmp(wil->sta[cid].addr, src, ETH_ALEN))  in wil_find_tx_bcast_2()
1608 			wil_dbg_txrx(wil, "BCAST DUP -> ring %d\n", i);  in wil_find_tx_bcast_2()
1609 			wil_set_da_for_vring(wil, skb2, i);  in wil_find_tx_bcast_2()
1610 			wil_tx_ring(wil, vif, v2, skb2);  in wil_find_tx_bcast_2()
1614 			wil_err(wil, "skb_copy failed\n");  in wil_find_tx_bcast_2()
1725 static int __wil_tx_vring_tso(struct wil6210_priv *wil, struct wil6210_vif *vif,  in __wil_tx_vring_tso()  argument
1728 	struct device *dev = wil_to_dev(wil);  in __wil_tx_vring_tso()
1751 	int vring_index = vring - wil->ring_tx;  in __wil_tx_vring_tso()
1752 	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[vring_index];  in __wil_tx_vring_tso()
1766 	wil_dbg_txrx(wil, "tx_vring_tso: %d bytes to vring %d\n", skb->len,  in __wil_tx_vring_tso()
1778 		wil_err_ratelimited(wil,  in __wil_tx_vring_tso()
1824 		wil_err(wil, "TSO: Skb head DMA map error\n");  in __wil_tx_vring_tso()
1828 	wil->txrx_ops.tx_desc_map((union wil_tx_desc *)hdr_desc, pa,  in __wil_tx_vring_tso()
1843 			wil_dbg_txrx(wil, "TSO: process skb head, len %u\n",  in __wil_tx_vring_tso()
1848 			wil_dbg_txrx(wil, "TSO: frag[%d]: len %u\n", f, len);  in __wil_tx_vring_tso()
1852 			wil_dbg_txrx(wil,  in __wil_tx_vring_tso()
1857 				wil_err_ratelimited(wil, "TSO: ring overflow\n");  in __wil_tx_vring_tso()
1864 			wil_dbg_txrx(wil, "TSO: lenmss %d, i %d\n", lenmss, i);  in __wil_tx_vring_tso()
1882 				wil_err(wil, "TSO: DMA map page error\n");  in __wil_tx_vring_tso()
1896 			wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d,  in __wil_tx_vring_tso()
1910 			wil_dbg_txrx(wil,  in __wil_tx_vring_tso()
1979 	if (wil_val_in_range(wil->ring_idle_trsh,  in __wil_tx_vring_tso()
1982 		wil_dbg_txrx(wil,  "Ring[%2d] not idle %d -> %d\n",  in __wil_tx_vring_tso()
1995 	wil_dbg_txrx(wil, "TSO: Tx swhead %d -> %d\n", swhead, vring->swhead);  in __wil_tx_vring_tso()
2002 	if (wil->tx_latency)  in __wil_tx_vring_tso()
2007 	wil_w(wil, vring->hwtail, vring->swhead);  in __wil_tx_vring_tso()
2028 static int __wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif,  in __wil_tx_ring()  argument
2031 	struct device *dev = wil_to_dev(wil);  in __wil_tx_ring()
2038 	int ring_index = ring - wil->ring_tx;  in __wil_tx_ring()
2039 	struct wil_ring_tx_data  *txdata = &wil->ring_tx_data[ring_index];  in __wil_tx_ring()
2046 	wil_dbg_txrx(wil, "tx_ring: %d bytes to ring %d, nr_frags %d\n",  in __wil_tx_ring()
2053 		wil_err_ratelimited(wil,  in __wil_tx_ring()
2062 	wil_dbg_txrx(wil, "Tx[%2d] skb %d bytes 0x%p -> %pad\n", ring_index,  in __wil_tx_ring()
2071 	wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d, pa, len,  in __wil_tx_ring()
2080 		wil_err(wil, "Tx[%2d] Failed to set cksum, drop packet\n",  in __wil_tx_ring()
2094 		wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", ring_index, i);  in __wil_tx_ring()
2102 			wil_err(wil, "Tx[%2d] failed to map fragment\n",  in __wil_tx_ring()
2107 		wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d,  in __wil_tx_ring()
2120 	wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", ring_index, i);  in __wil_tx_ring()
2132 	if (wil_val_in_range(wil->ring_idle_trsh,  in __wil_tx_ring()
2135 		wil_dbg_txrx(wil,  "Ring[%2d] not idle %d -> %d\n",  in __wil_tx_ring()
2148 	wil_dbg_txrx(wil, "Tx[%2d] swhead %d -> %d\n", ring_index, swhead,  in __wil_tx_ring()
2157 	if (wil->tx_latency)  in __wil_tx_ring()
2162 	wil_w(wil, ring->hwtail, ring->swhead);  in __wil_tx_ring()
2176 		wil->txrx_ops.tx_desc_unmap(dev,  in __wil_tx_ring()
2186 static int wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif,  in wil_tx_ring()  argument
2189 	int ring_index = ring - wil->ring_tx;  in wil_tx_ring()
2190 	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_index];  in wil_tx_ring()
2195 	if (test_bit(wil_status_suspending, wil->status) ||  in wil_tx_ring()
2196 	    test_bit(wil_status_suspended, wil->status) ||  in wil_tx_ring()
2197 	    test_bit(wil_status_resuming, wil->status)) {  in wil_tx_ring()
2198 		wil_dbg_txrx(wil,  in wil_tx_ring()
2204 	rc = (skb_is_gso(skb) ? wil->txrx_ops.tx_ring_tso : __wil_tx_ring)  in wil_tx_ring()
2205 	     (wil, vif, ring, skb);  in wil_tx_ring()
2228 static inline void __wil_update_net_queues(struct wil6210_priv *wil,  in __wil_update_net_queues()  argument
2234 	int min_ring_id = wil_get_min_tx_ring_id(wil);  in __wil_update_net_queues()
2240 		wil_dbg_txrx(wil, "vring %d, mid %d, check_stop=%d, stopped=%d",  in __wil_update_net_queues()
2241 			     (int)(ring - wil->ring_tx), vif->mid, check_stop,  in __wil_update_net_queues()
2244 		wil_dbg_txrx(wil, "check_stop=%d, mid=%d, stopped=%d",  in __wil_update_net_queues()
2260 			wil_dbg_txrx(wil, "netif_tx_stop called\n");  in __wil_update_net_queues()
2266 	if (test_bit(wil_status_suspending, wil->status) ||  in __wil_update_net_queues()
2267 	    test_bit(wil_status_suspended, wil->status))  in __wil_update_net_queues()
2272 		struct wil_ring *cur_ring = &wil->ring_tx[i];  in __wil_update_net_queues()
2273 		struct wil_ring_tx_data  *txdata = &wil->ring_tx_data[i];  in __wil_update_net_queues()
2280 			wil_dbg_txrx(wil, "ring %d full, can't wake\n",  in __wil_update_net_queues()
2281 				     (int)(cur_ring - wil->ring_tx));  in __wil_update_net_queues()
2288 		wil_dbg_txrx(wil, "calling netif_tx_wake\n");  in __wil_update_net_queues()
2294 void wil_update_net_queues(struct wil6210_priv *wil, struct wil6210_vif *vif,  in wil_update_net_queues()  argument
2297 	spin_lock(&wil->net_queue_lock);  in wil_update_net_queues()
2298 	__wil_update_net_queues(wil, vif, ring, check_stop);  in wil_update_net_queues()
2299 	spin_unlock(&wil->net_queue_lock);  in wil_update_net_queues()
2302 void wil_update_net_queues_bh(struct wil6210_priv *wil, struct wil6210_vif *vif,  in wil_update_net_queues_bh()  argument
2305 	spin_lock_bh(&wil->net_queue_lock);  in wil_update_net_queues_bh()
2306 	__wil_update_net_queues(wil, vif, ring, check_stop);  in wil_update_net_queues_bh()
2307 	spin_unlock_bh(&wil->net_queue_lock);  in wil_update_net_queues_bh()
2313 	struct wil6210_priv *wil = vif_to_wil(vif);  in wil_start_xmit()  local
2320 	wil_dbg_txrx(wil, "start_xmit\n");  in wil_start_xmit()
2321 	if (unlikely(!test_bit(wil_status_fwready, wil->status))) {  in wil_start_xmit()
2323 			wil_err(wil, "FW not ready\n");  in wil_start_xmit()
2329 		wil_dbg_ratelimited(wil,  in wil_start_xmit()
2334 		wil_err(wil, "Xmit in monitor mode not supported\n");  in wil_start_xmit()
2342 		ring = wil_find_tx_ring_sta(wil, vif, skb);  in wil_start_xmit()
2344 		if (vif->pbss || wil_check_multicast_to_unicast(wil, skb))  in wil_start_xmit()
2348 			ring = wil_find_tx_bcast_2(wil, vif, skb);  in wil_start_xmit()
2351 			ring = wil_find_tx_bcast_1(wil, vif, skb);  in wil_start_xmit()
2356 			ring = wil_find_tx_bcast_2(wil, vif, skb);  in wil_start_xmit()
2359 		ring = wil_find_tx_ucast(wil, vif, skb);  in wil_start_xmit()
2362 		wil_dbg_txrx(wil, "No Tx RING found for %pM\n", da);  in wil_start_xmit()
2366 	rc = wil_tx_ring(wil, vif, ring, skb);  in wil_start_xmit()
2371 		wil_update_net_queues_bh(wil, vif, ring, true);  in wil_start_xmit()
2389 void wil_tx_latency_calc(struct wil6210_priv *wil, struct sk_buff *skb,  in wil_tx_latency_calc()  argument
2395 	if (!wil->tx_latency)  in wil_tx_latency_calc()
2402 	bin = skb_time_us / wil->tx_latency_res;  in wil_tx_latency_calc()
2405 	wil_dbg_txrx(wil, "skb time %dus => bin %d\n", skb_time_us, bin);  in wil_tx_latency_calc()
2422 	struct wil6210_priv *wil = vif_to_wil(vif);  in wil_tx_complete()  local
2424 	struct device *dev = wil_to_dev(wil);  in wil_tx_complete()
2425 	struct wil_ring *vring = &wil->ring_tx[ringid];  in wil_tx_complete()
2426 	struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ringid];  in wil_tx_complete()
2428 	int cid = wil->ring2cid_tid[ringid][0];  in wil_tx_complete()
2435 		wil_err(wil, "Tx irq[%d]: vring not initialized\n", ringid);  in wil_tx_complete()
2440 		wil_info(wil, "Tx irq[%d]: vring disabled\n", ringid);  in wil_tx_complete()
2444 	wil_dbg_txrx(wil, "tx_complete: (%d)\n", ringid);  in wil_tx_complete()
2448 	if (cid < wil->max_assoc_sta)  in wil_tx_complete()
2449 		stats = &wil->sta[cid].stats;  in wil_tx_complete()
2480 			wil_dbg_txrx(wil,  in wil_tx_complete()
2487 			wil->txrx_ops.tx_desc_unmap(dev,  in wil_tx_complete()
2499 						wil_tx_latency_calc(wil, skb,  in wil_tx_complete()
2500 							&wil->sta[cid]);  in wil_tx_complete()
2532 	if (wil_val_in_range(wil->ring_idle_trsh,  in wil_tx_complete()
2534 		wil_dbg_txrx(wil, "Ring[%2d] idle %d -> %d\n",  in wil_tx_complete()
2541 		wil_update_net_queues(wil, vif, vring, false);  in wil_tx_complete()
2546 static inline int wil_tx_init(struct wil6210_priv *wil)  in wil_tx_init()  argument
2551 static inline void wil_tx_fini(struct wil6210_priv *wil) {}  in wil_tx_fini()  argument
2553 static void wil_get_reorder_params(struct wil6210_priv *wil,  in wil_get_reorder_params()  argument
2567 void wil_init_txrx_ops_legacy_dma(struct wil6210_priv *wil)  in wil_init_txrx_ops_legacy_dma()  argument
2569 	wil->txrx_ops.configure_interrupt_moderation =  in wil_init_txrx_ops_legacy_dma()
2572 	wil->txrx_ops.tx_desc_map = wil_tx_desc_map;  in wil_init_txrx_ops_legacy_dma()
2573 	wil->txrx_ops.tx_desc_unmap = wil_txdesc_unmap;  in wil_init_txrx_ops_legacy_dma()
2574 	wil->txrx_ops.tx_ring_tso =  __wil_tx_vring_tso;  in wil_init_txrx_ops_legacy_dma()
2575 	wil->txrx_ops.ring_init_tx = wil_vring_init_tx;  in wil_init_txrx_ops_legacy_dma()
2576 	wil->txrx_ops.ring_fini_tx = wil_vring_free;  in wil_init_txrx_ops_legacy_dma()
2577 	wil->txrx_ops.ring_init_bcast = wil_vring_init_bcast;  in wil_init_txrx_ops_legacy_dma()
2578 	wil->txrx_ops.tx_init = wil_tx_init;  in wil_init_txrx_ops_legacy_dma()
2579 	wil->txrx_ops.tx_fini = wil_tx_fini;  in wil_init_txrx_ops_legacy_dma()
2580 	wil->txrx_ops.tx_ring_modify = wil_tx_vring_modify;  in wil_init_txrx_ops_legacy_dma()
2582 	wil->txrx_ops.rx_init = wil_rx_init;  in wil_init_txrx_ops_legacy_dma()
2583 	wil->txrx_ops.wmi_addba_rx_resp = wmi_addba_rx_resp;  in wil_init_txrx_ops_legacy_dma()
2584 	wil->txrx_ops.get_reorder_params = wil_get_reorder_params;  in wil_init_txrx_ops_legacy_dma()
2585 	wil->txrx_ops.get_netif_rx_params =  in wil_init_txrx_ops_legacy_dma()
2587 	wil->txrx_ops.rx_crypto_check = wil_rx_crypto_check;  in wil_init_txrx_ops_legacy_dma()
2588 	wil->txrx_ops.rx_error_check = wil_rx_error_check;  in wil_init_txrx_ops_legacy_dma()
2589 	wil->txrx_ops.is_rx_idle = wil_is_rx_idle;  in wil_init_txrx_ops_legacy_dma()
2590 	wil->txrx_ops.rx_fini = wil_rx_fini;  in wil_init_txrx_ops_legacy_dma()