115 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)  in vmxnet3_tq_stopped()  argument
117 	return tq->stopped;  in vmxnet3_tq_stopped()
122 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)  in vmxnet3_tq_start()  argument
124 	tq->stopped = false;  in vmxnet3_tq_start()
125 	netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);  in vmxnet3_tq_start()
130 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)  in vmxnet3_tq_wake()  argument
132 	tq->stopped = false;  in vmxnet3_tq_wake()
133 	netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));  in vmxnet3_tq_wake()
138 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)  in vmxnet3_tq_stop()  argument
140 	tq->stopped = true;  in vmxnet3_tq_stop()
141 	tq->num_stop++;  in vmxnet3_tq_stop()
142 	netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));  in vmxnet3_tq_stop()
355 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,  in vmxnet3_unmap_pkt()  argument
362 	BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);  in vmxnet3_unmap_pkt()
363 	BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);  in vmxnet3_unmap_pkt()
365 	skb = tq->buf_info[eop_idx].skb;  in vmxnet3_unmap_pkt()
367 	tq->buf_info[eop_idx].skb = NULL;  in vmxnet3_unmap_pkt()
369 	VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);  in vmxnet3_unmap_pkt()
371 	while (tq->tx_ring.next2comp != eop_idx) {  in vmxnet3_unmap_pkt()
372 		vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,  in vmxnet3_unmap_pkt()
380 		vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);  in vmxnet3_unmap_pkt()
390 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,  in vmxnet3_tq_tx_complete()  argument
396 	gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;  in vmxnet3_tq_tx_complete()
397 	while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {  in vmxnet3_tq_tx_complete()
404 					       &gdesc->tcd), tq, adapter->pdev,  in vmxnet3_tq_tx_complete()
407 		vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);  in vmxnet3_tq_tx_complete()
408 		gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;  in vmxnet3_tq_tx_complete()
412 		spin_lock(&tq->tx_lock);  in vmxnet3_tq_tx_complete()
413 		if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&  in vmxnet3_tq_tx_complete()
414 			     vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >  in vmxnet3_tq_tx_complete()
415 			     VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&  in vmxnet3_tq_tx_complete()
417 			vmxnet3_tq_wake(tq, adapter);  in vmxnet3_tq_tx_complete()
419 		spin_unlock(&tq->tx_lock);  in vmxnet3_tq_tx_complete()
426 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,  in vmxnet3_tq_cleanup()  argument
431 	while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {  in vmxnet3_tq_cleanup()
434 		tbi = tq->buf_info + tq->tx_ring.next2comp;  in vmxnet3_tq_cleanup()
441 		vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);  in vmxnet3_tq_cleanup()
445 	for (i = 0; i < tq->tx_ring.size; i++) {  in vmxnet3_tq_cleanup()
446 		BUG_ON(tq->buf_info[i].skb != NULL ||  in vmxnet3_tq_cleanup()
447 		       tq->buf_info[i].map_type != VMXNET3_MAP_NONE);  in vmxnet3_tq_cleanup()
450 	tq->tx_ring.gen = VMXNET3_INIT_GEN;  in vmxnet3_tq_cleanup()
451 	tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;  in vmxnet3_tq_cleanup()
453 	tq->comp_ring.gen = VMXNET3_INIT_GEN;  in vmxnet3_tq_cleanup()
454 	tq->comp_ring.next2proc = 0;  in vmxnet3_tq_cleanup()
459 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,  in vmxnet3_tq_destroy()  argument
462 	if (tq->tx_ring.base) {  in vmxnet3_tq_destroy()
463 		dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size *  in vmxnet3_tq_destroy()
465 				  tq->tx_ring.base, tq->tx_ring.basePA);  in vmxnet3_tq_destroy()
466 		tq->tx_ring.base = NULL;  in vmxnet3_tq_destroy()
468 	if (tq->data_ring.base) {  in vmxnet3_tq_destroy()
470 				  tq->data_ring.size * tq->txdata_desc_size,  in vmxnet3_tq_destroy()
471 				  tq->data_ring.base, tq->data_ring.basePA);  in vmxnet3_tq_destroy()
472 		tq->data_ring.base = NULL;  in vmxnet3_tq_destroy()
474 	if (tq->comp_ring.base) {  in vmxnet3_tq_destroy()
475 		dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size *  in vmxnet3_tq_destroy()
477 				  tq->comp_ring.base, tq->comp_ring.basePA);  in vmxnet3_tq_destroy()
478 		tq->comp_ring.base = NULL;  in vmxnet3_tq_destroy()
480 	kfree(tq->buf_info);  in vmxnet3_tq_destroy()
481 	tq->buf_info = NULL;  in vmxnet3_tq_destroy()
497 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,  in vmxnet3_tq_init()  argument
503 	memset(tq->tx_ring.base, 0, tq->tx_ring.size *  in vmxnet3_tq_init()
505 	tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;  in vmxnet3_tq_init()
506 	tq->tx_ring.gen = VMXNET3_INIT_GEN;  in vmxnet3_tq_init()
508 	memset(tq->data_ring.base, 0,  in vmxnet3_tq_init()
509 	       tq->data_ring.size * tq->txdata_desc_size);  in vmxnet3_tq_init()
512 	memset(tq->comp_ring.base, 0, tq->comp_ring.size *  in vmxnet3_tq_init()
514 	tq->comp_ring.next2proc = 0;  in vmxnet3_tq_init()
515 	tq->comp_ring.gen = VMXNET3_INIT_GEN;  in vmxnet3_tq_init()
518 	memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);  in vmxnet3_tq_init()
519 	for (i = 0; i < tq->tx_ring.size; i++)  in vmxnet3_tq_init()
520 		tq->buf_info[i].map_type = VMXNET3_MAP_NONE;  in vmxnet3_tq_init()
527 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,  in vmxnet3_tq_create()  argument
530 	BUG_ON(tq->tx_ring.base || tq->data_ring.base ||  in vmxnet3_tq_create()
531 	       tq->comp_ring.base || tq->buf_info);  in vmxnet3_tq_create()
533 	tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev,  in vmxnet3_tq_create()
534 			tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc),  in vmxnet3_tq_create()
535 			&tq->tx_ring.basePA, GFP_KERNEL);  in vmxnet3_tq_create()
536 	if (!tq->tx_ring.base) {  in vmxnet3_tq_create()
541 	tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev,  in vmxnet3_tq_create()
542 			tq->data_ring.size * tq->txdata_desc_size,  in vmxnet3_tq_create()
543 			&tq->data_ring.basePA, GFP_KERNEL);  in vmxnet3_tq_create()
544 	if (!tq->data_ring.base) {  in vmxnet3_tq_create()
549 	tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev,  in vmxnet3_tq_create()
550 			tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc),  in vmxnet3_tq_create()
551 			&tq->comp_ring.basePA, GFP_KERNEL);  in vmxnet3_tq_create()
552 	if (!tq->comp_ring.base) {  in vmxnet3_tq_create()
557 	tq->buf_info = kcalloc_node(tq->tx_ring.size, sizeof(tq->buf_info[0]),  in vmxnet3_tq_create()
560 	if (!tq->buf_info)  in vmxnet3_tq_create()
566 	vmxnet3_tq_destroy(tq, adapter);  in vmxnet3_tq_create()
700 		struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,  in vmxnet3_map_pkt()  argument
712 	dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;  in vmxnet3_map_pkt()
714 	ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;  in vmxnet3_map_pkt()
719 		ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +  in vmxnet3_map_pkt()
720 					tq->tx_ring.next2fill *  in vmxnet3_map_pkt()
721 					tq->txdata_desc_size);  in vmxnet3_map_pkt()
725 		tbi = tq->buf_info + tq->tx_ring.next2fill;  in vmxnet3_map_pkt()
730 			tq->tx_ring.next2fill,  in vmxnet3_map_pkt()
733 		vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);  in vmxnet3_map_pkt()
736 		dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;  in vmxnet3_map_pkt()
753 		tbi = tq->buf_info + tq->tx_ring.next2fill;  in vmxnet3_map_pkt()
763 		gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;  in vmxnet3_map_pkt()
764 		BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);  in vmxnet3_map_pkt()
772 			tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),  in vmxnet3_map_pkt()
774 		vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);  in vmxnet3_map_pkt()
775 		dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;  in vmxnet3_map_pkt()
788 			tbi = tq->buf_info + tq->tx_ring.next2fill;  in vmxnet3_map_pkt()
805 			gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;  in vmxnet3_map_pkt()
806 			BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);  in vmxnet3_map_pkt()
814 				tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),  in vmxnet3_map_pkt()
816 			vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);  in vmxnet3_map_pkt()
817 			dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;  in vmxnet3_map_pkt()
828 	tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;  in vmxnet3_map_pkt()
863 vmxnet3_parse_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,  in vmxnet3_parse_hdr()  argument
932 					       tq->txdata_desc_size,  in vmxnet3_parse_hdr()
944 	if (unlikely(ctx->copy_size > tq->txdata_desc_size)) {  in vmxnet3_parse_hdr()
945 		tq->stats.oversized_hdr++;  in vmxnet3_parse_hdr()
966 vmxnet3_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,  in vmxnet3_copy_hdr()  argument
972 	tdd = (struct Vmxnet3_TxDataDesc *)((u8 *)tq->data_ring.base +  in vmxnet3_copy_hdr()
973 					    tq->tx_ring.next2fill *  in vmxnet3_copy_hdr()
974 					    tq->txdata_desc_size);  in vmxnet3_copy_hdr()
979 		ctx->copy_size, tq->tx_ring.next2fill);  in vmxnet3_copy_hdr()
1046 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,  in vmxnet3_tq_xmit()  argument
1071 				tq->stats.drop_tso++;  in vmxnet3_tq_xmit()
1074 			tq->stats.copy_skb_header++;  in vmxnet3_tq_xmit()
1081 				tq->stats.drop_too_many_frags++;  in vmxnet3_tq_xmit()
1084 			tq->stats.linearized++;  in vmxnet3_tq_xmit()
1089 				tq->stats.drop_too_many_frags++;  in vmxnet3_tq_xmit()
1105 				tq->stats.drop_too_many_frags++;  in vmxnet3_tq_xmit()
1108 			tq->stats.linearized++;  in vmxnet3_tq_xmit()
1115 	ret = vmxnet3_parse_hdr(skb, tq, &ctx, adapter);  in vmxnet3_tq_xmit()
1122 				tq->stats.drop_oversized_hdr++;  in vmxnet3_tq_xmit()
1130 					tq->stats.drop_oversized_hdr++;  in vmxnet3_tq_xmit()
1136 		tq->stats.drop_hdr_inspect_err++;  in vmxnet3_tq_xmit()
1140 	spin_lock_irqsave(&tq->tx_lock, flags);  in vmxnet3_tq_xmit()
1142 	if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {  in vmxnet3_tq_xmit()
1143 		tq->stats.tx_ring_full++;  in vmxnet3_tq_xmit()
1147 			tq->tx_ring.next2comp, tq->tx_ring.next2fill);  in vmxnet3_tq_xmit()
1149 		vmxnet3_tq_stop(tq, adapter);  in vmxnet3_tq_xmit()
1150 		spin_unlock_irqrestore(&tq->tx_lock, flags);  in vmxnet3_tq_xmit()
1155 	vmxnet3_copy_hdr(skb, tq, &ctx, adapter);  in vmxnet3_tq_xmit()
1158 	if (vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter))  in vmxnet3_tq_xmit()
1172 	tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred);  in vmxnet3_tq_xmit()
1219 	le32_add_cpu(&tq->shared->txNumDeferred, num_pkts);  in vmxnet3_tq_xmit()
1246 		tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),  in vmxnet3_tq_xmit()
1249 	spin_unlock_irqrestore(&tq->tx_lock, flags);  in vmxnet3_tq_xmit()
1251 	if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) {  in vmxnet3_tq_xmit()
1252 		tq->shared->txNumDeferred = 0;  in vmxnet3_tq_xmit()
1254 				       adapter->tx_prod_offset + tq->qid * 8,  in vmxnet3_tq_xmit()
1255 				       tq->tx_ring.next2fill);  in vmxnet3_tq_xmit()
1261 	spin_unlock_irqrestore(&tq->tx_lock, flags);  in vmxnet3_tq_xmit()
1263 	tq->stats.drop_total++;  in vmxnet3_tq_xmit()
2097 		struct vmxnet3_tx_queue *tq =  in vmxnet3_poll_rx_only()  local
2099 		vmxnet3_tq_tx_complete(tq, adapter);  in vmxnet3_poll_rx_only()
2122 	struct vmxnet3_tx_queue *tq = data;  in vmxnet3_msix_tx()  local
2123 	struct vmxnet3_adapter *adapter = tq->adapter;  in vmxnet3_msix_tx()
2126 		vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);  in vmxnet3_msix_tx()
2136 		vmxnet3_tq_tx_complete(tq, adapter);  in vmxnet3_msix_tx()
2138 	vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);  in vmxnet3_msix_tx()
2635 		struct vmxnet3_tx_queue	*tq = &adapter->tx_queue[i];  in vmxnet3_setup_driver_shared()  local
2638 		tqc->txRingBasePA   = cpu_to_le64(tq->tx_ring.basePA);  in vmxnet3_setup_driver_shared()
2639 		tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);  in vmxnet3_setup_driver_shared()
2640 		tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);  in vmxnet3_setup_driver_shared()
2642 		tqc->txRingSize     = cpu_to_le32(tq->tx_ring.size);  in vmxnet3_setup_driver_shared()
2643 		tqc->dataRingSize   = cpu_to_le32(tq->data_ring.size);  in vmxnet3_setup_driver_shared()
2644 		tqc->txDataRingDescSize = cpu_to_le32(tq->txdata_desc_size);  in vmxnet3_setup_driver_shared()
2645 		tqc->compRingSize   = cpu_to_le32(tq->comp_ring.size);  in vmxnet3_setup_driver_shared()
2647 		tqc->intrIdx        = tq->comp_ring.intr_idx;  in vmxnet3_setup_driver_shared()
3123 		struct vmxnet3_tx_queue	*tq = &adapter->tx_queue[i];  in vmxnet3_create_queues()  local
3124 		tq->tx_ring.size   = tx_ring_size;  in vmxnet3_create_queues()
3125 		tq->data_ring.size = tx_ring_size;  in vmxnet3_create_queues()
3126 		tq->comp_ring.size = tx_ring_size;  in vmxnet3_create_queues()
3127 		tq->txdata_desc_size = txdata_desc_size;  in vmxnet3_create_queues()
3128 		tq->shared = &adapter->tqd_start[i].ctrl;  in vmxnet3_create_queues()
3129 		tq->stopped = true;  in vmxnet3_create_queues()
3130 		tq->adapter = adapter;  in vmxnet3_create_queues()
3131 		tq->qid = i;  in vmxnet3_create_queues()
3132 		err = vmxnet3_tq_create(tq, adapter);  in vmxnet3_create_queues()