102 static void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg)  in otx2_dma_unmap_skb_frags()  argument
106 	for (seg = 0; seg < sg->num_segs; seg++) {  in otx2_dma_unmap_skb_frags()
107 		otx2_dma_unmap_page(pfvf, sg->dma_addr[seg],  in otx2_dma_unmap_skb_frags()
108 				    sg->size[seg], DMA_TO_DEVICE);  in otx2_dma_unmap_skb_frags()
110 	sg->num_segs = 0;  in otx2_dma_unmap_skb_frags()
118 	struct sg_list *sg;  in otx2_xdp_snd_pkt_handler()  local
122 	sg = &sq->sg[snd_comp->sqe_id];  in otx2_xdp_snd_pkt_handler()
124 	pa = otx2_iova_to_phys(pfvf->iommu_domain, sg->dma_addr[0]);  in otx2_xdp_snd_pkt_handler()
125 	otx2_dma_unmap_page(pfvf, sg->dma_addr[0],  in otx2_xdp_snd_pkt_handler()
126 			    sg->size[0], DMA_TO_DEVICE);  in otx2_xdp_snd_pkt_handler()
141 	struct sg_list *sg;  in otx2_snd_pkt_handler()  local
149 	sg = &sq->sg[snd_comp->sqe_id];  in otx2_snd_pkt_handler()
150 	skb = (struct sk_buff *)sg->skb;  in otx2_snd_pkt_handler()
169 	otx2_dma_unmap_skb_frags(pfvf, sg);  in otx2_snd_pkt_handler()
171 	sg->skb = (u64)NULL;  in otx2_snd_pkt_handler()
258 	struct nix_rx_sg_s *sg = &cqe->sg;  in otx2_free_rcv_seg()  local
263 	start = (void *)sg;  in otx2_free_rcv_seg()
266 		sg = (struct nix_rx_sg_s *)start;  in otx2_free_rcv_seg()
267 		seg_addr = &sg->seg_addr;  in otx2_free_rcv_seg()
268 		for (seg = 0; seg < sg->segs; seg++, seg_addr++)  in otx2_free_rcv_seg()
271 		start += sizeof(*sg);  in otx2_free_rcv_seg()
334 	if (cqe->sg.segs)  in otx2_check_rcv_errors()
345 	struct nix_rx_sg_s *sg = &cqe->sg;  in otx2_rcv_pkt_handler()  local
365 	start = (void *)sg;  in otx2_rcv_pkt_handler()
368 		sg = (struct nix_rx_sg_s *)start;  in otx2_rcv_pkt_handler()
369 		seg_addr = &sg->seg_addr;  in otx2_rcv_pkt_handler()
370 		seg_size = (void *)sg;  in otx2_rcv_pkt_handler()
371 		for (seg = 0; seg < sg->segs; seg++, seg_addr++) {  in otx2_rcv_pkt_handler()
376 		start += sizeof(*sg);  in otx2_rcv_pkt_handler()
404 		    !cqe->sg.seg_addr) {  in otx2_rx_napi_handler()
415 		cqe->sg.seg_addr = 0x00;  in otx2_rx_napi_handler()
588 	struct nix_sqe_sg_s *sg = NULL;  in otx2_sqe_add_sg()  local
593 	sq->sg[sq->head].num_segs = 0;  in otx2_sqe_add_sg()
597 			sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);  in otx2_sqe_add_sg()
598 			sg->ld_type = NIX_SEND_LDTYPE_LDD;  in otx2_sqe_add_sg()
599 			sg->subdc = NIX_SUBDC_SG;  in otx2_sqe_add_sg()
600 			sg->segs = 0;  in otx2_sqe_add_sg()
601 			sg_lens = (void *)sg;  in otx2_sqe_add_sg()
602 			iova = (void *)sg + sizeof(*sg);  in otx2_sqe_add_sg()
607 				*offset += sizeof(*sg) + (3 * sizeof(u64));  in otx2_sqe_add_sg()
609 				*offset += sizeof(*sg) + sizeof(u64);  in otx2_sqe_add_sg()
616 		sg->segs++;  in otx2_sqe_add_sg()
620 		sq->sg[sq->head].dma_addr[seg] = dma_addr;  in otx2_sqe_add_sg()
621 		sq->sg[sq->head].size[seg] = len;  in otx2_sqe_add_sg()
622 		sq->sg[sq->head].num_segs++;  in otx2_sqe_add_sg()
625 	sq->sg[sq->head].skb = (u64)skb;  in otx2_sqe_add_sg()
775 	struct sg_list *sg = &sq->sg[sqe];  in otx2_dma_map_tso_skb()  local
779 	sg->num_segs = 0;  in otx2_dma_map_tso_skb()
793 		sg->dma_addr[sg->num_segs] = dma_addr;  in otx2_dma_map_tso_skb()
794 		sg->size[sg->num_segs] = len;  in otx2_dma_map_tso_skb()
795 		sg->num_segs++;  in otx2_dma_map_tso_skb()
799 	otx2_dma_unmap_skb_frags(pfvf, sg);  in otx2_dma_map_tso_skb()
807 	struct sg_list *sg = &sq->sg[sqe];  in otx2_tso_frag_dma_addr()  local
812 		return sg->dma_addr[0] + (seg_addr - (u64)skb->data);  in otx2_tso_frag_dma_addr()
818 	return sg->dma_addr[seg] + offset;  in otx2_tso_frag_dma_addr()
824 	struct nix_sqe_sg_s *sg = NULL;  in otx2_sqe_tso_add_sg()  local
832 			sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);  in otx2_sqe_tso_add_sg()
833 			sg->ld_type = NIX_SEND_LDTYPE_LDD;  in otx2_sqe_tso_add_sg()
834 			sg->subdc = NIX_SUBDC_SG;  in otx2_sqe_tso_add_sg()
835 			sg->segs = 0;  in otx2_sqe_tso_add_sg()
836 			sg_lens = (void *)sg;  in otx2_sqe_tso_add_sg()
837 			iova = (void *)sg + sizeof(*sg);  in otx2_sqe_tso_add_sg()
842 				*offset += sizeof(*sg) + (3 * sizeof(u64));  in otx2_sqe_tso_add_sg()
844 				*offset += sizeof(*sg) + sizeof(u64);  in otx2_sqe_tso_add_sg()
848 		sg->segs++;  in otx2_sqe_tso_add_sg()
925 			sq->sg[first_sqe].skb = (u64)skb;  in otx2_sq_append_tso()
1128 		otx2_dma_unmap_skb_frags(pfvf, &sq->sg[sq->head]);  in otx2_sq_append_skb()
1164 		if (cqe->sg.segs > 1) {  in otx2_cleanup_rx_cqes()
1168 		iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM;  in otx2_cleanup_rx_cqes()
1185 	struct sg_list *sg;  in otx2_cleanup_tx_cqes()  local
1199 		sg = &sq->sg[cqe->comp.sqe_id];  in otx2_cleanup_tx_cqes()
1200 		skb = (struct sk_buff *)sg->skb;  in otx2_cleanup_tx_cqes()
1202 			otx2_dma_unmap_skb_frags(pfvf, sg);  in otx2_cleanup_tx_cqes()
1204 			sg->skb = (u64)NULL;  in otx2_cleanup_tx_cqes()
1237 	struct nix_sqe_sg_s *sg = NULL;  in otx2_xdp_sqe_add_sg()  local
1240 	sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);  in otx2_xdp_sqe_add_sg()
1241 	sg->ld_type = NIX_SEND_LDTYPE_LDD;  in otx2_xdp_sqe_add_sg()
1242 	sg->subdc = NIX_SUBDC_SG;  in otx2_xdp_sqe_add_sg()
1243 	sg->segs = 1;  in otx2_xdp_sqe_add_sg()
1244 	sg->seg1_size = len;  in otx2_xdp_sqe_add_sg()
1245 	iova = (void *)sg + sizeof(*sg);  in otx2_xdp_sqe_add_sg()
1247 	*offset += sizeof(*sg) + sizeof(u64);  in otx2_xdp_sqe_add_sg()
1249 	sq->sg[sq->head].dma_addr[0] = dma_addr;  in otx2_xdp_sqe_add_sg()
1250 	sq->sg[sq->head].size[0] = len;  in otx2_xdp_sqe_add_sg()
1251 	sq->sg[sq->head].num_segs = 1;  in otx2_xdp_sqe_add_sg()
1300 	iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM;  in otx2_xdp_rcv_pkt_handler()
1309 			 cqe->sg.seg_size, false);  in otx2_xdp_rcv_pkt_handler()
1320 					      cqe->sg.seg_size, qidx);  in otx2_xdp_rcv_pkt_handler()