176 static int vhost_net_buf_produce(struct vhost_net_virtqueue *nvq)  in vhost_net_buf_produce()  argument
178 	struct vhost_net_buf *rxq = &nvq->rxq;  in vhost_net_buf_produce()
181 	rxq->tail = ptr_ring_consume_batched(nvq->rx_ring, rxq->queue,  in vhost_net_buf_produce()
186 static void vhost_net_buf_unproduce(struct vhost_net_virtqueue *nvq)  in vhost_net_buf_unproduce()  argument
188 	struct vhost_net_buf *rxq = &nvq->rxq;  in vhost_net_buf_unproduce()
190 	if (nvq->rx_ring && !vhost_net_buf_is_empty(rxq)) {  in vhost_net_buf_unproduce()
191 		ptr_ring_unconsume(nvq->rx_ring, rxq->queue + rxq->head,  in vhost_net_buf_unproduce()
209 static int vhost_net_buf_peek(struct vhost_net_virtqueue *nvq)  in vhost_net_buf_peek()  argument
211 	struct vhost_net_buf *rxq = &nvq->rxq;  in vhost_net_buf_peek()
216 	if (!vhost_net_buf_produce(nvq))  in vhost_net_buf_peek()
361 	struct vhost_net_virtqueue *nvq =  in vhost_zerocopy_signal_used()  local
366 	for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) {  in vhost_zerocopy_signal_used()
376 		add = min(UIO_MAXIOV - nvq->done_idx, j);  in vhost_zerocopy_signal_used()
378 					    &vq->heads[nvq->done_idx], add);  in vhost_zerocopy_signal_used()
379 		nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV;  in vhost_zerocopy_signal_used()
426 	struct vhost_net_virtqueue *nvq =  in vhost_net_disable_vq()  local
428 	struct vhost_poll *poll = n->poll + (nvq - n->vqs);  in vhost_net_disable_vq()
437 	struct vhost_net_virtqueue *nvq =  in vhost_net_enable_vq()  local
439 	struct vhost_poll *poll = n->poll + (nvq - n->vqs);  in vhost_net_enable_vq()
449 static void vhost_net_signal_used(struct vhost_net_virtqueue *nvq)  in vhost_net_signal_used()  argument
451 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_net_signal_used()
454 	if (!nvq->done_idx)  in vhost_net_signal_used()
457 	vhost_add_used_and_signal_n(dev, vq, vq->heads, nvq->done_idx);  in vhost_net_signal_used()
458 	nvq->done_idx = 0;  in vhost_net_signal_used()
462 			   struct vhost_net_virtqueue *nvq,  in vhost_tx_batch()  argument
468 		.num = nvq->batched_xdp,  in vhost_tx_batch()
469 		.ptr = nvq->xdp,  in vhost_tx_batch()
473 	if (nvq->batched_xdp == 0)  in vhost_tx_batch()
480 		vq_err(&nvq->vq, "Fail to batch sending packets\n");  in vhost_tx_batch()
486 		for (i = 0; i < nvq->batched_xdp; ++i)  in vhost_tx_batch()
487 			put_page(virt_to_head_page(nvq->xdp[i].data));  in vhost_tx_batch()
488 		nvq->batched_xdp = 0;  in vhost_tx_batch()
489 		nvq->done_idx = 0;  in vhost_tx_batch()
494 	vhost_net_signal_used(nvq);  in vhost_tx_batch()
495 	nvq->batched_xdp = 0;  in vhost_tx_batch()
601 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in vhost_exceeds_maxpend()  local
602 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_exceeds_maxpend()
604 	return (nvq->upend_idx + UIO_MAXIOV - nvq->done_idx) % UIO_MAXIOV >  in vhost_exceeds_maxpend()
621 		       struct vhost_net_virtqueue *nvq,  in get_tx_bufs()  argument
626 	struct vhost_virtqueue *vq = &nvq->vq;  in get_tx_bufs()
629 	ret = vhost_net_tx_get_vq_desc(net, nvq, out, in, msg, busyloop_intr);  in get_tx_bufs()
641 	*len = init_iov_iter(vq, &msg->msg_iter, nvq->vhost_hlen, *out);  in get_tx_bufs()
644 			*len, nvq->vhost_hlen);  in get_tx_bufs()
694 static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,  in vhost_net_build_xdp()  argument
697 	struct vhost_virtqueue *vq = &nvq->vq;  in vhost_net_build_xdp()
703 	struct xdp_buff *xdp = &nvq->xdp[nvq->batched_xdp];  in vhost_net_build_xdp()
708 	int pad = SKB_DATA_ALIGN(VHOST_NET_RX_PAD + headroom + nvq->sock_hlen);  in vhost_net_build_xdp()
709 	int sock_hlen = nvq->sock_hlen;  in vhost_net_build_xdp()
713 	if (unlikely(len < nvq->sock_hlen))  in vhost_net_build_xdp()
763 	++nvq->batched_xdp;  in vhost_net_build_xdp()
770 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in handle_tx_copy()  local
771 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_tx_copy()
789 		if (nvq->done_idx == VHOST_NET_BATCH)  in handle_tx_copy()
790 			vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
792 		head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,  in handle_tx_copy()
815 			err = vhost_net_build_xdp(nvq, &msg.msg_iter);  in handle_tx_copy()
819 				vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
829 			vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
850 		vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head);  in handle_tx_copy()
851 		vq->heads[nvq->done_idx].len = 0;  in handle_tx_copy()
852 		++nvq->done_idx;  in handle_tx_copy()
855 	vhost_tx_batch(net, nvq, sock, &msg);  in handle_tx_copy()
860 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in handle_tx_zerocopy()  local
861 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_tx_zerocopy()
886 		head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,  in handle_tx_zerocopy()
908 			ubuf = nvq->ubuf_info + nvq->upend_idx;  in handle_tx_zerocopy()
909 			vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);  in handle_tx_zerocopy()
910 			vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;  in handle_tx_zerocopy()
911 			ubuf->ctx = nvq->ubufs;  in handle_tx_zerocopy()
912 			ubuf->desc = nvq->upend_idx;  in handle_tx_zerocopy()
920 			ubufs = nvq->ubufs;  in handle_tx_zerocopy()
922 			nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV;  in handle_tx_zerocopy()
940 				nvq->upend_idx = ((unsigned)nvq->upend_idx - 1)  in handle_tx_zerocopy()
964 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];  in handle_tx()  local
965 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_tx()
1111 	struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX];  in handle_rx()  local
1112 	struct vhost_virtqueue *vq = &nvq->vq;  in handle_rx()
1148 	vhost_hlen = nvq->vhost_hlen;  in handle_rx()
1149 	sock_hlen = nvq->sock_hlen;  in handle_rx()
1162 		headcount = get_rx_bufs(vq, vq->heads + nvq->done_idx,  in handle_rx()
1183 		if (nvq->rx_ring)  in handle_rx()
1184 			msg.msg_control = vhost_net_buf_consume(&nvq->rxq);  in handle_rx()
1237 		nvq->done_idx += headcount;  in handle_rx()
1238 		if (nvq->done_idx > VHOST_NET_BATCH)  in handle_rx()
1239 			vhost_net_signal_used(nvq);  in handle_rx()
1251 	vhost_net_signal_used(nvq);  in handle_rx()
1358 	struct vhost_net_virtqueue *nvq =  in vhost_net_stop_vq()  local
1365 	vhost_net_buf_unproduce(nvq);  in vhost_net_stop_vq()
1366 	nvq->rx_ring = NULL;  in vhost_net_stop_vq()
1497 	struct vhost_net_virtqueue *nvq;  in vhost_net_set_backend()  local
1511 	nvq = &n->vqs[index];  in vhost_net_set_backend()
1537 		vhost_net_buf_unproduce(nvq);  in vhost_net_set_backend()
1546 				nvq->rx_ring = get_tap_ptr_ring(sock->file);  in vhost_net_set_backend()
1548 				nvq->rx_ring = NULL;  in vhost_net_set_backend()
1551 		oldubufs = nvq->ubufs;  in vhost_net_set_backend()
1552 		nvq->ubufs = ubufs;  in vhost_net_set_backend()