Lines Matching refs:vq
133 struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_TX]; in handle_tx() local
141 .msg_iov = vq->iov, in handle_tx()
152 sock = rcu_dereference_check(vq->private_data, 1); in handle_tx()
158 mutex_lock(&vq->mutex); in handle_tx()
160 mutex_unlock(&vq->mutex); in handle_tx()
164 mutex_lock(&vq->mutex); in handle_tx()
165 vhost_disable_notify(&net->dev, vq); in handle_tx()
169 hdr_size = vq->vhost_hlen; in handle_tx()
175 vhost_zerocopy_signal_used(vq); in handle_tx()
177 head = vhost_get_vq_desc(&net->dev, vq, vq->iov, in handle_tx()
178 ARRAY_SIZE(vq->iov), in handle_tx()
185 if (head == vq->num) { in handle_tx()
197 num_pends = likely(vq->upend_idx >= vq->done_idx) ? in handle_tx()
198 (vq->upend_idx - vq->done_idx) : in handle_tx()
199 (vq->upend_idx + UIO_MAXIOV - vq->done_idx); in handle_tx()
205 if (unlikely(vhost_enable_notify(&net->dev, vq))) { in handle_tx()
206 vhost_disable_notify(&net->dev, vq); in handle_tx()
212 vq_err(vq, "Unexpected descriptor format for TX: " in handle_tx()
217 s = move_iovec_hdr(vq->iov, vq->hdr, hdr_size, out); in handle_tx()
219 len = iov_length(vq->iov, out); in handle_tx()
222 vq_err(vq, "Unexpected header len for TX: " in handle_tx()
224 iov_length(vq->hdr, s), hdr_size); in handle_tx()
229 vq->heads[vq->upend_idx].id = head; in handle_tx()
232 vq->heads[vq->upend_idx].len = in handle_tx()
239 ubuf = vq->ubuf_info + vq->upend_idx; in handle_tx()
241 vq->heads[vq->upend_idx].len = len; in handle_tx()
243 ubuf->ctx = vq->ubufs; in handle_tx()
244 ubuf->desc = vq->upend_idx; in handle_tx()
247 ubufs = vq->ubufs; in handle_tx()
250 vq->upend_idx = (vq->upend_idx + 1) % UIO_MAXIOV; in handle_tx()
258 vq->upend_idx = ((unsigned)vq->upend_idx - 1) % in handle_tx()
261 vhost_discard_vq_desc(vq, 1); in handle_tx()
269 vhost_add_used_and_signal(&net->dev, vq, head, 0); in handle_tx()
272 vhost_poll_queue(&vq->poll); in handle_tx()
277 mutex_unlock(&vq->mutex); in handle_tx()
308 static int get_rx_bufs(struct vhost_virtqueue *vq, in get_rx_bufs() argument
327 r = vhost_get_vq_desc(vq->dev, vq, vq->iov + seg, in get_rx_bufs()
328 ARRAY_SIZE(vq->iov) - seg, &out, in get_rx_bufs()
334 if (d == vq->num) { in get_rx_bufs()
339 vq_err(vq, "unexpected descriptor format for RX: " in get_rx_bufs()
349 heads[headcount].len = iov_length(vq->iov + seg, in); in get_rx_bufs()
366 vhost_discard_vq_desc(vq, headcount); in get_rx_bufs()
374 struct vhost_virtqueue *vq = &net->dev.vqs[VHOST_NET_VQ_RX]; in handle_rx() local
382 .msg_iov = vq->iov, in handle_rx()
395 struct socket *sock = rcu_dereference_check(vq->private_data, 1); in handle_rx()
400 mutex_lock(&vq->mutex); in handle_rx()
401 vhost_disable_notify(&net->dev, vq); in handle_rx()
402 vhost_hlen = vq->vhost_hlen; in handle_rx()
403 sock_hlen = vq->sock_hlen; in handle_rx()
406 vq->log : NULL; in handle_rx()
412 headcount = get_rx_bufs(vq, vq->heads, vhost_len, in handle_rx()
428 if (unlikely(vhost_enable_notify(&net->dev, vq))) { in handle_rx()
431 vhost_disable_notify(&net->dev, vq); in handle_rx()
441 move_iovec_hdr(vq->iov, vq->hdr, vhost_hlen, in); in handle_rx()
445 copy_iovec_hdr(vq->iov, vq->hdr, sock_hlen, in); in handle_rx()
455 vhost_discard_vq_desc(vq, headcount); in handle_rx()
459 memcpy_toiovecend(vq->hdr, (unsigned char *)&hdr, 0, in handle_rx()
461 vq_err(vq, "Unable to write vnet_hdr at addr %p\n", in handle_rx()
462 vq->iov->iov_base); in handle_rx()
467 memcpy_toiovecend(vq->hdr, (unsigned char *)&headcount, in handle_rx()
470 vq_err(vq, "Failed num_buffers write"); in handle_rx()
471 vhost_discard_vq_desc(vq, headcount); in handle_rx()
474 vhost_add_used_and_signal_n(&net->dev, vq, vq->heads, in handle_rx()
477 vhost_log_write(vq, vq_log, log, vhost_len); in handle_rx()
480 vhost_poll_queue(&vq->poll); in handle_rx()
485 mutex_unlock(&vq->mutex); in handle_rx()
490 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, in handle_tx_kick() local
492 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); in handle_tx_kick()
499 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, in handle_rx_kick() local
501 struct vhost_net *net = container_of(vq->dev, struct vhost_net, dev); in handle_rx_kick()
548 struct vhost_virtqueue *vq) in vhost_net_disable_vq() argument
550 if (!vq->private_data) in vhost_net_disable_vq()
552 if (vq == n->vqs + VHOST_NET_VQ_TX) { in vhost_net_disable_vq()
560 struct vhost_virtqueue *vq) in vhost_net_enable_vq() argument
564 sock = rcu_dereference_protected(vq->private_data, in vhost_net_enable_vq()
565 lockdep_is_held(&vq->mutex)); in vhost_net_enable_vq()
568 if (vq == n->vqs + VHOST_NET_VQ_TX) { in vhost_net_enable_vq()
576 struct vhost_virtqueue *vq) in vhost_net_stop_vq() argument
580 mutex_lock(&vq->mutex); in vhost_net_stop_vq()
581 sock = rcu_dereference_protected(vq->private_data, in vhost_net_stop_vq()
582 lockdep_is_held(&vq->mutex)); in vhost_net_stop_vq()
583 vhost_net_disable_vq(n, vq); in vhost_net_stop_vq()
584 rcu_assign_pointer(vq->private_data, NULL); in vhost_net_stop_vq()
585 mutex_unlock(&vq->mutex); in vhost_net_stop_vq()
696 struct vhost_virtqueue *vq; in vhost_net_set_backend() local
709 vq = n->vqs + index; in vhost_net_set_backend()
710 mutex_lock(&vq->mutex); in vhost_net_set_backend()
713 if (!vhost_vq_access_ok(vq)) { in vhost_net_set_backend()
724 oldsock = rcu_dereference_protected(vq->private_data, in vhost_net_set_backend()
725 lockdep_is_held(&vq->mutex)); in vhost_net_set_backend()
727 ubufs = vhost_ubuf_alloc(vq, sock && vhost_sock_zcopy(sock)); in vhost_net_set_backend()
732 oldubufs = vq->ubufs; in vhost_net_set_backend()
733 vq->ubufs = ubufs; in vhost_net_set_backend()
734 vhost_net_disable_vq(n, vq); in vhost_net_set_backend()
735 rcu_assign_pointer(vq->private_data, sock); in vhost_net_set_backend()
736 vhost_net_enable_vq(n, vq); in vhost_net_set_backend()
738 r = vhost_init_used(vq); in vhost_net_set_backend()
743 mutex_unlock(&vq->mutex); in vhost_net_set_backend()
747 mutex_lock(&vq->mutex); in vhost_net_set_backend()
748 vhost_zerocopy_signal_used(vq); in vhost_net_set_backend()
749 mutex_unlock(&vq->mutex); in vhost_net_set_backend()
763 mutex_unlock(&vq->mutex); in vhost_net_set_backend()