42 #define vhost_used_event(vq) ((u16 __user *)&vq->avail->ring[vq->num])  argument
43 #define vhost_avail_event(vq) ((u16 __user *)&vq->used->ring[vq->num])  argument
160 			   struct vhost_virtqueue *vq)  in vhost_vq_reset()  argument
162 	vq->num = 1;  in vhost_vq_reset()
163 	vq->desc = NULL;  in vhost_vq_reset()
164 	vq->avail = NULL;  in vhost_vq_reset()
165 	vq->used = NULL;  in vhost_vq_reset()
166 	vq->last_avail_idx = 0;  in vhost_vq_reset()
167 	vq->avail_idx = 0;  in vhost_vq_reset()
168 	vq->last_used_idx = 0;  in vhost_vq_reset()
169 	vq->signalled_used = 0;  in vhost_vq_reset()
170 	vq->signalled_used_valid = false;  in vhost_vq_reset()
171 	vq->used_flags = 0;  in vhost_vq_reset()
172 	vq->log_used = false;  in vhost_vq_reset()
173 	vq->log_addr = -1ull;  in vhost_vq_reset()
174 	vq->vhost_hlen = 0;  in vhost_vq_reset()
175 	vq->sock_hlen = 0;  in vhost_vq_reset()
176 	vq->private_data = NULL;  in vhost_vq_reset()
177 	vq->log_base = NULL;  in vhost_vq_reset()
178 	vq->error_ctx = NULL;  in vhost_vq_reset()
179 	vq->error = NULL;  in vhost_vq_reset()
180 	vq->kick = NULL;  in vhost_vq_reset()
181 	vq->call_ctx = NULL;  in vhost_vq_reset()
182 	vq->call = NULL;  in vhost_vq_reset()
183 	vq->log_ctx = NULL;  in vhost_vq_reset()
184 	vq->upend_idx = 0;  in vhost_vq_reset()
185 	vq->done_idx = 0;  in vhost_vq_reset()
186 	vq->ubufs = NULL;  in vhost_vq_reset()
235 static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq)  in vhost_vq_free_iovecs()  argument
237 	kfree(vq->indirect);  in vhost_vq_free_iovecs()
238 	vq->indirect = NULL;  in vhost_vq_free_iovecs()
239 	kfree(vq->log);  in vhost_vq_free_iovecs()
240 	vq->log = NULL;  in vhost_vq_free_iovecs()
241 	kfree(vq->heads);  in vhost_vq_free_iovecs()
242 	vq->heads = NULL;  in vhost_vq_free_iovecs()
243 	kfree(vq->ubuf_info);  in vhost_vq_free_iovecs()
244 	vq->ubuf_info = NULL;  in vhost_vq_free_iovecs()
247 void vhost_enable_zcopy(int vq)  in vhost_enable_zcopy()  argument
249 	vhost_zcopy_mask |= 0x1 << vq;  in vhost_enable_zcopy()
420 int vhost_zerocopy_signal_used(struct vhost_virtqueue *vq)  in vhost_zerocopy_signal_used()  argument
425 	for (i = vq->done_idx; i != vq->upend_idx; i = (i + 1) % UIO_MAXIOV) {  in vhost_zerocopy_signal_used()
426 		if ((vq->heads[i].len == VHOST_DMA_DONE_LEN)) {  in vhost_zerocopy_signal_used()
427 			vq->heads[i].len = VHOST_DMA_CLEAR_LEN;  in vhost_zerocopy_signal_used()
428 			vhost_add_used_and_signal(vq->dev, vq,  in vhost_zerocopy_signal_used()
429 						  vq->heads[i].id, 0);  in vhost_zerocopy_signal_used()
435 		vq->done_idx = i;  in vhost_zerocopy_signal_used()
577 static int vq_log_access_ok(struct vhost_dev *d, struct vhost_virtqueue *vq,  in vq_log_access_ok()  argument
583 	mp = rcu_dereference_protected(vq->dev->memory,  in vq_log_access_ok()
584 				       lockdep_is_held(&vq->mutex));  in vq_log_access_ok()
586 			    vhost_has_feature(vq->dev, VHOST_F_LOG_ALL)) &&  in vq_log_access_ok()
587 		(!vq->log_used || log_access_ok(log_base, vq->log_addr,  in vq_log_access_ok()
588 					sizeof *vq->used +  in vq_log_access_ok()
589 					vq->num * sizeof *vq->used->ring + s));  in vq_log_access_ok()
594 int vhost_vq_access_ok(struct vhost_virtqueue *vq)  in vhost_vq_access_ok()  argument
596 	return vq_access_ok(vq->dev, vq->num, vq->desc, vq->avail, vq->used) &&  in vhost_vq_access_ok()
597 		vq_log_access_ok(vq->dev, vq, vq->log_base);  in vhost_vq_access_ok()
641 	struct vhost_virtqueue *vq;  in vhost_set_vring()  local
654 	vq = d->vqs + idx;  in vhost_set_vring()
656 	mutex_lock(&vq->mutex);  in vhost_set_vring()
662 		if (vq->private_data) {  in vhost_set_vring()
674 		vq->num = s.num;  in vhost_set_vring()
679 		if (vq->private_data) {  in vhost_set_vring()
691 		vq->last_avail_idx = s.num;  in vhost_set_vring()
693 		vq->avail_idx = vq->last_avail_idx;  in vhost_set_vring()
697 		s.num = vq->last_avail_idx;  in vhost_set_vring()
718 		if ((a.avail_user_addr & (sizeof *vq->avail->ring - 1)) ||  in vhost_set_vring()
719 		    (a.used_user_addr & (sizeof *vq->used->ring - 1)) ||  in vhost_set_vring()
720 		    (a.log_guest_addr & (sizeof *vq->used->ring - 1))) {  in vhost_set_vring()
728 		if (vq->private_data) {  in vhost_set_vring()
729 			if (!vq_access_ok(d, vq->num,  in vhost_set_vring()
739 			    !log_access_ok(vq->log_base, a.log_guest_addr,  in vhost_set_vring()
740 					   sizeof *vq->used +  in vhost_set_vring()
741 					   vq->num * sizeof *vq->used->ring)) {  in vhost_set_vring()
747 		vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG));  in vhost_set_vring()
748 		vq->desc = (void __user *)(unsigned long)a.desc_user_addr;  in vhost_set_vring()
749 		vq->avail = (void __user *)(unsigned long)a.avail_user_addr;  in vhost_set_vring()
750 		vq->log_addr = a.log_guest_addr;  in vhost_set_vring()
751 		vq->used = (void __user *)(unsigned long)a.used_user_addr;  in vhost_set_vring()
763 		if (eventfp != vq->kick) {  in vhost_set_vring()
764 			pollstop = filep = vq->kick;  in vhost_set_vring()
765 			pollstart = vq->kick = eventfp;  in vhost_set_vring()
779 		if (eventfp != vq->call) {  in vhost_set_vring()
780 			filep = vq->call;  in vhost_set_vring()
781 			ctx = vq->call_ctx;  in vhost_set_vring()
782 			vq->call = eventfp;  in vhost_set_vring()
783 			vq->call_ctx = eventfp ?  in vhost_set_vring()
798 		if (eventfp != vq->error) {  in vhost_set_vring()
799 			filep = vq->error;  in vhost_set_vring()
800 			vq->error = eventfp;  in vhost_set_vring()
801 			ctx = vq->error_ctx;  in vhost_set_vring()
802 			vq->error_ctx = eventfp ?  in vhost_set_vring()
811 	if (pollstop && vq->handle_kick)  in vhost_set_vring()
812 		vhost_poll_stop(&vq->poll);  in vhost_set_vring()
819 	if (pollstart && vq->handle_kick)  in vhost_set_vring()
820 		vhost_poll_start(&vq->poll, vq->kick);  in vhost_set_vring()
822 	mutex_unlock(&vq->mutex);  in vhost_set_vring()
824 	if (pollstop && vq->handle_kick)  in vhost_set_vring()
825 		vhost_poll_flush(&vq->poll);  in vhost_set_vring()
864 			struct vhost_virtqueue *vq;  in vhost_dev_ioctl()  local
866 			vq = d->vqs + i;  in vhost_dev_ioctl()
867 			mutex_lock(&vq->mutex);  in vhost_dev_ioctl()
869 			if (vq->private_data && !vq_log_access_ok(d, vq, base))  in vhost_dev_ioctl()
872 				vq->log_base = base;  in vhost_dev_ioctl()
873 			mutex_unlock(&vq->mutex);  in vhost_dev_ioctl()
977 int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,  in vhost_log_write()  argument
986 		r = log_write(vq->log_base, log[i].addr, l);  in vhost_log_write()
991 			if (vq->log_ctx)  in vhost_log_write()
992 				eventfd_signal(vq->log_ctx, 1);  in vhost_log_write()
1001 static int vhost_update_used_flags(struct vhost_virtqueue *vq)  in vhost_update_used_flags()  argument
1004 	if (__put_user(vq->used_flags, &vq->used->flags) < 0)  in vhost_update_used_flags()
1006 	if (unlikely(vq->log_used)) {  in vhost_update_used_flags()
1010 		used = &vq->used->flags;  in vhost_update_used_flags()
1011 		log_write(vq->log_base, vq->log_addr +  in vhost_update_used_flags()
1012 			  (used - (void __user *)vq->used),  in vhost_update_used_flags()
1013 			  sizeof vq->used->flags);  in vhost_update_used_flags()
1014 		if (vq->log_ctx)  in vhost_update_used_flags()
1015 			eventfd_signal(vq->log_ctx, 1);  in vhost_update_used_flags()
1020 static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event)  in vhost_update_avail_event()  argument
1022 	if (__put_user(vq->avail_idx, vhost_avail_event(vq)))  in vhost_update_avail_event()
1024 	if (unlikely(vq->log_used)) {  in vhost_update_avail_event()
1029 		used = vhost_avail_event(vq);  in vhost_update_avail_event()
1030 		log_write(vq->log_base, vq->log_addr +  in vhost_update_avail_event()
1031 			  (used - (void __user *)vq->used),  in vhost_update_avail_event()
1032 			  sizeof *vhost_avail_event(vq));  in vhost_update_avail_event()
1033 		if (vq->log_ctx)  in vhost_update_avail_event()
1034 			eventfd_signal(vq->log_ctx, 1);  in vhost_update_avail_event()
1039 int vhost_init_used(struct vhost_virtqueue *vq)  in vhost_init_used()  argument
1042 	if (!vq->private_data)  in vhost_init_used()
1045 	r = vhost_update_used_flags(vq);  in vhost_init_used()
1048 	vq->signalled_used_valid = false;  in vhost_init_used()
1049 	return get_user(vq->last_used_idx, &vq->used->idx);  in vhost_init_used()
1110 static int get_indirect(struct vhost_dev *dev, struct vhost_virtqueue *vq,  in get_indirect()  argument
1122 		vq_err(vq, "Invalid length in indirect descriptor: "  in get_indirect()
1129 	ret = translate_desc(dev, indirect->addr, indirect->len, vq->indirect,  in get_indirect()
1132 		vq_err(vq, "Translation failure %d in indirect.\n", ret);  in get_indirect()
1144 		vq_err(vq, "Indirect buffer length too big: %d\n",  in get_indirect()
1152 			vq_err(vq, "Loop detected: last one at %u "  in get_indirect()
1158 					      vq->indirect, sizeof desc))) {  in get_indirect()
1159 			vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n",  in get_indirect()
1164 			vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n",  in get_indirect()
1172 			vq_err(vq, "Translation failure %d indirect idx %d\n",  in get_indirect()
1188 				vq_err(vq, "Indirect descriptor "  in get_indirect()
1206 int vhost_get_vq_desc(struct vhost_dev *dev, struct vhost_virtqueue *vq,  in vhost_get_vq_desc()  argument
1217 	last_avail_idx = vq->last_avail_idx;  in vhost_get_vq_desc()
1218 	if (unlikely(__get_user(vq->avail_idx, &vq->avail->idx))) {  in vhost_get_vq_desc()
1219 		vq_err(vq, "Failed to access avail idx at %p\n",  in vhost_get_vq_desc()
1220 		       &vq->avail->idx);  in vhost_get_vq_desc()
1224 	if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {  in vhost_get_vq_desc()
1225 		vq_err(vq, "Guest moved used index from %u to %u",  in vhost_get_vq_desc()
1226 		       last_avail_idx, vq->avail_idx);  in vhost_get_vq_desc()
1231 	if (vq->avail_idx == last_avail_idx)  in vhost_get_vq_desc()
1232 		return vq->num;  in vhost_get_vq_desc()
1240 				&vq->avail->ring[last_avail_idx % vq->num]))) {  in vhost_get_vq_desc()
1241 		vq_err(vq, "Failed to read head: idx %d address %p\n",  in vhost_get_vq_desc()
1243 		       &vq->avail->ring[last_avail_idx % vq->num]);  in vhost_get_vq_desc()
1248 	if (unlikely(head >= vq->num)) {  in vhost_get_vq_desc()
1249 		vq_err(vq, "Guest says index %u > %u is available",  in vhost_get_vq_desc()
1250 		       head, vq->num);  in vhost_get_vq_desc()
1262 		if (unlikely(i >= vq->num)) {  in vhost_get_vq_desc()
1263 			vq_err(vq, "Desc index is %u > %u, head = %u",  in vhost_get_vq_desc()
1264 			       i, vq->num, head);  in vhost_get_vq_desc()
1267 		if (unlikely(++found > vq->num)) {  in vhost_get_vq_desc()
1268 			vq_err(vq, "Loop detected: last one at %u "  in vhost_get_vq_desc()
1270 			       i, vq->num, head);  in vhost_get_vq_desc()
1273 		ret = __copy_from_user(&desc, vq->desc + i, sizeof desc);  in vhost_get_vq_desc()
1275 			vq_err(vq, "Failed to get descriptor: idx %d addr %p\n",  in vhost_get_vq_desc()
1276 			       i, vq->desc + i);  in vhost_get_vq_desc()
1280 			ret = get_indirect(dev, vq, iov, iov_size,  in vhost_get_vq_desc()
1284 				vq_err(vq, "Failure detected "  in vhost_get_vq_desc()
1294 			vq_err(vq, "Translation failure %d descriptor idx %d\n",  in vhost_get_vq_desc()
1311 				vq_err(vq, "Descriptor has out after in: "  in vhost_get_vq_desc()
1320 	vq->last_avail_idx++;  in vhost_get_vq_desc()
1324 	BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY));  in vhost_get_vq_desc()
1329 void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n)  in vhost_discard_vq_desc()  argument
1331 	vq->last_avail_idx -= n;  in vhost_discard_vq_desc()
1336 int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)  in vhost_add_used()  argument
1342 	used = &vq->used->ring[vq->last_used_idx % vq->num];  in vhost_add_used()
1344 		vq_err(vq, "Failed to write used id");  in vhost_add_used()
1348 		vq_err(vq, "Failed to write used len");  in vhost_add_used()
1353 	if (__put_user(vq->last_used_idx + 1, &vq->used->idx)) {  in vhost_add_used()
1354 		vq_err(vq, "Failed to increment used idx");  in vhost_add_used()
1357 	if (unlikely(vq->log_used)) {  in vhost_add_used()
1361 		log_write(vq->log_base,  in vhost_add_used()
1362 			  vq->log_addr +  in vhost_add_used()
1363 			   ((void __user *)used - (void __user *)vq->used),  in vhost_add_used()
1366 		log_write(vq->log_base,  in vhost_add_used()
1367 			  vq->log_addr + offsetof(struct vring_used, idx),  in vhost_add_used()
1368 			  sizeof vq->used->idx);  in vhost_add_used()
1369 		if (vq->log_ctx)  in vhost_add_used()
1370 			eventfd_signal(vq->log_ctx, 1);  in vhost_add_used()
1372 	vq->last_used_idx++;  in vhost_add_used()
1377 	if (unlikely(vq->last_used_idx == vq->signalled_used))  in vhost_add_used()
1378 		vq->signalled_used_valid = false;  in vhost_add_used()
1382 static int __vhost_add_used_n(struct vhost_virtqueue *vq,  in __vhost_add_used_n()  argument
1390 	start = vq->last_used_idx % vq->num;  in __vhost_add_used_n()
1391 	used = vq->used->ring + start;  in __vhost_add_used_n()
1393 		vq_err(vq, "Failed to write used");  in __vhost_add_used_n()
1396 	if (unlikely(vq->log_used)) {  in __vhost_add_used_n()
1400 		log_write(vq->log_base,  in __vhost_add_used_n()
1401 			  vq->log_addr +  in __vhost_add_used_n()
1402 			   ((void __user *)used - (void __user *)vq->used),  in __vhost_add_used_n()
1405 	old = vq->last_used_idx;  in __vhost_add_used_n()
1406 	new = (vq->last_used_idx += count);  in __vhost_add_used_n()
1411 	if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old)))  in __vhost_add_used_n()
1412 		vq->signalled_used_valid = false;  in __vhost_add_used_n()
1418 int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads,  in vhost_add_used_n()  argument
1423 	start = vq->last_used_idx % vq->num;  in vhost_add_used_n()
1424 	n = vq->num - start;  in vhost_add_used_n()
1426 		r = __vhost_add_used_n(vq, heads, n);  in vhost_add_used_n()
1432 	r = __vhost_add_used_n(vq, heads, count);  in vhost_add_used_n()
1436 	if (put_user(vq->last_used_idx, &vq->used->idx)) {  in vhost_add_used_n()
1437 		vq_err(vq, "Failed to increment used idx");  in vhost_add_used_n()
1440 	if (unlikely(vq->log_used)) {  in vhost_add_used_n()
1442 		log_write(vq->log_base,  in vhost_add_used_n()
1443 			  vq->log_addr + offsetof(struct vring_used, idx),  in vhost_add_used_n()
1444 			  sizeof vq->used->idx);  in vhost_add_used_n()
1445 		if (vq->log_ctx)  in vhost_add_used_n()
1446 			eventfd_signal(vq->log_ctx, 1);  in vhost_add_used_n()
1451 static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)  in vhost_notify()  argument
1461 	    unlikely(vq->avail_idx == vq->last_avail_idx))  in vhost_notify()
1466 		if (__get_user(flags, &vq->avail->flags)) {  in vhost_notify()
1467 			vq_err(vq, "Failed to get flags");  in vhost_notify()
1472 	old = vq->signalled_used;  in vhost_notify()
1473 	v = vq->signalled_used_valid;  in vhost_notify()
1474 	new = vq->signalled_used = vq->last_used_idx;  in vhost_notify()
1475 	vq->signalled_used_valid = true;  in vhost_notify()
1480 	if (get_user(event, vhost_used_event(vq))) {  in vhost_notify()
1481 		vq_err(vq, "Failed to get used event idx");  in vhost_notify()
1488 void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)  in vhost_signal()  argument
1491 	if (vq->call_ctx && vhost_notify(dev, vq))  in vhost_signal()
1492 		eventfd_signal(vq->call_ctx, 1);  in vhost_signal()
1497 			       struct vhost_virtqueue *vq,  in vhost_add_used_and_signal()  argument
1500 	vhost_add_used(vq, head, len);  in vhost_add_used_and_signal()
1501 	vhost_signal(dev, vq);  in vhost_add_used_and_signal()
1506 				 struct vhost_virtqueue *vq,  in vhost_add_used_and_signal_n()  argument
1509 	vhost_add_used_n(vq, heads, count);  in vhost_add_used_and_signal_n()
1510 	vhost_signal(dev, vq);  in vhost_add_used_and_signal_n()
1514 bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)  in vhost_enable_notify()  argument
1519 	if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))  in vhost_enable_notify()
1521 	vq->used_flags &= ~VRING_USED_F_NO_NOTIFY;  in vhost_enable_notify()
1523 		r = vhost_update_used_flags(vq);  in vhost_enable_notify()
1525 			vq_err(vq, "Failed to enable notification at %p: %d\n",  in vhost_enable_notify()
1526 			       &vq->used->flags, r);  in vhost_enable_notify()
1530 		r = vhost_update_avail_event(vq, vq->avail_idx);  in vhost_enable_notify()
1532 			vq_err(vq, "Failed to update avail event index at %p: %d\n",  in vhost_enable_notify()
1533 			       vhost_avail_event(vq), r);  in vhost_enable_notify()
1540 	r = __get_user(avail_idx, &vq->avail->idx);  in vhost_enable_notify()
1542 		vq_err(vq, "Failed to check avail idx at %p: %d\n",  in vhost_enable_notify()
1543 		       &vq->avail->idx, r);  in vhost_enable_notify()
1547 	return avail_idx != vq->avail_idx;  in vhost_enable_notify()
1551 void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)  in vhost_disable_notify()  argument
1555 	if (vq->used_flags & VRING_USED_F_NO_NOTIFY)  in vhost_disable_notify()
1557 	vq->used_flags |= VRING_USED_F_NO_NOTIFY;  in vhost_disable_notify()
1559 		r = vhost_update_used_flags(vq);  in vhost_disable_notify()
1561 			vq_err(vq, "Failed to enable notification at %p: %d\n",  in vhost_disable_notify()
1562 			       &vq->used->flags, r);  in vhost_disable_notify()
1573 struct vhost_ubuf_ref *vhost_ubuf_alloc(struct vhost_virtqueue *vq,  in vhost_ubuf_alloc()  argument
1585 	ubufs->vq = vq;  in vhost_ubuf_alloc()
1604 	struct vhost_virtqueue *vq = ubufs->vq;  in vhost_zerocopy_callback()  local
1606 	vhost_poll_queue(&vq->poll);  in vhost_zerocopy_callback()
1608 	vq->heads[ubuf->desc].len = VHOST_DMA_DONE_LEN;  in vhost_zerocopy_callback()