20 #define BAD_RING(_vq, fmt, args...)				\  argument
22 		dev_err(&(_vq)->vq.vdev->dev,			\
23 			"%s:"fmt, (_vq)->vq.name, ##args);	\
27 #define START_USE(_vq)						\  argument
29 		if ((_vq)->in_use)				\
31 			      (_vq)->vq.name, (_vq)->in_use);	\
32 		(_vq)->in_use = __LINE__;			\
34 #define END_USE(_vq) \  argument
35 	do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
36 #define LAST_ADD_TIME_UPDATE(_vq)				\  argument
41 		if ((_vq)->last_add_time_valid)			\
43 				(_vq)->last_add_time)) > 100);	\
44 		(_vq)->last_add_time = now;			\
45 		(_vq)->last_add_time_valid = true;		\
47 #define LAST_ADD_TIME_CHECK(_vq)				\  argument
49 		if ((_vq)->last_add_time_valid) {		\
51 				      (_vq)->last_add_time)) > 100); \
54 #define LAST_ADD_TIME_INVALID(_vq)				\  argument
55 	((_vq)->last_add_time_valid = false)
57 #define BAD_RING(_vq, fmt, args...)				\  argument
59 		dev_err(&_vq->vq.vdev->dev,			\
60 			"%s:"fmt, (_vq)->vq.name, ##args);	\
61 		(_vq)->broken = true;				\
236 static void vring_free(struct virtqueue *_vq);
242 #define to_vvq(_vq) container_of_const(_vq, struct vring_virtqueue, vq)  argument
492 static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,  in alloc_indirect_split()  argument
511 		desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);  in alloc_indirect_split()
544 static inline int virtqueue_add_split(struct virtqueue *_vq,  in virtqueue_add_split()  argument
553 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_add_split()
577 		desc = alloc_indirect_split(_vq, total_sg, gfp);  in virtqueue_add_split()
621 			i = virtqueue_add_desc_split(_vq, desc, i, addr, sg->length,  in virtqueue_add_split()
637 			i = virtqueue_add_desc_split(_vq, desc, i, addr,  in virtqueue_add_split()
645 	desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);  in virtqueue_add_split()
662 		virtqueue_add_desc_split(_vq, vq->split.vring.desc,  in virtqueue_add_split()
688 	vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);  in virtqueue_add_split()
694 	vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_add_split()
704 		virtqueue_kick(_vq);  in virtqueue_add_split()
721 			i = virtio16_to_cpu(_vq->vdev, desc[i].next);  in virtqueue_add_split()
734 static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)  in virtqueue_kick_prepare_split()  argument
736 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_kick_prepare_split()
753 		needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,  in virtqueue_kick_prepare_split()
758 					cpu_to_virtio16(_vq->vdev,  in virtqueue_kick_prepare_split()
823 static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,  in virtqueue_get_buf_ctx_split()  argument
827 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_buf_ctx_split()
849 	i = virtio32_to_cpu(_vq->vdev,  in virtqueue_get_buf_ctx_split()
851 	*len = virtio32_to_cpu(_vq->vdev,  in virtqueue_get_buf_ctx_split()
873 				cpu_to_virtio16(_vq->vdev, vq->last_used_idx));  in virtqueue_get_buf_ctx_split()
881 static void virtqueue_disable_cb_split(struct virtqueue *_vq)  in virtqueue_disable_cb_split()  argument
883 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_disable_cb_split()
900 				cpu_to_virtio16(_vq->vdev,  in virtqueue_disable_cb_split()
905 static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)  in virtqueue_enable_cb_prepare_split()  argument
907 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_prepare_split()
921 				cpu_to_virtio16(_vq->vdev,  in virtqueue_enable_cb_prepare_split()
924 	vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,  in virtqueue_enable_cb_prepare_split()
930 static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned int last_used_idx)  in virtqueue_poll_split()  argument
932 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_poll_split()
934 	return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,  in virtqueue_poll_split()
938 static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)  in virtqueue_enable_cb_delayed_split()  argument
940 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_delayed_split()
954 				cpu_to_virtio16(_vq->vdev,  in virtqueue_enable_cb_delayed_split()
962 			cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));  in virtqueue_enable_cb_delayed_split()
964 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)  in virtqueue_enable_cb_delayed_split()
974 static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)  in virtqueue_detach_unused_buf_split()  argument
976 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_detach_unused_buf_split()
989 		vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,  in virtqueue_detach_unused_buf_split()
1175 static int virtqueue_resize_split(struct virtqueue *_vq, u32 num)  in virtqueue_resize_split()  argument
1178 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_resize_split()
1179 	struct virtio_device *vdev = _vq->vdev;  in virtqueue_resize_split()
1401 static inline int virtqueue_add_packed(struct virtqueue *_vq,  in virtqueue_add_packed()  argument
1410 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_add_packed()
1553 static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)  in virtqueue_kick_prepare_packed()  argument
1555 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_kick_prepare_packed()
1673 static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,  in virtqueue_get_buf_ctx_packed()  argument
1677 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_buf_ctx_packed()
1742 static void virtqueue_disable_cb_packed(struct virtqueue *_vq)  in virtqueue_disable_cb_packed()  argument
1744 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_disable_cb_packed()
1761 static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)  in virtqueue_enable_cb_prepare_packed()  argument
1763 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_prepare_packed()
1794 static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)  in virtqueue_poll_packed()  argument
1796 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_poll_packed()
1806 static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)  in virtqueue_enable_cb_delayed_packed()  argument
1808 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_delayed_packed()
1867 static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)  in virtqueue_detach_unused_buf_packed()  argument
1869 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_detach_unused_buf_packed()
2125 static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num)  in virtqueue_resize_packed()  argument
2128 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_resize_packed()
2129 	struct virtio_device *vdev = _vq->vdev;  in virtqueue_resize_packed()
2155 static int virtqueue_disable_and_recycle(struct virtqueue *_vq,  in virtqueue_disable_and_recycle()  argument
2158 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_disable_and_recycle()
2172 	err = vdev->config->disable_vq_and_reset(_vq);  in virtqueue_disable_and_recycle()
2176 	while ((buf = virtqueue_detach_unused_buf(_vq)) != NULL)  in virtqueue_disable_and_recycle()
2177 		recycle(_vq, buf);  in virtqueue_disable_and_recycle()
2182 static int virtqueue_enable_after_reset(struct virtqueue *_vq)  in virtqueue_enable_after_reset()  argument
2184 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_after_reset()
2187 	if (vdev->config->enable_vq_after_reset(_vq))  in virtqueue_enable_after_reset()
2197 static inline int virtqueue_add(struct virtqueue *_vq,  in virtqueue_add()  argument
2206 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_add()
2208 	return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,  in virtqueue_add()
2210 				 virtqueue_add_split(_vq, sgs, total_sg,  in virtqueue_add()
2228 int virtqueue_add_sgs(struct virtqueue *_vq,  in virtqueue_add_sgs()  argument
2244 	return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,  in virtqueue_add_sgs()
2323 struct device *virtqueue_dma_dev(struct virtqueue *_vq)  in virtqueue_dma_dev()  argument
2325 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_dma_dev()
2345 bool virtqueue_kick_prepare(struct virtqueue *_vq)  in virtqueue_kick_prepare()  argument
2347 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_kick_prepare()
2349 	return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :  in virtqueue_kick_prepare()
2350 				 virtqueue_kick_prepare_split(_vq);  in virtqueue_kick_prepare()
2362 bool virtqueue_notify(struct virtqueue *_vq)  in virtqueue_notify()  argument
2364 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_notify()
2370 	if (!vq->notify(_vq)) {  in virtqueue_notify()
2415 void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,  in virtqueue_get_buf_ctx()  argument
2418 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_buf_ctx()
2420 	return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :  in virtqueue_get_buf_ctx()
2421 				 virtqueue_get_buf_ctx_split(_vq, len, ctx);  in virtqueue_get_buf_ctx()
2425 void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)  in virtqueue_get_buf()  argument
2427 	return virtqueue_get_buf_ctx(_vq, len, NULL);  in virtqueue_get_buf()
2439 void virtqueue_disable_cb(struct virtqueue *_vq)  in virtqueue_disable_cb()  argument
2441 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_disable_cb()
2444 		virtqueue_disable_cb_packed(_vq);  in virtqueue_disable_cb()
2446 		virtqueue_disable_cb_split(_vq);  in virtqueue_disable_cb()
2462 unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)  in virtqueue_enable_cb_prepare()  argument
2464 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_prepare()
2469 	return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :  in virtqueue_enable_cb_prepare()
2470 				 virtqueue_enable_cb_prepare_split(_vq);  in virtqueue_enable_cb_prepare()
2483 bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)  in virtqueue_poll()  argument
2485 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_poll()
2491 	return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :  in virtqueue_poll()
2492 				 virtqueue_poll_split(_vq, last_used_idx);  in virtqueue_poll()
2507 bool virtqueue_enable_cb(struct virtqueue *_vq)  in virtqueue_enable_cb()  argument
2509 	unsigned int last_used_idx = virtqueue_enable_cb_prepare(_vq);  in virtqueue_enable_cb()
2511 	return !virtqueue_poll(_vq, last_used_idx);  in virtqueue_enable_cb()
2528 bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)  in virtqueue_enable_cb_delayed()  argument
2530 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_enable_cb_delayed()
2535 	return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :  in virtqueue_enable_cb_delayed()
2536 				 virtqueue_enable_cb_delayed_split(_vq);  in virtqueue_enable_cb_delayed()
2548 void *virtqueue_detach_unused_buf(struct virtqueue *_vq)  in virtqueue_detach_unused_buf()  argument
2550 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_detach_unused_buf()
2552 	return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :  in virtqueue_detach_unused_buf()
2553 				 virtqueue_detach_unused_buf_split(_vq);  in virtqueue_detach_unused_buf()
2570 irqreturn_t vring_interrupt(int irq, void *_vq)  in vring_interrupt()  argument
2572 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_interrupt()
2738 int virtqueue_resize(struct virtqueue *_vq, u32 num,  in virtqueue_resize()  argument
2741 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_resize()
2753 	err = virtqueue_disable_and_recycle(_vq, recycle);  in virtqueue_resize()
2758 		err = virtqueue_resize_packed(_vq, num);  in virtqueue_resize()
2760 		err = virtqueue_resize_split(_vq, num);  in virtqueue_resize()
2762 	return virtqueue_enable_after_reset(_vq);  in virtqueue_resize()
2787 int virtqueue_set_dma_premapped(struct virtqueue *_vq)  in virtqueue_set_dma_premapped()  argument
2789 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_set_dma_premapped()
2829 int virtqueue_reset(struct virtqueue *_vq,  in virtqueue_reset()  argument
2832 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_reset()
2835 	err = virtqueue_disable_and_recycle(_vq, recycle);  in virtqueue_reset()
2844 	return virtqueue_enable_after_reset(_vq);  in virtqueue_reset()
2872 static void vring_free(struct virtqueue *_vq)  in vring_free()  argument
2874 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_free()
2912 void vring_del_virtqueue(struct virtqueue *_vq)  in vring_del_virtqueue()  argument
2914 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_del_virtqueue()
2917 	list_del(&_vq->list);  in vring_del_virtqueue()
2920 	vring_free(_vq);  in vring_del_virtqueue()
2926 u32 vring_notification_data(struct virtqueue *_vq)  in vring_notification_data()  argument
2928 	struct vring_virtqueue *vq = to_vvq(_vq);  in vring_notification_data()
2939 	return next << 16 | _vq->index;  in vring_notification_data()
2979 unsigned int virtqueue_get_vring_size(const struct virtqueue *_vq)  in virtqueue_get_vring_size()  argument
2982 	const struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_vring_size()
2991 void __virtqueue_break(struct virtqueue *_vq)  in __virtqueue_break()  argument
2993 	struct vring_virtqueue *vq = to_vvq(_vq);  in __virtqueue_break()
3003 void __virtqueue_unbreak(struct virtqueue *_vq)  in __virtqueue_unbreak()  argument
3005 	struct vring_virtqueue *vq = to_vvq(_vq);  in __virtqueue_unbreak()
3012 bool virtqueue_is_broken(const struct virtqueue *_vq)  in virtqueue_is_broken()  argument
3014 	const struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_is_broken()
3026 	struct virtqueue *_vq;  in virtio_break_device()  local
3029 	list_for_each_entry(_vq, &dev->vqs, list) {  in virtio_break_device()
3030 		struct vring_virtqueue *vq = to_vvq(_vq);  in virtio_break_device()
3048 	struct virtqueue *_vq;  in __virtio_unbreak_device()  local
3051 	list_for_each_entry(_vq, &dev->vqs, list) {  in __virtio_unbreak_device()
3052 		struct vring_virtqueue *vq = to_vvq(_vq);  in __virtio_unbreak_device()
3061 dma_addr_t virtqueue_get_desc_addr(const struct virtqueue *_vq)  in virtqueue_get_desc_addr()  argument
3063 	const struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_desc_addr()
3074 dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *_vq)  in virtqueue_get_avail_addr()  argument
3076 	const struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_avail_addr()
3088 dma_addr_t virtqueue_get_used_addr(const struct virtqueue *_vq)  in virtqueue_get_used_addr()  argument
3090 	const struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_get_used_addr()
3122 dma_addr_t virtqueue_dma_map_single_attrs(struct virtqueue *_vq, void *ptr,  in virtqueue_dma_map_single_attrs()  argument
3127 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_dma_map_single_attrs()
3147 void virtqueue_dma_unmap_single_attrs(struct virtqueue *_vq, dma_addr_t addr,  in virtqueue_dma_unmap_single_attrs()  argument
3151 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_dma_unmap_single_attrs()
3167 int virtqueue_dma_mapping_error(struct virtqueue *_vq, dma_addr_t addr)  in virtqueue_dma_mapping_error()  argument
3169 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_dma_mapping_error()
3188 bool virtqueue_dma_need_sync(struct virtqueue *_vq, dma_addr_t addr)  in virtqueue_dma_need_sync()  argument
3190 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_dma_need_sync()
3211 void virtqueue_dma_sync_single_range_for_cpu(struct virtqueue *_vq,  in virtqueue_dma_sync_single_range_for_cpu()  argument
3216 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_dma_sync_single_range_for_cpu()
3237 void virtqueue_dma_sync_single_range_for_device(struct virtqueue *_vq,  in virtqueue_dma_sync_single_range_for_device()  argument
3242 	struct vring_virtqueue *vq = to_vvq(_vq);  in virtqueue_dma_sync_single_range_for_device()