132 	struct virtqueue *vq;  member
164 	void (*service)(struct virtqueue *vq);
206 #define lg_last_avail(vq)	((vq)->last_avail_idx)  argument
625 static void trigger_irq(struct virtqueue *vq)  in trigger_irq()  argument
627 	unsigned long buf[] = { LHREQ_IRQ, vq->config.irq };  in trigger_irq()
630 	if (!vq->pending_used)  in trigger_irq()
632 	vq->pending_used = 0;  in trigger_irq()
635 	if (vq->vring.avail->flags & VRING_AVAIL_F_NO_INTERRUPT) {  in trigger_irq()
641 		err(1, "Triggering irq %i", vq->config.irq);  in trigger_irq()
652 static unsigned wait_for_vq_desc(struct virtqueue *vq,  in wait_for_vq_desc()  argument
658 	u16 last_avail = lg_last_avail(vq);  in wait_for_vq_desc()
661 	while (last_avail == vq->vring.avail->idx) {  in wait_for_vq_desc()
668 		trigger_irq(vq);  in wait_for_vq_desc()
671 		vq->vring.used->flags &= ~VRING_USED_F_NO_NOTIFY;  in wait_for_vq_desc()
678 		if (last_avail != vq->vring.avail->idx) {  in wait_for_vq_desc()
679 			vq->vring.used->flags |= VRING_USED_F_NO_NOTIFY;  in wait_for_vq_desc()
684 		if (read(vq->eventfd, &event, sizeof(event)) != sizeof(event))  in wait_for_vq_desc()
688 		vq->vring.used->flags |= VRING_USED_F_NO_NOTIFY;  in wait_for_vq_desc()
692 	if ((u16)(vq->vring.avail->idx - last_avail) > vq->vring.num)  in wait_for_vq_desc()
694 		     last_avail, vq->vring.avail->idx);  in wait_for_vq_desc()
700 	head = vq->vring.avail->ring[last_avail % vq->vring.num];  in wait_for_vq_desc()
701 	lg_last_avail(vq)++;  in wait_for_vq_desc()
704 	if (head >= vq->vring.num)  in wait_for_vq_desc()
710 	max = vq->vring.num;  in wait_for_vq_desc()
711 	desc = vq->vring.desc;  in wait_for_vq_desc()
758 static void add_used(struct virtqueue *vq, unsigned int head, int len)  in add_used()  argument
766 	used = &vq->vring.used->ring[vq->vring.used->idx % vq->vring.num];  in add_used()
771 	vq->vring.used->idx++;  in add_used()
772 	vq->pending_used++;  in add_used()
776 static void add_used_and_trigger(struct virtqueue *vq, unsigned head, int len)  in add_used_and_trigger()  argument
778 	add_used(vq, head, len);  in add_used_and_trigger()
779 	trigger_irq(vq);  in add_used_and_trigger()
795 static void console_input(struct virtqueue *vq)  in console_input()  argument
799 	struct console_abort *abort = vq->dev->priv;  in console_input()
800 	struct iovec iov[vq->vring.num];  in console_input()
803 	head = wait_for_vq_desc(vq, iov, &out_num, &in_num);  in console_input()
821 	add_used_and_trigger(vq, head, len);  in console_input()
850 static void console_output(struct virtqueue *vq)  in console_output()  argument
853 	struct iovec iov[vq->vring.num];  in console_output()
856 	head = wait_for_vq_desc(vq, iov, &out, &in);  in console_output()
874 	add_used(vq, head, 0);  in console_output()
887 static void net_output(struct virtqueue *vq)  in net_output()  argument
889 	struct net_info *net_info = vq->dev->priv;  in net_output()
891 	struct iovec iov[vq->vring.num];  in net_output()
894 	head = wait_for_vq_desc(vq, iov, &out, &in);  in net_output()
908 	add_used(vq, head, 0);  in net_output()
931 static void net_input(struct virtqueue *vq)  in net_input()  argument
935 	struct iovec iov[vq->vring.num];  in net_input()
936 	struct net_info *net_info = vq->dev->priv;  in net_input()
942 	head = wait_for_vq_desc(vq, iov, &out, &in);  in net_input()
950 	if (vq->pending_used && will_block(net_info->tunfd))  in net_input()
951 		trigger_irq(vq);  in net_input()
965 	add_used(vq, head, len);  in net_input()
972 	struct virtqueue *vq = _vq;  in do_thread()  local
975 		vq->service(vq);  in do_thread()
990 	struct virtqueue *vq;  in reset_device()  local
1001 	for (vq = dev->vq; vq; vq = vq->next) {  in reset_device()
1002 		if (vq->thread != (pid_t)-1) {  in reset_device()
1003 			kill(vq->thread, SIGTERM);  in reset_device()
1004 			waitpid(vq->thread, NULL, 0);  in reset_device()
1005 			vq->thread = (pid_t)-1;  in reset_device()
1007 		memset(vq->vring.desc, 0,  in reset_device()
1008 		       vring_size(vq->config.num, LGUEST_VRING_ALIGN));  in reset_device()
1009 		lg_last_avail(vq) = 0;  in reset_device()
1020 static void create_thread(struct virtqueue *vq)  in create_thread()  argument
1028 				 vq->config.pfn*getpagesize(), 0 };  in create_thread()
1031 	vq->eventfd = eventfd(0, 0);  in create_thread()
1032 	if (vq->eventfd < 0)  in create_thread()
1034 	args[2] = vq->eventfd;  in create_thread()
1047 	vq->thread = clone(do_thread, stack + 32768, CLONE_VM | SIGCHLD, vq);  in create_thread()
1048 	if (vq->thread == (pid_t)-1)  in create_thread()
1052 	close(vq->eventfd);  in create_thread()
1058 	struct virtqueue *vq;  in start_device()  local
1068 	for (vq = dev->vq; vq; vq = vq->next) {  in start_device()
1069 		if (vq->service)  in start_device()
1070 			create_thread(vq);  in start_device()
1114 		struct virtqueue *vq;  in handle_output()  local
1126 		for (vq = i->vq; vq; vq = vq->next) {  in handle_output()
1127 			if (addr != vq->config.pfn*getpagesize())  in handle_output()
1199 	struct virtqueue **i, *vq = malloc(sizeof(*vq));  in add_virtqueue()  local
1208 	vq->next = NULL;  in add_virtqueue()
1209 	vq->last_avail_idx = 0;  in add_virtqueue()
1210 	vq->dev = dev;  in add_virtqueue()
1216 	vq->service = service;  in add_virtqueue()
1217 	vq->thread = (pid_t)-1;  in add_virtqueue()
1220 	vq->config.num = num_descs;  in add_virtqueue()
1221 	vq->config.irq = devices.next_irq++;  in add_virtqueue()
1222 	vq->config.pfn = to_guest_phys(p) / getpagesize();  in add_virtqueue()
1225 	vring_init(&vq->vring, num_descs, p, LGUEST_VRING_ALIGN);  in add_virtqueue()
1234 	memcpy(device_config(dev), &vq->config, sizeof(vq->config));  in add_virtqueue()
1244 	for (i = &dev->vq; *i; i = &(*i)->next);  in add_virtqueue()
1245 	*i = vq;  in add_virtqueue()
1298 	dev->vq = NULL;  in new_device()
1590 static void blk_request(struct virtqueue *vq)  in blk_request()  argument
1592 	struct vblk_info *vblk = vq->dev->priv;  in blk_request()
1597 	struct iovec iov[vq->vring.num];  in blk_request()
1604 	head = wait_for_vq_desc(vq, iov, &out_num, &in_num);  in blk_request()
1686 	add_used(vq, head, wlen);  in blk_request()
1741 static void rng_input(struct virtqueue *vq)  in rng_input()  argument
1745 	struct rng_info *rng_info = vq->dev->priv;  in rng_input()
1746 	struct iovec iov[vq->vring.num];  in rng_input()
1749 	head = wait_for_vq_desc(vq, iov, &out_num, &in_num);  in rng_input()
1766 	add_used(vq, head, totlen);  in rng_input()