77 #define vu_err(vu_dev, ...)	dev_err(&(vu_dev)->pdev->dev, ##__VA_ARGS__)  argument
128 static int vhost_user_recv(struct virtio_uml_device *vu_dev,  in vhost_user_recv()  argument
160 static void vhost_user_check_reset(struct virtio_uml_device *vu_dev,  in vhost_user_check_reset()  argument
163 	struct virtio_uml_platform_data *pdata = vu_dev->pdata;  in vhost_user_check_reset()
168 	if (!vu_dev->registered)  in vhost_user_check_reset()
171 	virtio_break_device(&vu_dev->vdev);  in vhost_user_check_reset()
175 static int vhost_user_recv_resp(struct virtio_uml_device *vu_dev,  in vhost_user_recv_resp()  argument
179 	int rc = vhost_user_recv(vu_dev, vu_dev->sock, msg,  in vhost_user_recv_resp()
183 		vhost_user_check_reset(vu_dev, rc);  in vhost_user_recv_resp()
193 static int vhost_user_recv_u64(struct virtio_uml_device *vu_dev,  in vhost_user_recv_u64()  argument
197 	int rc = vhost_user_recv_resp(vu_dev, &msg,  in vhost_user_recv_u64()
208 static int vhost_user_recv_req(struct virtio_uml_device *vu_dev,  in vhost_user_recv_req()  argument
212 	int rc = vhost_user_recv(vu_dev, vu_dev->req_fd, msg,  in vhost_user_recv_req()
225 static int vhost_user_send(struct virtio_uml_device *vu_dev,  in vhost_user_send()  argument
242 	if (!(vu_dev->protocol_features &  in vhost_user_send()
249 	spin_lock_irqsave(&vu_dev->sock_lock, flags);  in vhost_user_send()
250 	rc = full_sendmsg_fds(vu_dev->sock, msg, size, fds, num_fds);  in vhost_user_send()
257 		rc = vhost_user_recv_u64(vu_dev, &status);  in vhost_user_send()
262 			vu_err(vu_dev, "slave reports error: %llu\n", status);  in vhost_user_send()
269 	spin_unlock_irqrestore(&vu_dev->sock_lock, flags);  in vhost_user_send()
273 static int vhost_user_send_no_payload(struct virtio_uml_device *vu_dev,  in vhost_user_send_no_payload()  argument
280 	return vhost_user_send(vu_dev, need_response, &msg, NULL, 0);  in vhost_user_send_no_payload()
283 static int vhost_user_send_no_payload_fd(struct virtio_uml_device *vu_dev,  in vhost_user_send_no_payload_fd()  argument
290 	return vhost_user_send(vu_dev, false, &msg, &fd, 1);  in vhost_user_send_no_payload_fd()
293 static int vhost_user_send_u64(struct virtio_uml_device *vu_dev,  in vhost_user_send_u64()  argument
302 	return vhost_user_send(vu_dev, false, &msg, NULL, 0);  in vhost_user_send_u64()
305 static int vhost_user_set_owner(struct virtio_uml_device *vu_dev)  in vhost_user_set_owner()  argument
307 	return vhost_user_send_no_payload(vu_dev, false, VHOST_USER_SET_OWNER);  in vhost_user_set_owner()
310 static int vhost_user_get_features(struct virtio_uml_device *vu_dev,  in vhost_user_get_features()  argument
313 	int rc = vhost_user_send_no_payload(vu_dev, true,  in vhost_user_get_features()
318 	return vhost_user_recv_u64(vu_dev, features);  in vhost_user_get_features()
321 static int vhost_user_set_features(struct virtio_uml_device *vu_dev,  in vhost_user_set_features()  argument
324 	return vhost_user_send_u64(vu_dev, VHOST_USER_SET_FEATURES, features);  in vhost_user_set_features()
327 static int vhost_user_get_protocol_features(struct virtio_uml_device *vu_dev,  in vhost_user_get_protocol_features()  argument
330 	int rc = vhost_user_send_no_payload(vu_dev, true,  in vhost_user_get_protocol_features()
335 	return vhost_user_recv_u64(vu_dev, protocol_features);  in vhost_user_get_protocol_features()
338 static int vhost_user_set_protocol_features(struct virtio_uml_device *vu_dev,  in vhost_user_set_protocol_features()  argument
341 	return vhost_user_send_u64(vu_dev, VHOST_USER_SET_PROTOCOL_FEATURES,  in vhost_user_set_protocol_features()
345 static void vhost_user_reply(struct virtio_uml_device *vu_dev,  in vhost_user_reply()  argument
359 	rc = full_sendmsg_fds(vu_dev->req_fd, &reply, size, NULL, 0);  in vhost_user_reply()
362 		vu_err(vu_dev,  in vhost_user_reply()
367 static irqreturn_t vu_req_read_message(struct virtio_uml_device *vu_dev,  in vu_req_read_message()  argument
380 		rc = vhost_user_recv_req(vu_dev, &msg.msg,  in vu_req_read_message()
388 			vu_dev->config_changed_irq = true;  in vu_req_read_message()
392 			virtio_device_for_each_vq((&vu_dev->vdev), vq) {  in vu_req_read_message()
395 					vu_dev->vq_irq_vq_map |= BIT_ULL(vq->index);  in vu_req_read_message()
405 			vu_err(vu_dev, "unexpected slave request %d\n",  in vu_req_read_message()
409 		if (ev && !vu_dev->suspended)  in vu_req_read_message()
413 			vhost_user_reply(vu_dev, &msg.msg, response);  in vu_req_read_message()
417 	vu_dev->recv_rc = (rc == -EAGAIN) ? 0 : rc;  in vu_req_read_message()
423 	struct virtio_uml_device *vu_dev = data;  in vu_req_interrupt()  local
427 		ret = vu_req_read_message(vu_dev, NULL);  in vu_req_interrupt()
429 	if (vu_dev->recv_rc) {  in vu_req_interrupt()
430 		vhost_user_check_reset(vu_dev, vu_dev->recv_rc);  in vu_req_interrupt()
431 	} else if (vu_dev->vq_irq_vq_map) {  in vu_req_interrupt()
434 		virtio_device_for_each_vq((&vu_dev->vdev), vq) {  in vu_req_interrupt()
435 			if (vu_dev->vq_irq_vq_map & BIT_ULL(vq->index))  in vu_req_interrupt()
438 		vu_dev->vq_irq_vq_map = 0;  in vu_req_interrupt()
439 	} else if (vu_dev->config_changed_irq) {  in vu_req_interrupt()
440 		virtio_config_changed(&vu_dev->vdev);  in vu_req_interrupt()
441 		vu_dev->config_changed_irq = false;  in vu_req_interrupt()
453 static int vhost_user_init_slave_req(struct virtio_uml_device *vu_dev)  in vhost_user_init_slave_req()  argument
461 	vu_dev->req_fd = req_fds[0];  in vhost_user_init_slave_req()
463 	rc = um_request_irq_tt(UM_IRQ_ALLOC, vu_dev->req_fd, IRQ_READ,  in vhost_user_init_slave_req()
465 			       vu_dev->pdev->name, vu_dev,  in vhost_user_init_slave_req()
470 	vu_dev->irq = rc;  in vhost_user_init_slave_req()
472 	rc = vhost_user_send_no_payload_fd(vu_dev, VHOST_USER_SET_SLAVE_REQ_FD,  in vhost_user_init_slave_req()
480 	um_free_irq(vu_dev->irq, vu_dev);  in vhost_user_init_slave_req()
489 static int vhost_user_init(struct virtio_uml_device *vu_dev)  in vhost_user_init()  argument
491 	int rc = vhost_user_set_owner(vu_dev);  in vhost_user_init()
495 	rc = vhost_user_get_features(vu_dev, &vu_dev->features);  in vhost_user_init()
499 	if (vu_dev->features & BIT_ULL(VHOST_USER_F_PROTOCOL_FEATURES)) {  in vhost_user_init()
500 		rc = vhost_user_get_protocol_features(vu_dev,  in vhost_user_init()
501 				&vu_dev->protocol_features);  in vhost_user_init()
504 		vu_dev->protocol_features &= VHOST_USER_SUPPORTED_PROTOCOL_F;  in vhost_user_init()
505 		rc = vhost_user_set_protocol_features(vu_dev,  in vhost_user_init()
506 				vu_dev->protocol_features);  in vhost_user_init()
511 	if (vu_dev->protocol_features &  in vhost_user_init()
513 		rc = vhost_user_init_slave_req(vu_dev);  in vhost_user_init()
521 static void vhost_user_get_config(struct virtio_uml_device *vu_dev,  in vhost_user_get_config()  argument
530 	if (!(vu_dev->protocol_features &  in vhost_user_get_config()
542 	rc = vhost_user_send(vu_dev, true, msg, NULL, 0);  in vhost_user_get_config()
544 		vu_err(vu_dev, "sending VHOST_USER_GET_CONFIG failed: %d\n",  in vhost_user_get_config()
549 	rc = vhost_user_recv_resp(vu_dev, msg, msg_size);  in vhost_user_get_config()
551 		vu_err(vu_dev,  in vhost_user_get_config()
560 		vu_err(vu_dev,  in vhost_user_get_config()
572 static void vhost_user_set_config(struct virtio_uml_device *vu_dev,  in vhost_user_set_config()  argument
580 	if (!(vu_dev->protocol_features &  in vhost_user_set_config()
593 	rc = vhost_user_send(vu_dev, false, msg, NULL, 0);  in vhost_user_set_config()
595 		vu_err(vu_dev, "sending VHOST_USER_SET_CONFIG failed: %d\n",  in vhost_user_set_config()
623 static int vhost_user_set_mem_table(struct virtio_uml_device *vu_dev)  in vhost_user_set_mem_table()  argument
683 	return vhost_user_send(vu_dev, false, &msg, fds,  in vhost_user_set_mem_table()
687 static int vhost_user_set_vring_state(struct virtio_uml_device *vu_dev,  in vhost_user_set_vring_state()  argument
697 	return vhost_user_send(vu_dev, false, &msg, NULL, 0);  in vhost_user_set_vring_state()
700 static int vhost_user_set_vring_num(struct virtio_uml_device *vu_dev,  in vhost_user_set_vring_num()  argument
703 	return vhost_user_set_vring_state(vu_dev, VHOST_USER_SET_VRING_NUM,  in vhost_user_set_vring_num()
707 static int vhost_user_set_vring_base(struct virtio_uml_device *vu_dev,  in vhost_user_set_vring_base()  argument
710 	return vhost_user_set_vring_state(vu_dev, VHOST_USER_SET_VRING_BASE,  in vhost_user_set_vring_base()
714 static int vhost_user_set_vring_addr(struct virtio_uml_device *vu_dev,  in vhost_user_set_vring_addr()  argument
728 	return vhost_user_send(vu_dev, false, &msg, NULL, 0);  in vhost_user_set_vring_addr()
731 static int vhost_user_set_vring_fd(struct virtio_uml_device *vu_dev,  in vhost_user_set_vring_fd()  argument
744 		return vhost_user_send(vu_dev, false, &msg, NULL, 0);  in vhost_user_set_vring_fd()
746 	return vhost_user_send(vu_dev, false, &msg, &fd, 1);  in vhost_user_set_vring_fd()
749 static int vhost_user_set_vring_call(struct virtio_uml_device *vu_dev,  in vhost_user_set_vring_call()  argument
752 	return vhost_user_set_vring_fd(vu_dev, VHOST_USER_SET_VRING_CALL,  in vhost_user_set_vring_call()
756 static int vhost_user_set_vring_kick(struct virtio_uml_device *vu_dev,  in vhost_user_set_vring_kick()  argument
759 	return vhost_user_set_vring_fd(vu_dev, VHOST_USER_SET_VRING_KICK,  in vhost_user_set_vring_kick()
763 static int vhost_user_set_vring_enable(struct virtio_uml_device *vu_dev,  in vhost_user_set_vring_enable()  argument
766 	if (!(vu_dev->features & BIT_ULL(VHOST_USER_F_PROTOCOL_FEATURES)))  in vhost_user_set_vring_enable()
769 	return vhost_user_set_vring_state(vu_dev, VHOST_USER_SET_VRING_ENABLE,  in vhost_user_set_vring_enable()
788 		struct virtio_uml_device *vu_dev;  in vu_notify()  local
790 		vu_dev = to_virtio_uml_device(vq->vdev);  in vu_notify()
792 		return vhost_user_set_vring_state(vu_dev, VHOST_USER_VRING_KICK,  in vu_notify()
823 	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);  in vu_get()  local
825 	vhost_user_get_config(vu_dev, offset, buf, len);  in vu_get()
831 	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);  in vu_set()  local
833 	vhost_user_set_config(vu_dev, offset, buf, len);  in vu_set()
838 	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);  in vu_get_status()  local
840 	return vu_dev->status;  in vu_get_status()
845 	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);  in vu_set_status()  local
847 	vu_dev->status = status;  in vu_set_status()
852 	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);  in vu_reset()  local
854 	vu_dev->status = 0;  in vu_reset()
862 		struct virtio_uml_device *vu_dev;  in vu_del_vq()  local
864 		vu_dev = to_virtio_uml_device(vq->vdev);  in vu_del_vq()
866 		um_free_irq(vu_dev->irq, vq);  in vu_del_vq()
879 	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);  in vu_del_vqs()  local
885 		WARN_ON(vhost_user_set_vring_enable(vu_dev, vq->index, false));  in vu_del_vqs()
888 	WARN_ON(vhost_user_get_features(vu_dev, &features));  in vu_del_vqs()
894 static int vu_setup_vq_call_fd(struct virtio_uml_device *vu_dev,  in vu_setup_vq_call_fd()  argument
902 	if (vu_dev->protocol_features &  in vu_setup_vq_call_fd()
904 	    vu_dev->protocol_features &  in vu_setup_vq_call_fd()
916 	rc = um_request_irq(vu_dev->irq, info->call_fd, IRQ_READ,  in vu_setup_vq_call_fd()
921 	rc = vhost_user_set_vring_call(vu_dev, vq->index, call_fds[1]);  in vu_setup_vq_call_fd()
928 	um_free_irq(vu_dev->irq, vq);  in vu_setup_vq_call_fd()
942 	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);  in vu_setup_vq()  local
943 	struct platform_device *pdev = vu_dev->pdev;  in vu_setup_vq()
967 	if (vu_dev->protocol_features &  in vu_setup_vq()
977 	rc = vu_setup_vq_call_fd(vu_dev, vq);  in vu_setup_vq()
981 	rc = vhost_user_set_vring_num(vu_dev, index, num);  in vu_setup_vq()
985 	rc = vhost_user_set_vring_base(vu_dev, index, 0);  in vu_setup_vq()
989 	rc = vhost_user_set_vring_addr(vu_dev, index,  in vu_setup_vq()
1001 		um_free_irq(vu_dev->irq, vq);  in vu_setup_vq()
1020 	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);  in vu_find_vqs()  local
1028 	rc = vhost_user_set_mem_table(vu_dev);  in vu_find_vqs()
1050 			rc = vhost_user_set_vring_kick(vu_dev, vq->index,  in vu_find_vqs()
1056 		rc = vhost_user_set_vring_enable(vu_dev, vq->index, true);  in vu_find_vqs()
1070 	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);  in vu_get_features()  local
1072 	return vu_dev->features;  in vu_get_features()
1077 	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);  in vu_finalize_features()  local
1081 	vu_dev->features = vdev->features | supported;  in vu_finalize_features()
1083 	return vhost_user_set_features(vu_dev, vu_dev->features);  in vu_finalize_features()
1088 	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);  in vu_bus_name()  local
1090 	return vu_dev->pdev->name;  in vu_bus_name()
1110 	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);  in virtio_uml_release_dev()  local
1115 	if (vu_dev->req_fd >= 0) {  in virtio_uml_release_dev()
1116 		um_free_irq(vu_dev->irq, vu_dev);  in virtio_uml_release_dev()
1117 		os_close_file(vu_dev->req_fd);  in virtio_uml_release_dev()
1120 	os_close_file(vu_dev->sock);  in virtio_uml_release_dev()
1121 	kfree(vu_dev);  in virtio_uml_release_dev()
1127 	struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);  in virtio_uml_set_no_vq_suspend()  local
1132 	vu_dev->no_vq_suspend = no_vq_suspend;  in virtio_uml_set_no_vq_suspend()
1180 	struct virtio_uml_device *vu_dev;  in virtio_uml_probe()  local
1189 	vu_dev = kzalloc(sizeof(*vu_dev), GFP_KERNEL);  in virtio_uml_probe()
1190 	if (!vu_dev)  in virtio_uml_probe()
1193 	vu_dev->pdata = pdata;  in virtio_uml_probe()
1194 	vu_dev->vdev.dev.parent = &pdev->dev;  in virtio_uml_probe()
1195 	vu_dev->vdev.dev.release = virtio_uml_release_dev;  in virtio_uml_probe()
1196 	vu_dev->vdev.config = &virtio_uml_config_ops;  in virtio_uml_probe()
1197 	vu_dev->vdev.id.device = pdata->virtio_device_id;  in virtio_uml_probe()
1198 	vu_dev->vdev.id.vendor = VIRTIO_DEV_ANY_ID;  in virtio_uml_probe()
1199 	vu_dev->pdev = pdev;  in virtio_uml_probe()
1200 	vu_dev->req_fd = -1;  in virtio_uml_probe()
1209 	vu_dev->sock = rc;  in virtio_uml_probe()
1211 	spin_lock_init(&vu_dev->sock_lock);  in virtio_uml_probe()
1213 	rc = vhost_user_init(vu_dev);  in virtio_uml_probe()
1217 	platform_set_drvdata(pdev, vu_dev);  in virtio_uml_probe()
1219 	device_set_wakeup_capable(&vu_dev->vdev.dev, true);  in virtio_uml_probe()
1221 	rc = register_virtio_device(&vu_dev->vdev);  in virtio_uml_probe()
1223 		put_device(&vu_dev->vdev.dev);  in virtio_uml_probe()
1224 	vu_dev->registered = 1;  in virtio_uml_probe()
1228 	os_close_file(vu_dev->sock);  in virtio_uml_probe()
1230 	kfree(vu_dev);  in virtio_uml_probe()
1236 	struct virtio_uml_device *vu_dev = platform_get_drvdata(pdev);  in virtio_uml_remove()  local
1238 	unregister_virtio_device(&vu_dev->vdev);  in virtio_uml_remove()
1386 	struct virtio_uml_device *vu_dev = platform_get_drvdata(pdev);  in virtio_uml_suspend()  local
1388 	if (!vu_dev->no_vq_suspend) {  in virtio_uml_suspend()
1391 		virtio_device_for_each_vq((&vu_dev->vdev), vq) {  in virtio_uml_suspend()
1395 			vhost_user_set_vring_enable(vu_dev, vq->index, false);  in virtio_uml_suspend()
1399 	if (!device_may_wakeup(&vu_dev->vdev.dev)) {  in virtio_uml_suspend()
1400 		vu_dev->suspended = true;  in virtio_uml_suspend()
1404 	return irq_set_irq_wake(vu_dev->irq, 1);  in virtio_uml_suspend()
1409 	struct virtio_uml_device *vu_dev = platform_get_drvdata(pdev);  in virtio_uml_resume()  local
1411 	if (!vu_dev->no_vq_suspend) {  in virtio_uml_resume()
1414 		virtio_device_for_each_vq((&vu_dev->vdev), vq) {  in virtio_uml_resume()
1418 			vhost_user_set_vring_enable(vu_dev, vq->index, true);  in virtio_uml_resume()
1422 	vu_dev->suspended = false;  in virtio_uml_resume()
1424 	if (!device_may_wakeup(&vu_dev->vdev.dev))  in virtio_uml_resume()
1427 	return irq_set_irq_wake(vu_dev->irq, 0);  in virtio_uml_resume()