42 static struct vdpasim *vdpa_to_sim(struct vdpa_device *vdpa)  in vdpa_to_sim()
44 	return container_of(vdpa, struct vdpasim, vdpa);  in vdpa_to_sim()
47 static struct vdpasim *dev_to_sim(struct device *dev)  in dev_to_sim()
65 static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)  in vdpasim_queue_ready()  argument
67 	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];  in vdpasim_queue_ready()
69 	vringh_init_iotlb(&vq->vring, vdpasim->dev_attr.supported_features,  in vdpasim_queue_ready()
80 static void vdpasim_vq_reset(struct vdpasim *vdpasim,  in vdpasim_vq_reset()  argument
89 	vringh_init_iotlb(&vq->vring, vdpasim->dev_attr.supported_features,  in vdpasim_vq_reset()
95 static void vdpasim_do_reset(struct vdpasim *vdpasim)  in vdpasim_do_reset()  argument
99 	spin_lock(&vdpasim->iommu_lock);  in vdpasim_do_reset()
101 	for (i = 0; i < vdpasim->dev_attr.nvqs; i++) {  in vdpasim_do_reset()
102 		vdpasim_vq_reset(vdpasim, &vdpasim->vqs[i]);  in vdpasim_do_reset()
103 		vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0],  in vdpasim_do_reset()
104 				 &vdpasim->iommu_lock);  in vdpasim_do_reset()
107 	for (i = 0; i < vdpasim->dev_attr.nas; i++)  in vdpasim_do_reset()
108 		vhost_iotlb_reset(&vdpasim->iommu[i]);  in vdpasim_do_reset()
110 	spin_unlock(&vdpasim->iommu_lock);  in vdpasim_do_reset()
112 	vdpasim->features = 0;  in vdpasim_do_reset()
113 	vdpasim->status = 0;  in vdpasim_do_reset()
114 	++vdpasim->generation;  in vdpasim_do_reset()
138 static dma_addr_t vdpasim_map_range(struct vdpasim *vdpasim, phys_addr_t paddr,  in vdpasim_map_range()  argument
146 	iova = alloc_iova(&vdpasim->iova, size >> iova_shift(&vdpasim->iova),  in vdpasim_map_range()
151 	dma_addr = iova_dma_addr(&vdpasim->iova, iova);  in vdpasim_map_range()
153 	spin_lock(&vdpasim->iommu_lock);  in vdpasim_map_range()
154 	ret = vhost_iotlb_add_range(&vdpasim->iommu[0], (u64)dma_addr,  in vdpasim_map_range()
156 	spin_unlock(&vdpasim->iommu_lock);  in vdpasim_map_range()
159 		__free_iova(&vdpasim->iova, iova);  in vdpasim_map_range()
166 static void vdpasim_unmap_range(struct vdpasim *vdpasim, dma_addr_t dma_addr,  in vdpasim_unmap_range()  argument
169 	spin_lock(&vdpasim->iommu_lock);  in vdpasim_unmap_range()
170 	vhost_iotlb_del_range(&vdpasim->iommu[0], (u64)dma_addr,  in vdpasim_unmap_range()
172 	spin_unlock(&vdpasim->iommu_lock);  in vdpasim_unmap_range()
174 	free_iova(&vdpasim->iova, iova_pfn(&vdpasim->iova, dma_addr));  in vdpasim_unmap_range()
182 	struct vdpasim *vdpasim = dev_to_sim(dev);  in vdpasim_map_page()  local
189 	return vdpasim_map_range(vdpasim, paddr, size, perm);  in vdpasim_map_page()
196 	struct vdpasim *vdpasim = dev_to_sim(dev);  in vdpasim_unmap_page()  local
198 	vdpasim_unmap_range(vdpasim, dma_addr, size);  in vdpasim_unmap_page()
205 	struct vdpasim *vdpasim = dev_to_sim(dev);  in vdpasim_alloc_coherent()  local
217 	*dma_addr = vdpasim_map_range(vdpasim, paddr, size, VHOST_MAP_RW);  in vdpasim_alloc_coherent()
230 	struct vdpasim *vdpasim = dev_to_sim(dev);  in vdpasim_free_coherent()  local
232 	vdpasim_unmap_range(vdpasim, dma_addr, size);  in vdpasim_free_coherent()
247 struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr)  in vdpasim_create()
250 	struct vdpasim *vdpasim;  in vdpasim_create()  local
259 	vdpasim = vdpa_alloc_device(struct vdpasim, vdpa, NULL, ops,  in vdpasim_create()
262 	if (IS_ERR(vdpasim)) {  in vdpasim_create()
263 		ret = PTR_ERR(vdpasim);  in vdpasim_create()
267 	vdpasim->dev_attr = *dev_attr;  in vdpasim_create()
268 	INIT_WORK(&vdpasim->work, dev_attr->work_fn);  in vdpasim_create()
269 	spin_lock_init(&vdpasim->lock);  in vdpasim_create()
270 	spin_lock_init(&vdpasim->iommu_lock);  in vdpasim_create()
272 	dev = &vdpasim->vdpa.dev;  in vdpasim_create()
277 	vdpasim->vdpa.mdev = dev_attr->mgmt_dev;  in vdpasim_create()
279 	vdpasim->config = kzalloc(dev_attr->config_size, GFP_KERNEL);  in vdpasim_create()
280 	if (!vdpasim->config)  in vdpasim_create()
283 	vdpasim->vqs = kcalloc(dev_attr->nvqs, sizeof(struct vdpasim_virtqueue),  in vdpasim_create()
285 	if (!vdpasim->vqs)  in vdpasim_create()
288 	vdpasim->iommu = kmalloc_array(vdpasim->dev_attr.nas,  in vdpasim_create()
289 				       sizeof(*vdpasim->iommu), GFP_KERNEL);  in vdpasim_create()
290 	if (!vdpasim->iommu)  in vdpasim_create()
293 	for (i = 0; i < vdpasim->dev_attr.nas; i++)  in vdpasim_create()
294 		vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0);  in vdpasim_create()
296 	vdpasim->buffer = kvmalloc(dev_attr->buffer_size, GFP_KERNEL);  in vdpasim_create()
297 	if (!vdpasim->buffer)  in vdpasim_create()
301 		vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0],  in vdpasim_create()
302 				 &vdpasim->iommu_lock);  in vdpasim_create()
309 	init_iova_domain(&vdpasim->iova, 1, 0);  in vdpasim_create()
311 	vdpasim->vdpa.dma_dev = dev;  in vdpasim_create()
313 	return vdpasim;  in vdpasim_create()
326 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_set_vq_address()  local
327 	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];  in vdpasim_set_vq_address()
338 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_set_vq_num()  local
339 	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];  in vdpasim_set_vq_num()
346 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_kick_vq()  local
347 	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];  in vdpasim_kick_vq()
350 		schedule_work(&vdpasim->work);  in vdpasim_kick_vq()
356 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_set_vq_cb()  local
357 	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];  in vdpasim_set_vq_cb()
365 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_set_vq_ready()  local
366 	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];  in vdpasim_set_vq_ready()
369 	spin_lock(&vdpasim->lock);  in vdpasim_set_vq_ready()
373 		vdpasim_queue_ready(vdpasim, idx);  in vdpasim_set_vq_ready()
375 	spin_unlock(&vdpasim->lock);  in vdpasim_set_vq_ready()
380 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_get_vq_ready()  local
381 	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];  in vdpasim_get_vq_ready()
389 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_set_vq_state()  local
390 	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];  in vdpasim_set_vq_state()
393 	spin_lock(&vdpasim->lock);  in vdpasim_set_vq_state()
395 	spin_unlock(&vdpasim->lock);  in vdpasim_set_vq_state()
403 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_get_vq_state()  local
404 	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];  in vdpasim_get_vq_state()
427 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_get_device_features()  local
429 	return vdpasim->dev_attr.supported_features;  in vdpasim_get_device_features()
434 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_set_driver_features()  local
440 	vdpasim->features = features & vdpasim->dev_attr.supported_features;  in vdpasim_set_driver_features()
447 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_get_driver_features()  local
449 	return vdpasim->features;  in vdpasim_get_driver_features()
465 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_get_device_id()  local
467 	return vdpasim->dev_attr.id;  in vdpasim_get_device_id()
477 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_get_status()  local
480 	spin_lock(&vdpasim->lock);  in vdpasim_get_status()
481 	status = vdpasim->status;  in vdpasim_get_status()
482 	spin_unlock(&vdpasim->lock);  in vdpasim_get_status()
489 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_set_status()  local
491 	spin_lock(&vdpasim->lock);  in vdpasim_set_status()
492 	vdpasim->status = status;  in vdpasim_set_status()
493 	spin_unlock(&vdpasim->lock);  in vdpasim_set_status()
498 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_reset()  local
500 	spin_lock(&vdpasim->lock);  in vdpasim_reset()
501 	vdpasim->status = 0;  in vdpasim_reset()
502 	vdpasim_do_reset(vdpasim);  in vdpasim_reset()
503 	spin_unlock(&vdpasim->lock);  in vdpasim_reset()
510 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_get_config_size()  local
512 	return vdpasim->dev_attr.config_size;  in vdpasim_get_config_size()
518 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_get_config()  local
520 	if (offset + len > vdpasim->dev_attr.config_size)  in vdpasim_get_config()
523 	if (vdpasim->dev_attr.get_config)  in vdpasim_get_config()
524 		vdpasim->dev_attr.get_config(vdpasim, vdpasim->config);  in vdpasim_get_config()
526 	memcpy(buf, vdpasim->config + offset, len);  in vdpasim_get_config()
532 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_set_config()  local
534 	if (offset + len > vdpasim->dev_attr.config_size)  in vdpasim_set_config()
537 	memcpy(vdpasim->config + offset, buf, len);  in vdpasim_set_config()
539 	if (vdpasim->dev_attr.set_config)  in vdpasim_set_config()
540 		vdpasim->dev_attr.set_config(vdpasim, vdpasim->config);  in vdpasim_set_config()
545 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_get_generation()  local
547 	return vdpasim->generation;  in vdpasim_get_generation()
563 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_set_group_asid()  local
567 	if (group > vdpasim->dev_attr.ngroups)  in vdpasim_set_group_asid()
570 	if (asid >= vdpasim->dev_attr.nas)  in vdpasim_set_group_asid()
573 	iommu = &vdpasim->iommu[asid];  in vdpasim_set_group_asid()
575 	spin_lock(&vdpasim->lock);  in vdpasim_set_group_asid()
577 	for (i = 0; i < vdpasim->dev_attr.nvqs; i++)  in vdpasim_set_group_asid()
579 			vringh_set_iotlb(&vdpasim->vqs[i].vring, iommu,  in vdpasim_set_group_asid()
580 					 &vdpasim->iommu_lock);  in vdpasim_set_group_asid()
582 	spin_unlock(&vdpasim->lock);  in vdpasim_set_group_asid()
590 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_set_map()  local
596 	if (asid >= vdpasim->dev_attr.nas)  in vdpasim_set_map()
599 	spin_lock(&vdpasim->iommu_lock);  in vdpasim_set_map()
601 	iommu = &vdpasim->iommu[asid];  in vdpasim_set_map()
611 	spin_unlock(&vdpasim->iommu_lock);  in vdpasim_set_map()
616 	spin_unlock(&vdpasim->iommu_lock);  in vdpasim_set_map()
624 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_dma_map()  local
627 	if (asid >= vdpasim->dev_attr.nas)  in vdpasim_dma_map()
630 	spin_lock(&vdpasim->iommu_lock);  in vdpasim_dma_map()
631 	ret = vhost_iotlb_add_range_ctx(&vdpasim->iommu[asid], iova,  in vdpasim_dma_map()
633 	spin_unlock(&vdpasim->iommu_lock);  in vdpasim_dma_map()
641 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_dma_unmap()  local
643 	if (asid >= vdpasim->dev_attr.nas)  in vdpasim_dma_unmap()
646 	spin_lock(&vdpasim->iommu_lock);  in vdpasim_dma_unmap()
647 	vhost_iotlb_del_range(&vdpasim->iommu[asid], iova, iova + size - 1);  in vdpasim_dma_unmap()
648 	spin_unlock(&vdpasim->iommu_lock);  in vdpasim_dma_unmap()
655 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);  in vdpasim_free()  local
658 	cancel_work_sync(&vdpasim->work);  in vdpasim_free()
660 	for (i = 0; i < vdpasim->dev_attr.nvqs; i++) {  in vdpasim_free()
661 		vringh_kiov_cleanup(&vdpasim->vqs[i].out_iov);  in vdpasim_free()
662 		vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov);  in vdpasim_free()
666 		put_iova_domain(&vdpasim->iova);  in vdpasim_free()
670 	kvfree(vdpasim->buffer);  in vdpasim_free()
671 	vhost_iotlb_free(vdpasim->iommu);  in vdpasim_free()
672 	kfree(vdpasim->vqs);  in vdpasim_free()
673 	kfree(vdpasim->config);  in vdpasim_free()