1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #include <linux/init.h>
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <linux/pci.h>
7 #include <linux/device.h>
8 #include <linux/sched/task.h>
9 #include <linux/intel-svm.h>
10 #include <linux/io-64-nonatomic-lo-hi.h>
11 #include <linux/cdev.h>
12 #include <linux/fs.h>
13 #include <linux/poll.h>
14 #include <linux/iommu.h>
15 #include <uapi/linux/idxd.h>
16 #include "registers.h"
17 #include "idxd.h"
18 
19 struct idxd_cdev_context {
20 	const char *name;
21 	dev_t devt;
22 	struct ida minor_ida;
23 };
24 
25 /*
26  * ictx is an array based off of accelerator types. enum idxd_type
27  * is used as index
28  */
29 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
30 	{ .name = "dsa" },
31 	{ .name = "iax" }
32 };
33 
34 struct idxd_user_context {
35 	struct idxd_wq *wq;
36 	struct task_struct *task;
37 	unsigned int pasid;
38 	unsigned int flags;
39 	struct iommu_sva *sva;
40 };
41 
idxd_cdev_dev_release(struct device * dev)42 static void idxd_cdev_dev_release(struct device *dev)
43 {
44 	struct idxd_cdev *idxd_cdev = dev_to_cdev(dev);
45 	struct idxd_cdev_context *cdev_ctx;
46 	struct idxd_wq *wq = idxd_cdev->wq;
47 
48 	cdev_ctx = &ictx[wq->idxd->data->type];
49 	ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
50 	kfree(idxd_cdev);
51 }
52 
53 static struct device_type idxd_cdev_device_type = {
54 	.name = "idxd_cdev",
55 	.release = idxd_cdev_dev_release,
56 };
57 
inode_idxd_cdev(struct inode * inode)58 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
59 {
60 	struct cdev *cdev = inode->i_cdev;
61 
62 	return container_of(cdev, struct idxd_cdev, cdev);
63 }
64 
inode_wq(struct inode * inode)65 static inline struct idxd_wq *inode_wq(struct inode *inode)
66 {
67 	struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
68 
69 	return idxd_cdev->wq;
70 }
71 
idxd_cdev_open(struct inode * inode,struct file * filp)72 static int idxd_cdev_open(struct inode *inode, struct file *filp)
73 {
74 	struct idxd_user_context *ctx;
75 	struct idxd_device *idxd;
76 	struct idxd_wq *wq;
77 	struct device *dev;
78 	int rc = 0;
79 	struct iommu_sva *sva;
80 	unsigned int pasid;
81 
82 	wq = inode_wq(inode);
83 	idxd = wq->idxd;
84 	dev = &idxd->pdev->dev;
85 
86 	dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
87 
88 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
89 	if (!ctx)
90 		return -ENOMEM;
91 
92 	mutex_lock(&wq->wq_lock);
93 
94 	if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {
95 		rc = -EBUSY;
96 		goto failed;
97 	}
98 
99 	ctx->wq = wq;
100 	filp->private_data = ctx;
101 
102 	if (device_user_pasid_enabled(idxd)) {
103 		sva = iommu_sva_bind_device(dev, current->mm, NULL);
104 		if (IS_ERR(sva)) {
105 			rc = PTR_ERR(sva);
106 			dev_err(dev, "pasid allocation failed: %d\n", rc);
107 			goto failed;
108 		}
109 
110 		pasid = iommu_sva_get_pasid(sva);
111 		if (pasid == IOMMU_PASID_INVALID) {
112 			iommu_sva_unbind_device(sva);
113 			rc = -EINVAL;
114 			goto failed;
115 		}
116 
117 		ctx->sva = sva;
118 		ctx->pasid = pasid;
119 
120 		if (wq_dedicated(wq)) {
121 			rc = idxd_wq_set_pasid(wq, pasid);
122 			if (rc < 0) {
123 				iommu_sva_unbind_device(sva);
124 				dev_err(dev, "wq set pasid failed: %d\n", rc);
125 				goto failed;
126 			}
127 		}
128 	}
129 
130 	idxd_wq_get(wq);
131 	mutex_unlock(&wq->wq_lock);
132 	return 0;
133 
134  failed:
135 	mutex_unlock(&wq->wq_lock);
136 	kfree(ctx);
137 	return rc;
138 }
139 
idxd_cdev_release(struct inode * node,struct file * filep)140 static int idxd_cdev_release(struct inode *node, struct file *filep)
141 {
142 	struct idxd_user_context *ctx = filep->private_data;
143 	struct idxd_wq *wq = ctx->wq;
144 	struct idxd_device *idxd = wq->idxd;
145 	struct device *dev = &idxd->pdev->dev;
146 	int rc;
147 
148 	dev_dbg(dev, "%s called\n", __func__);
149 	filep->private_data = NULL;
150 
151 	/* Wait for in-flight operations to complete. */
152 	if (wq_shared(wq)) {
153 		idxd_device_drain_pasid(idxd, ctx->pasid);
154 	} else {
155 		if (device_user_pasid_enabled(idxd)) {
156 			/* The wq disable in the disable pasid function will drain the wq */
157 			rc = idxd_wq_disable_pasid(wq);
158 			if (rc < 0)
159 				dev_err(dev, "wq disable pasid failed.\n");
160 		} else {
161 			idxd_wq_drain(wq);
162 		}
163 	}
164 
165 	if (ctx->sva)
166 		iommu_sva_unbind_device(ctx->sva);
167 	kfree(ctx);
168 	mutex_lock(&wq->wq_lock);
169 	idxd_wq_put(wq);
170 	mutex_unlock(&wq->wq_lock);
171 	return 0;
172 }
173 
check_vma(struct idxd_wq * wq,struct vm_area_struct * vma,const char * func)174 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
175 		     const char *func)
176 {
177 	struct device *dev = &wq->idxd->pdev->dev;
178 
179 	if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
180 		dev_info_ratelimited(dev,
181 				     "%s: %s: mapping too large: %lu\n",
182 				     current->comm, func,
183 				     vma->vm_end - vma->vm_start);
184 		return -EINVAL;
185 	}
186 
187 	return 0;
188 }
189 
idxd_cdev_mmap(struct file * filp,struct vm_area_struct * vma)190 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
191 {
192 	struct idxd_user_context *ctx = filp->private_data;
193 	struct idxd_wq *wq = ctx->wq;
194 	struct idxd_device *idxd = wq->idxd;
195 	struct pci_dev *pdev = idxd->pdev;
196 	phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
197 	unsigned long pfn;
198 	int rc;
199 
200 	dev_dbg(&pdev->dev, "%s called\n", __func__);
201 	rc = check_vma(wq, vma, __func__);
202 	if (rc < 0)
203 		return rc;
204 
205 	vma->vm_flags |= VM_DONTCOPY;
206 	pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
207 				IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
208 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
209 	vma->vm_private_data = ctx;
210 
211 	return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
212 			vma->vm_page_prot);
213 }
214 
idxd_cdev_poll(struct file * filp,struct poll_table_struct * wait)215 static __poll_t idxd_cdev_poll(struct file *filp,
216 			       struct poll_table_struct *wait)
217 {
218 	struct idxd_user_context *ctx = filp->private_data;
219 	struct idxd_wq *wq = ctx->wq;
220 	struct idxd_device *idxd = wq->idxd;
221 	__poll_t out = 0;
222 
223 	poll_wait(filp, &wq->err_queue, wait);
224 	spin_lock(&idxd->dev_lock);
225 	if (idxd->sw_err.valid)
226 		out = EPOLLIN | EPOLLRDNORM;
227 	spin_unlock(&idxd->dev_lock);
228 
229 	return out;
230 }
231 
232 static const struct file_operations idxd_cdev_fops = {
233 	.owner = THIS_MODULE,
234 	.open = idxd_cdev_open,
235 	.release = idxd_cdev_release,
236 	.mmap = idxd_cdev_mmap,
237 	.poll = idxd_cdev_poll,
238 };
239 
idxd_cdev_get_major(struct idxd_device * idxd)240 int idxd_cdev_get_major(struct idxd_device *idxd)
241 {
242 	return MAJOR(ictx[idxd->data->type].devt);
243 }
244 
idxd_wq_add_cdev(struct idxd_wq * wq)245 int idxd_wq_add_cdev(struct idxd_wq *wq)
246 {
247 	struct idxd_device *idxd = wq->idxd;
248 	struct idxd_cdev *idxd_cdev;
249 	struct cdev *cdev;
250 	struct device *dev;
251 	struct idxd_cdev_context *cdev_ctx;
252 	int rc, minor;
253 
254 	idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL);
255 	if (!idxd_cdev)
256 		return -ENOMEM;
257 
258 	idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV;
259 	idxd_cdev->wq = wq;
260 	cdev = &idxd_cdev->cdev;
261 	dev = cdev_dev(idxd_cdev);
262 	cdev_ctx = &ictx[wq->idxd->data->type];
263 	minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
264 	if (minor < 0) {
265 		kfree(idxd_cdev);
266 		return minor;
267 	}
268 	idxd_cdev->minor = minor;
269 
270 	device_initialize(dev);
271 	dev->parent = wq_confdev(wq);
272 	dev->bus = &dsa_bus_type;
273 	dev->type = &idxd_cdev_device_type;
274 	dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
275 
276 	rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);
277 	if (rc < 0)
278 		goto err;
279 
280 	wq->idxd_cdev = idxd_cdev;
281 	cdev_init(cdev, &idxd_cdev_fops);
282 	rc = cdev_device_add(cdev, dev);
283 	if (rc) {
284 		dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
285 		goto err;
286 	}
287 
288 	return 0;
289 
290  err:
291 	put_device(dev);
292 	wq->idxd_cdev = NULL;
293 	return rc;
294 }
295 
idxd_wq_del_cdev(struct idxd_wq * wq)296 void idxd_wq_del_cdev(struct idxd_wq *wq)
297 {
298 	struct idxd_cdev *idxd_cdev;
299 
300 	idxd_cdev = wq->idxd_cdev;
301 	wq->idxd_cdev = NULL;
302 	cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev));
303 	put_device(cdev_dev(idxd_cdev));
304 }
305 
idxd_user_drv_probe(struct idxd_dev * idxd_dev)306 static int idxd_user_drv_probe(struct idxd_dev *idxd_dev)
307 {
308 	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
309 	struct idxd_device *idxd = wq->idxd;
310 	int rc;
311 
312 	if (idxd->state != IDXD_DEV_ENABLED)
313 		return -ENXIO;
314 
315 	/*
316 	 * User type WQ is enabled only when SVA is enabled for two reasons:
317 	 *   - If no IOMMU or IOMMU Passthrough without SVA, userspace
318 	 *     can directly access physical address through the WQ.
319 	 *   - The IDXD cdev driver does not provide any ways to pin
320 	 *     user pages and translate the address from user VA to IOVA or
321 	 *     PA without IOMMU SVA. Therefore the application has no way
322 	 *     to instruct the device to perform DMA function. This makes
323 	 *     the cdev not usable for normal application usage.
324 	 */
325 	if (!device_user_pasid_enabled(idxd)) {
326 		idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU;
327 		dev_dbg(&idxd->pdev->dev,
328 			"User type WQ cannot be enabled without SVA.\n");
329 
330 		return -EOPNOTSUPP;
331 	}
332 
333 	mutex_lock(&wq->wq_lock);
334 	wq->type = IDXD_WQT_USER;
335 	rc = drv_enable_wq(wq);
336 	if (rc < 0)
337 		goto err;
338 
339 	rc = idxd_wq_add_cdev(wq);
340 	if (rc < 0) {
341 		idxd->cmd_status = IDXD_SCMD_CDEV_ERR;
342 		goto err_cdev;
343 	}
344 
345 	idxd->cmd_status = 0;
346 	mutex_unlock(&wq->wq_lock);
347 	return 0;
348 
349 err_cdev:
350 	drv_disable_wq(wq);
351 err:
352 	wq->type = IDXD_WQT_NONE;
353 	mutex_unlock(&wq->wq_lock);
354 	return rc;
355 }
356 
idxd_user_drv_remove(struct idxd_dev * idxd_dev)357 static void idxd_user_drv_remove(struct idxd_dev *idxd_dev)
358 {
359 	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
360 
361 	mutex_lock(&wq->wq_lock);
362 	idxd_wq_del_cdev(wq);
363 	drv_disable_wq(wq);
364 	wq->type = IDXD_WQT_NONE;
365 	mutex_unlock(&wq->wq_lock);
366 }
367 
368 static enum idxd_dev_type dev_types[] = {
369 	IDXD_DEV_WQ,
370 	IDXD_DEV_NONE,
371 };
372 
373 struct idxd_device_driver idxd_user_drv = {
374 	.probe = idxd_user_drv_probe,
375 	.remove = idxd_user_drv_remove,
376 	.name = "user",
377 	.type = dev_types,
378 };
379 EXPORT_SYMBOL_GPL(idxd_user_drv);
380 
idxd_cdev_register(void)381 int idxd_cdev_register(void)
382 {
383 	int rc, i;
384 
385 	for (i = 0; i < IDXD_TYPE_MAX; i++) {
386 		ida_init(&ictx[i].minor_ida);
387 		rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
388 					 ictx[i].name);
389 		if (rc)
390 			goto err_free_chrdev_region;
391 	}
392 
393 	return 0;
394 
395 err_free_chrdev_region:
396 	for (i--; i >= 0; i--)
397 		unregister_chrdev_region(ictx[i].devt, MINORMASK);
398 
399 	return rc;
400 }
401 
idxd_cdev_remove(void)402 void idxd_cdev_remove(void)
403 {
404 	int i;
405 
406 	for (i = 0; i < IDXD_TYPE_MAX; i++) {
407 		unregister_chrdev_region(ictx[i].devt, MINORMASK);
408 		ida_destroy(&ictx[i].minor_ida);
409 	}
410 }
411