1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
4  *     Author: Alex Williamson <alex.williamson@redhat.com>
5  *
6  * Derived from original vfio:
7  * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
8  * Author: Tom Lyon, pugs@cisco.com
9  */
10 
11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12 
13 #include <linux/aperture.h>
14 #include <linux/device.h>
15 #include <linux/eventfd.h>
16 #include <linux/file.h>
17 #include <linux/interrupt.h>
18 #include <linux/iommu.h>
19 #include <linux/module.h>
20 #include <linux/mutex.h>
21 #include <linux/notifier.h>
22 #include <linux/pci.h>
23 #include <linux/pm_runtime.h>
24 #include <linux/slab.h>
25 #include <linux/types.h>
26 #include <linux/uaccess.h>
27 #include <linux/vgaarb.h>
28 #include <linux/nospec.h>
29 #include <linux/sched/mm.h>
30 #include <linux/iommufd.h>
31 #if IS_ENABLED(CONFIG_EEH)
32 #include <asm/eeh.h>
33 #endif
34 
35 #include "vfio_pci_priv.h"
36 
37 #define DRIVER_AUTHOR   "Alex Williamson <alex.williamson@redhat.com>"
38 #define DRIVER_DESC "core driver for VFIO based PCI devices"
39 
40 static bool nointxmask;
41 static bool disable_vga;
42 static bool disable_idle_d3;
43 
44 /* List of PF's that vfio_pci_core_sriov_configure() has been called on */
45 static DEFINE_MUTEX(vfio_pci_sriov_pfs_mutex);
46 static LIST_HEAD(vfio_pci_sriov_pfs);
47 
48 struct vfio_pci_dummy_resource {
49 	struct resource		resource;
50 	int			index;
51 	struct list_head	res_next;
52 };
53 
54 struct vfio_pci_vf_token {
55 	struct mutex		lock;
56 	uuid_t			uuid;
57 	int			users;
58 };
59 
60 struct vfio_pci_mmap_vma {
61 	struct vm_area_struct	*vma;
62 	struct list_head	vma_next;
63 };
64 
vfio_vga_disabled(void)65 static inline bool vfio_vga_disabled(void)
66 {
67 #ifdef CONFIG_VFIO_PCI_VGA
68 	return disable_vga;
69 #else
70 	return true;
71 #endif
72 }
73 
74 /*
75  * Our VGA arbiter participation is limited since we don't know anything
76  * about the device itself.  However, if the device is the only VGA device
77  * downstream of a bridge and VFIO VGA support is disabled, then we can
78  * safely return legacy VGA IO and memory as not decoded since the user
79  * has no way to get to it and routing can be disabled externally at the
80  * bridge.
81  */
vfio_pci_set_decode(struct pci_dev * pdev,bool single_vga)82 static unsigned int vfio_pci_set_decode(struct pci_dev *pdev, bool single_vga)
83 {
84 	struct pci_dev *tmp = NULL;
85 	unsigned char max_busnr;
86 	unsigned int decodes;
87 
88 	if (single_vga || !vfio_vga_disabled() || pci_is_root_bus(pdev->bus))
89 		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |
90 		       VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;
91 
92 	max_busnr = pci_bus_max_busnr(pdev->bus);
93 	decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
94 
95 	while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) {
96 		if (tmp == pdev ||
97 		    pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) ||
98 		    pci_is_root_bus(tmp->bus))
99 			continue;
100 
101 		if (tmp->bus->number >= pdev->bus->number &&
102 		    tmp->bus->number <= max_busnr) {
103 			pci_dev_put(tmp);
104 			decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;
105 			break;
106 		}
107 	}
108 
109 	return decodes;
110 }
111 
vfio_pci_probe_mmaps(struct vfio_pci_core_device * vdev)112 static void vfio_pci_probe_mmaps(struct vfio_pci_core_device *vdev)
113 {
114 	struct resource *res;
115 	int i;
116 	struct vfio_pci_dummy_resource *dummy_res;
117 
118 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
119 		int bar = i + PCI_STD_RESOURCES;
120 
121 		res = &vdev->pdev->resource[bar];
122 
123 		if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP))
124 			goto no_mmap;
125 
126 		if (!(res->flags & IORESOURCE_MEM))
127 			goto no_mmap;
128 
129 		/*
130 		 * The PCI core shouldn't set up a resource with a
131 		 * type but zero size. But there may be bugs that
132 		 * cause us to do that.
133 		 */
134 		if (!resource_size(res))
135 			goto no_mmap;
136 
137 		if (resource_size(res) >= PAGE_SIZE) {
138 			vdev->bar_mmap_supported[bar] = true;
139 			continue;
140 		}
141 
142 		if (!(res->start & ~PAGE_MASK)) {
143 			/*
144 			 * Add a dummy resource to reserve the remainder
145 			 * of the exclusive page in case that hot-add
146 			 * device's bar is assigned into it.
147 			 */
148 			dummy_res =
149 				kzalloc(sizeof(*dummy_res), GFP_KERNEL_ACCOUNT);
150 			if (dummy_res == NULL)
151 				goto no_mmap;
152 
153 			dummy_res->resource.name = "vfio sub-page reserved";
154 			dummy_res->resource.start = res->end + 1;
155 			dummy_res->resource.end = res->start + PAGE_SIZE - 1;
156 			dummy_res->resource.flags = res->flags;
157 			if (request_resource(res->parent,
158 						&dummy_res->resource)) {
159 				kfree(dummy_res);
160 				goto no_mmap;
161 			}
162 			dummy_res->index = bar;
163 			list_add(&dummy_res->res_next,
164 					&vdev->dummy_resources_list);
165 			vdev->bar_mmap_supported[bar] = true;
166 			continue;
167 		}
168 		/*
169 		 * Here we don't handle the case when the BAR is not page
170 		 * aligned because we can't expect the BAR will be
171 		 * assigned into the same location in a page in guest
172 		 * when we passthrough the BAR. And it's hard to access
173 		 * this BAR in userspace because we have no way to get
174 		 * the BAR's location in a page.
175 		 */
176 no_mmap:
177 		vdev->bar_mmap_supported[bar] = false;
178 	}
179 }
180 
181 struct vfio_pci_group_info;
182 static void vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set);
183 static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
184 				      struct vfio_pci_group_info *groups,
185 				      struct iommufd_ctx *iommufd_ctx);
186 
187 /*
188  * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND
189  * _and_ the ability detect when the device is asserting INTx via PCI_STATUS.
190  * If a device implements the former but not the latter we would typically
191  * expect broken_intx_masking be set and require an exclusive interrupt.
192  * However since we do have control of the device's ability to assert INTx,
193  * we can instead pretend that the device does not implement INTx, virtualizing
194  * the pin register to report zero and maintaining DisINTx set on the host.
195  */
vfio_pci_nointx(struct pci_dev * pdev)196 static bool vfio_pci_nointx(struct pci_dev *pdev)
197 {
198 	switch (pdev->vendor) {
199 	case PCI_VENDOR_ID_INTEL:
200 		switch (pdev->device) {
201 		/* All i40e (XL710/X710/XXV710) 10/20/25/40GbE NICs */
202 		case 0x1572:
203 		case 0x1574:
204 		case 0x1580 ... 0x1581:
205 		case 0x1583 ... 0x158b:
206 		case 0x37d0 ... 0x37d2:
207 		/* X550 */
208 		case 0x1563:
209 			return true;
210 		default:
211 			return false;
212 		}
213 	}
214 
215 	return false;
216 }
217 
vfio_pci_probe_power_state(struct vfio_pci_core_device * vdev)218 static void vfio_pci_probe_power_state(struct vfio_pci_core_device *vdev)
219 {
220 	struct pci_dev *pdev = vdev->pdev;
221 	u16 pmcsr;
222 
223 	if (!pdev->pm_cap)
224 		return;
225 
226 	pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
227 
228 	vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
229 }
230 
231 /*
232  * pci_set_power_state() wrapper handling devices which perform a soft reset on
233  * D3->D0 transition.  Save state prior to D0/1/2->D3, stash it on the vdev,
234  * restore when returned to D0.  Saved separately from pci_saved_state for use
235  * by PM capability emulation and separately from pci_dev internal saved state
236  * to avoid it being overwritten and consumed around other resets.
237  */
vfio_pci_set_power_state(struct vfio_pci_core_device * vdev,pci_power_t state)238 int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t state)
239 {
240 	struct pci_dev *pdev = vdev->pdev;
241 	bool needs_restore = false, needs_save = false;
242 	int ret;
243 
244 	/* Prevent changing power state for PFs with VFs enabled */
245 	if (pci_num_vf(pdev) && state > PCI_D0)
246 		return -EBUSY;
247 
248 	if (vdev->needs_pm_restore) {
249 		if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) {
250 			pci_save_state(pdev);
251 			needs_save = true;
252 		}
253 
254 		if (pdev->current_state >= PCI_D3hot && state <= PCI_D0)
255 			needs_restore = true;
256 	}
257 
258 	ret = pci_set_power_state(pdev, state);
259 
260 	if (!ret) {
261 		/* D3 might be unsupported via quirk, skip unless in D3 */
262 		if (needs_save && pdev->current_state >= PCI_D3hot) {
263 			/*
264 			 * The current PCI state will be saved locally in
265 			 * 'pm_save' during the D3hot transition. When the
266 			 * device state is changed to D0 again with the current
267 			 * function, then pci_store_saved_state() will restore
268 			 * the state and will free the memory pointed by
269 			 * 'pm_save'. There are few cases where the PCI power
270 			 * state can be changed to D0 without the involvement
271 			 * of the driver. For these cases, free the earlier
272 			 * allocated memory first before overwriting 'pm_save'
273 			 * to prevent the memory leak.
274 			 */
275 			kfree(vdev->pm_save);
276 			vdev->pm_save = pci_store_saved_state(pdev);
277 		} else if (needs_restore) {
278 			pci_load_and_free_saved_state(pdev, &vdev->pm_save);
279 			pci_restore_state(pdev);
280 		}
281 	}
282 
283 	return ret;
284 }
285 
vfio_pci_runtime_pm_entry(struct vfio_pci_core_device * vdev,struct eventfd_ctx * efdctx)286 static int vfio_pci_runtime_pm_entry(struct vfio_pci_core_device *vdev,
287 				     struct eventfd_ctx *efdctx)
288 {
289 	/*
290 	 * The vdev power related flags are protected with 'memory_lock'
291 	 * semaphore.
292 	 */
293 	vfio_pci_zap_and_down_write_memory_lock(vdev);
294 	if (vdev->pm_runtime_engaged) {
295 		up_write(&vdev->memory_lock);
296 		return -EINVAL;
297 	}
298 
299 	vdev->pm_runtime_engaged = true;
300 	vdev->pm_wake_eventfd_ctx = efdctx;
301 	pm_runtime_put_noidle(&vdev->pdev->dev);
302 	up_write(&vdev->memory_lock);
303 
304 	return 0;
305 }
306 
vfio_pci_core_pm_entry(struct vfio_device * device,u32 flags,void __user * arg,size_t argsz)307 static int vfio_pci_core_pm_entry(struct vfio_device *device, u32 flags,
308 				  void __user *arg, size_t argsz)
309 {
310 	struct vfio_pci_core_device *vdev =
311 		container_of(device, struct vfio_pci_core_device, vdev);
312 	int ret;
313 
314 	ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0);
315 	if (ret != 1)
316 		return ret;
317 
318 	/*
319 	 * Inside vfio_pci_runtime_pm_entry(), only the runtime PM usage count
320 	 * will be decremented. The pm_runtime_put() will be invoked again
321 	 * while returning from the ioctl and then the device can go into
322 	 * runtime suspended state.
323 	 */
324 	return vfio_pci_runtime_pm_entry(vdev, NULL);
325 }
326 
vfio_pci_core_pm_entry_with_wakeup(struct vfio_device * device,u32 flags,struct vfio_device_low_power_entry_with_wakeup __user * arg,size_t argsz)327 static int vfio_pci_core_pm_entry_with_wakeup(
328 	struct vfio_device *device, u32 flags,
329 	struct vfio_device_low_power_entry_with_wakeup __user *arg,
330 	size_t argsz)
331 {
332 	struct vfio_pci_core_device *vdev =
333 		container_of(device, struct vfio_pci_core_device, vdev);
334 	struct vfio_device_low_power_entry_with_wakeup entry;
335 	struct eventfd_ctx *efdctx;
336 	int ret;
337 
338 	ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET,
339 				 sizeof(entry));
340 	if (ret != 1)
341 		return ret;
342 
343 	if (copy_from_user(&entry, arg, sizeof(entry)))
344 		return -EFAULT;
345 
346 	if (entry.wakeup_eventfd < 0)
347 		return -EINVAL;
348 
349 	efdctx = eventfd_ctx_fdget(entry.wakeup_eventfd);
350 	if (IS_ERR(efdctx))
351 		return PTR_ERR(efdctx);
352 
353 	ret = vfio_pci_runtime_pm_entry(vdev, efdctx);
354 	if (ret)
355 		eventfd_ctx_put(efdctx);
356 
357 	return ret;
358 }
359 
__vfio_pci_runtime_pm_exit(struct vfio_pci_core_device * vdev)360 static void __vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev)
361 {
362 	if (vdev->pm_runtime_engaged) {
363 		vdev->pm_runtime_engaged = false;
364 		pm_runtime_get_noresume(&vdev->pdev->dev);
365 
366 		if (vdev->pm_wake_eventfd_ctx) {
367 			eventfd_ctx_put(vdev->pm_wake_eventfd_ctx);
368 			vdev->pm_wake_eventfd_ctx = NULL;
369 		}
370 	}
371 }
372 
vfio_pci_runtime_pm_exit(struct vfio_pci_core_device * vdev)373 static void vfio_pci_runtime_pm_exit(struct vfio_pci_core_device *vdev)
374 {
375 	/*
376 	 * The vdev power related flags are protected with 'memory_lock'
377 	 * semaphore.
378 	 */
379 	down_write(&vdev->memory_lock);
380 	__vfio_pci_runtime_pm_exit(vdev);
381 	up_write(&vdev->memory_lock);
382 }
383 
vfio_pci_core_pm_exit(struct vfio_device * device,u32 flags,void __user * arg,size_t argsz)384 static int vfio_pci_core_pm_exit(struct vfio_device *device, u32 flags,
385 				 void __user *arg, size_t argsz)
386 {
387 	struct vfio_pci_core_device *vdev =
388 		container_of(device, struct vfio_pci_core_device, vdev);
389 	int ret;
390 
391 	ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET, 0);
392 	if (ret != 1)
393 		return ret;
394 
395 	/*
396 	 * The device is always in the active state here due to pm wrappers
397 	 * around ioctls. If the device had entered a low power state and
398 	 * pm_wake_eventfd_ctx is valid, vfio_pci_core_runtime_resume() has
399 	 * already signaled the eventfd and exited low power mode itself.
400 	 * pm_runtime_engaged protects the redundant call here.
401 	 */
402 	vfio_pci_runtime_pm_exit(vdev);
403 	return 0;
404 }
405 
406 #ifdef CONFIG_PM
vfio_pci_core_runtime_suspend(struct device * dev)407 static int vfio_pci_core_runtime_suspend(struct device *dev)
408 {
409 	struct vfio_pci_core_device *vdev = dev_get_drvdata(dev);
410 
411 	down_write(&vdev->memory_lock);
412 	/*
413 	 * The user can move the device into D3hot state before invoking
414 	 * power management IOCTL. Move the device into D0 state here and then
415 	 * the pci-driver core runtime PM suspend function will move the device
416 	 * into the low power state. Also, for the devices which have
417 	 * NoSoftRst-, it will help in restoring the original state
418 	 * (saved locally in 'vdev->pm_save').
419 	 */
420 	vfio_pci_set_power_state(vdev, PCI_D0);
421 	up_write(&vdev->memory_lock);
422 
423 	/*
424 	 * If INTx is enabled, then mask INTx before going into the runtime
425 	 * suspended state and unmask the same in the runtime resume.
426 	 * If INTx has already been masked by the user, then
427 	 * vfio_pci_intx_mask() will return false and in that case, INTx
428 	 * should not be unmasked in the runtime resume.
429 	 */
430 	vdev->pm_intx_masked = ((vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX) &&
431 				vfio_pci_intx_mask(vdev));
432 
433 	return 0;
434 }
435 
vfio_pci_core_runtime_resume(struct device * dev)436 static int vfio_pci_core_runtime_resume(struct device *dev)
437 {
438 	struct vfio_pci_core_device *vdev = dev_get_drvdata(dev);
439 
440 	/*
441 	 * Resume with a pm_wake_eventfd_ctx signals the eventfd and exit
442 	 * low power mode.
443 	 */
444 	down_write(&vdev->memory_lock);
445 	if (vdev->pm_wake_eventfd_ctx) {
446 		eventfd_signal(vdev->pm_wake_eventfd_ctx, 1);
447 		__vfio_pci_runtime_pm_exit(vdev);
448 	}
449 	up_write(&vdev->memory_lock);
450 
451 	if (vdev->pm_intx_masked)
452 		vfio_pci_intx_unmask(vdev);
453 
454 	return 0;
455 }
456 #endif /* CONFIG_PM */
457 
458 /*
459  * The pci-driver core runtime PM routines always save the device state
460  * before going into suspended state. If the device is going into low power
461  * state with only with runtime PM ops, then no explicit handling is needed
462  * for the devices which have NoSoftRst-.
463  */
464 static const struct dev_pm_ops vfio_pci_core_pm_ops = {
465 	SET_RUNTIME_PM_OPS(vfio_pci_core_runtime_suspend,
466 			   vfio_pci_core_runtime_resume,
467 			   NULL)
468 };
469 
vfio_pci_core_enable(struct vfio_pci_core_device * vdev)470 int vfio_pci_core_enable(struct vfio_pci_core_device *vdev)
471 {
472 	struct pci_dev *pdev = vdev->pdev;
473 	int ret;
474 	u16 cmd;
475 	u8 msix_pos;
476 
477 	if (!disable_idle_d3) {
478 		ret = pm_runtime_resume_and_get(&pdev->dev);
479 		if (ret < 0)
480 			return ret;
481 	}
482 
483 	/* Don't allow our initial saved state to include busmaster */
484 	pci_clear_master(pdev);
485 
486 	ret = pci_enable_device(pdev);
487 	if (ret)
488 		goto out_power;
489 
490 	/* If reset fails because of the device lock, fail this path entirely */
491 	ret = pci_try_reset_function(pdev);
492 	if (ret == -EAGAIN)
493 		goto out_disable_device;
494 
495 	vdev->reset_works = !ret;
496 	pci_save_state(pdev);
497 	vdev->pci_saved_state = pci_store_saved_state(pdev);
498 	if (!vdev->pci_saved_state)
499 		pci_dbg(pdev, "%s: Couldn't store saved state\n", __func__);
500 
501 	if (likely(!nointxmask)) {
502 		if (vfio_pci_nointx(pdev)) {
503 			pci_info(pdev, "Masking broken INTx support\n");
504 			vdev->nointx = true;
505 			pci_intx(pdev, 0);
506 		} else
507 			vdev->pci_2_3 = pci_intx_mask_supported(pdev);
508 	}
509 
510 	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
511 	if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {
512 		cmd &= ~PCI_COMMAND_INTX_DISABLE;
513 		pci_write_config_word(pdev, PCI_COMMAND, cmd);
514 	}
515 
516 	ret = vfio_pci_zdev_open_device(vdev);
517 	if (ret)
518 		goto out_free_state;
519 
520 	ret = vfio_config_init(vdev);
521 	if (ret)
522 		goto out_free_zdev;
523 
524 	msix_pos = pdev->msix_cap;
525 	if (msix_pos) {
526 		u16 flags;
527 		u32 table;
528 
529 		pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);
530 		pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);
531 
532 		vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;
533 		vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;
534 		vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;
535 		vdev->has_dyn_msix = pci_msix_can_alloc_dyn(pdev);
536 	} else {
537 		vdev->msix_bar = 0xFF;
538 		vdev->has_dyn_msix = false;
539 	}
540 
541 	if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))
542 		vdev->has_vga = true;
543 
544 
545 	return 0;
546 
547 out_free_zdev:
548 	vfio_pci_zdev_close_device(vdev);
549 out_free_state:
550 	kfree(vdev->pci_saved_state);
551 	vdev->pci_saved_state = NULL;
552 out_disable_device:
553 	pci_disable_device(pdev);
554 out_power:
555 	if (!disable_idle_d3)
556 		pm_runtime_put(&pdev->dev);
557 	return ret;
558 }
559 EXPORT_SYMBOL_GPL(vfio_pci_core_enable);
560 
vfio_pci_core_disable(struct vfio_pci_core_device * vdev)561 void vfio_pci_core_disable(struct vfio_pci_core_device *vdev)
562 {
563 	struct pci_dev *pdev = vdev->pdev;
564 	struct vfio_pci_dummy_resource *dummy_res, *tmp;
565 	struct vfio_pci_ioeventfd *ioeventfd, *ioeventfd_tmp;
566 	int i, bar;
567 
568 	/* For needs_reset */
569 	lockdep_assert_held(&vdev->vdev.dev_set->lock);
570 
571 	/*
572 	 * This function can be invoked while the power state is non-D0.
573 	 * This non-D0 power state can be with or without runtime PM.
574 	 * vfio_pci_runtime_pm_exit() will internally increment the usage
575 	 * count corresponding to pm_runtime_put() called during low power
576 	 * feature entry and then pm_runtime_resume() will wake up the device,
577 	 * if the device has already gone into the suspended state. Otherwise,
578 	 * the vfio_pci_set_power_state() will change the device power state
579 	 * to D0.
580 	 */
581 	vfio_pci_runtime_pm_exit(vdev);
582 	pm_runtime_resume(&pdev->dev);
583 
584 	/*
585 	 * This function calls __pci_reset_function_locked() which internally
586 	 * can use pci_pm_reset() for the function reset. pci_pm_reset() will
587 	 * fail if the power state is non-D0. Also, for the devices which
588 	 * have NoSoftRst-, the reset function can cause the PCI config space
589 	 * reset without restoring the original state (saved locally in
590 	 * 'vdev->pm_save').
591 	 */
592 	vfio_pci_set_power_state(vdev, PCI_D0);
593 
594 	/* Stop the device from further DMA */
595 	pci_clear_master(pdev);
596 
597 	vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE |
598 				VFIO_IRQ_SET_ACTION_TRIGGER,
599 				vdev->irq_type, 0, 0, NULL);
600 
601 	/* Device closed, don't need mutex here */
602 	list_for_each_entry_safe(ioeventfd, ioeventfd_tmp,
603 				 &vdev->ioeventfds_list, next) {
604 		vfio_virqfd_disable(&ioeventfd->virqfd);
605 		list_del(&ioeventfd->next);
606 		kfree(ioeventfd);
607 	}
608 	vdev->ioeventfds_nr = 0;
609 
610 	vdev->virq_disabled = false;
611 
612 	for (i = 0; i < vdev->num_regions; i++)
613 		vdev->region[i].ops->release(vdev, &vdev->region[i]);
614 
615 	vdev->num_regions = 0;
616 	kfree(vdev->region);
617 	vdev->region = NULL; /* don't krealloc a freed pointer */
618 
619 	vfio_config_free(vdev);
620 
621 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
622 		bar = i + PCI_STD_RESOURCES;
623 		if (!vdev->barmap[bar])
624 			continue;
625 		pci_iounmap(pdev, vdev->barmap[bar]);
626 		pci_release_selected_regions(pdev, 1 << bar);
627 		vdev->barmap[bar] = NULL;
628 	}
629 
630 	list_for_each_entry_safe(dummy_res, tmp,
631 				 &vdev->dummy_resources_list, res_next) {
632 		list_del(&dummy_res->res_next);
633 		release_resource(&dummy_res->resource);
634 		kfree(dummy_res);
635 	}
636 
637 	vdev->needs_reset = true;
638 
639 	vfio_pci_zdev_close_device(vdev);
640 
641 	/*
642 	 * If we have saved state, restore it.  If we can reset the device,
643 	 * even better.  Resetting with current state seems better than
644 	 * nothing, but saving and restoring current state without reset
645 	 * is just busy work.
646 	 */
647 	if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {
648 		pci_info(pdev, "%s: Couldn't reload saved state\n", __func__);
649 
650 		if (!vdev->reset_works)
651 			goto out;
652 
653 		pci_save_state(pdev);
654 	}
655 
656 	/*
657 	 * Disable INTx and MSI, presumably to avoid spurious interrupts
658 	 * during reset.  Stolen from pci_reset_function()
659 	 */
660 	pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
661 
662 	/*
663 	 * Try to get the locks ourselves to prevent a deadlock. The
664 	 * success of this is dependent on being able to lock the device,
665 	 * which is not always possible.
666 	 * We can not use the "try" reset interface here, which will
667 	 * overwrite the previously restored configuration information.
668 	 */
669 	if (vdev->reset_works && pci_dev_trylock(pdev)) {
670 		if (!__pci_reset_function_locked(pdev))
671 			vdev->needs_reset = false;
672 		pci_dev_unlock(pdev);
673 	}
674 
675 	pci_restore_state(pdev);
676 out:
677 	pci_disable_device(pdev);
678 
679 	vfio_pci_dev_set_try_reset(vdev->vdev.dev_set);
680 
681 	/* Put the pm-runtime usage counter acquired during enable */
682 	if (!disable_idle_d3)
683 		pm_runtime_put(&pdev->dev);
684 }
685 EXPORT_SYMBOL_GPL(vfio_pci_core_disable);
686 
vfio_pci_core_close_device(struct vfio_device * core_vdev)687 void vfio_pci_core_close_device(struct vfio_device *core_vdev)
688 {
689 	struct vfio_pci_core_device *vdev =
690 		container_of(core_vdev, struct vfio_pci_core_device, vdev);
691 
692 	if (vdev->sriov_pf_core_dev) {
693 		mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock);
694 		WARN_ON(!vdev->sriov_pf_core_dev->vf_token->users);
695 		vdev->sriov_pf_core_dev->vf_token->users--;
696 		mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock);
697 	}
698 #if IS_ENABLED(CONFIG_EEH)
699 	eeh_dev_release(vdev->pdev);
700 #endif
701 	vfio_pci_core_disable(vdev);
702 
703 	mutex_lock(&vdev->igate);
704 	if (vdev->err_trigger) {
705 		eventfd_ctx_put(vdev->err_trigger);
706 		vdev->err_trigger = NULL;
707 	}
708 	if (vdev->req_trigger) {
709 		eventfd_ctx_put(vdev->req_trigger);
710 		vdev->req_trigger = NULL;
711 	}
712 	mutex_unlock(&vdev->igate);
713 }
714 EXPORT_SYMBOL_GPL(vfio_pci_core_close_device);
715 
vfio_pci_core_finish_enable(struct vfio_pci_core_device * vdev)716 void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev)
717 {
718 	vfio_pci_probe_mmaps(vdev);
719 #if IS_ENABLED(CONFIG_EEH)
720 	eeh_dev_open(vdev->pdev);
721 #endif
722 
723 	if (vdev->sriov_pf_core_dev) {
724 		mutex_lock(&vdev->sriov_pf_core_dev->vf_token->lock);
725 		vdev->sriov_pf_core_dev->vf_token->users++;
726 		mutex_unlock(&vdev->sriov_pf_core_dev->vf_token->lock);
727 	}
728 }
729 EXPORT_SYMBOL_GPL(vfio_pci_core_finish_enable);
730 
vfio_pci_get_irq_count(struct vfio_pci_core_device * vdev,int irq_type)731 static int vfio_pci_get_irq_count(struct vfio_pci_core_device *vdev, int irq_type)
732 {
733 	if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) {
734 		u8 pin;
735 
736 		if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) ||
737 		    vdev->nointx || vdev->pdev->is_virtfn)
738 			return 0;
739 
740 		pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);
741 
742 		return pin ? 1 : 0;
743 	} else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) {
744 		u8 pos;
745 		u16 flags;
746 
747 		pos = vdev->pdev->msi_cap;
748 		if (pos) {
749 			pci_read_config_word(vdev->pdev,
750 					     pos + PCI_MSI_FLAGS, &flags);
751 			return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1);
752 		}
753 	} else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) {
754 		u8 pos;
755 		u16 flags;
756 
757 		pos = vdev->pdev->msix_cap;
758 		if (pos) {
759 			pci_read_config_word(vdev->pdev,
760 					     pos + PCI_MSIX_FLAGS, &flags);
761 
762 			return (flags & PCI_MSIX_FLAGS_QSIZE) + 1;
763 		}
764 	} else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) {
765 		if (pci_is_pcie(vdev->pdev))
766 			return 1;
767 	} else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) {
768 		return 1;
769 	}
770 
771 	return 0;
772 }
773 
vfio_pci_count_devs(struct pci_dev * pdev,void * data)774 static int vfio_pci_count_devs(struct pci_dev *pdev, void *data)
775 {
776 	(*(int *)data)++;
777 	return 0;
778 }
779 
780 struct vfio_pci_fill_info {
781 	struct vfio_pci_dependent_device __user *devices;
782 	struct vfio_pci_dependent_device __user *devices_end;
783 	struct vfio_device *vdev;
784 	u32 count;
785 	u32 flags;
786 };
787 
vfio_pci_fill_devs(struct pci_dev * pdev,void * data)788 static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data)
789 {
790 	struct vfio_pci_dependent_device info = {
791 		.segment = pci_domain_nr(pdev->bus),
792 		.bus = pdev->bus->number,
793 		.devfn = pdev->devfn,
794 	};
795 	struct vfio_pci_fill_info *fill = data;
796 
797 	fill->count++;
798 	if (fill->devices >= fill->devices_end)
799 		return 0;
800 
801 	if (fill->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID) {
802 		struct iommufd_ctx *iommufd = vfio_iommufd_device_ictx(fill->vdev);
803 		struct vfio_device_set *dev_set = fill->vdev->dev_set;
804 		struct vfio_device *vdev;
805 
806 		/*
807 		 * hot-reset requires all affected devices be represented in
808 		 * the dev_set.
809 		 */
810 		vdev = vfio_find_device_in_devset(dev_set, &pdev->dev);
811 		if (!vdev) {
812 			info.devid = VFIO_PCI_DEVID_NOT_OWNED;
813 		} else {
814 			int id = vfio_iommufd_get_dev_id(vdev, iommufd);
815 
816 			if (id > 0)
817 				info.devid = id;
818 			else if (id == -ENOENT)
819 				info.devid = VFIO_PCI_DEVID_OWNED;
820 			else
821 				info.devid = VFIO_PCI_DEVID_NOT_OWNED;
822 		}
823 		/* If devid is VFIO_PCI_DEVID_NOT_OWNED, clear owned flag. */
824 		if (info.devid == VFIO_PCI_DEVID_NOT_OWNED)
825 			fill->flags &= ~VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED;
826 	} else {
827 		struct iommu_group *iommu_group;
828 
829 		iommu_group = iommu_group_get(&pdev->dev);
830 		if (!iommu_group)
831 			return -EPERM; /* Cannot reset non-isolated devices */
832 
833 		info.group_id = iommu_group_id(iommu_group);
834 		iommu_group_put(iommu_group);
835 	}
836 
837 	if (copy_to_user(fill->devices, &info, sizeof(info)))
838 		return -EFAULT;
839 	fill->devices++;
840 	return 0;
841 }
842 
843 struct vfio_pci_group_info {
844 	int count;
845 	struct file **files;
846 };
847 
vfio_pci_dev_below_slot(struct pci_dev * pdev,struct pci_slot * slot)848 static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot)
849 {
850 	for (; pdev; pdev = pdev->bus->self)
851 		if (pdev->bus == slot->bus)
852 			return (pdev->slot == slot);
853 	return false;
854 }
855 
856 struct vfio_pci_walk_info {
857 	int (*fn)(struct pci_dev *pdev, void *data);
858 	void *data;
859 	struct pci_dev *pdev;
860 	bool slot;
861 	int ret;
862 };
863 
vfio_pci_walk_wrapper(struct pci_dev * pdev,void * data)864 static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data)
865 {
866 	struct vfio_pci_walk_info *walk = data;
867 
868 	if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot))
869 		walk->ret = walk->fn(pdev, walk->data);
870 
871 	return walk->ret;
872 }
873 
vfio_pci_for_each_slot_or_bus(struct pci_dev * pdev,int (* fn)(struct pci_dev *,void * data),void * data,bool slot)874 static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,
875 					 int (*fn)(struct pci_dev *,
876 						   void *data), void *data,
877 					 bool slot)
878 {
879 	struct vfio_pci_walk_info walk = {
880 		.fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0,
881 	};
882 
883 	pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk);
884 
885 	return walk.ret;
886 }
887 
msix_mmappable_cap(struct vfio_pci_core_device * vdev,struct vfio_info_cap * caps)888 static int msix_mmappable_cap(struct vfio_pci_core_device *vdev,
889 			      struct vfio_info_cap *caps)
890 {
891 	struct vfio_info_cap_header header = {
892 		.id = VFIO_REGION_INFO_CAP_MSIX_MAPPABLE,
893 		.version = 1
894 	};
895 
896 	return vfio_info_add_capability(caps, &header, sizeof(header));
897 }
898 
vfio_pci_core_register_dev_region(struct vfio_pci_core_device * vdev,unsigned int type,unsigned int subtype,const struct vfio_pci_regops * ops,size_t size,u32 flags,void * data)899 int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev,
900 				      unsigned int type, unsigned int subtype,
901 				      const struct vfio_pci_regops *ops,
902 				      size_t size, u32 flags, void *data)
903 {
904 	struct vfio_pci_region *region;
905 
906 	region = krealloc(vdev->region,
907 			  (vdev->num_regions + 1) * sizeof(*region),
908 			  GFP_KERNEL_ACCOUNT);
909 	if (!region)
910 		return -ENOMEM;
911 
912 	vdev->region = region;
913 	vdev->region[vdev->num_regions].type = type;
914 	vdev->region[vdev->num_regions].subtype = subtype;
915 	vdev->region[vdev->num_regions].ops = ops;
916 	vdev->region[vdev->num_regions].size = size;
917 	vdev->region[vdev->num_regions].flags = flags;
918 	vdev->region[vdev->num_regions].data = data;
919 
920 	vdev->num_regions++;
921 
922 	return 0;
923 }
924 EXPORT_SYMBOL_GPL(vfio_pci_core_register_dev_region);
925 
vfio_pci_info_atomic_cap(struct vfio_pci_core_device * vdev,struct vfio_info_cap * caps)926 static int vfio_pci_info_atomic_cap(struct vfio_pci_core_device *vdev,
927 				    struct vfio_info_cap *caps)
928 {
929 	struct vfio_device_info_cap_pci_atomic_comp cap = {
930 		.header.id = VFIO_DEVICE_INFO_CAP_PCI_ATOMIC_COMP,
931 		.header.version = 1
932 	};
933 	struct pci_dev *pdev = pci_physfn(vdev->pdev);
934 	u32 devcap2;
935 
936 	pcie_capability_read_dword(pdev, PCI_EXP_DEVCAP2, &devcap2);
937 
938 	if ((devcap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP32) &&
939 	    !pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32))
940 		cap.flags |= VFIO_PCI_ATOMIC_COMP32;
941 
942 	if ((devcap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP64) &&
943 	    !pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64))
944 		cap.flags |= VFIO_PCI_ATOMIC_COMP64;
945 
946 	if ((devcap2 & PCI_EXP_DEVCAP2_ATOMIC_COMP128) &&
947 	    !pci_enable_atomic_ops_to_root(pdev,
948 					   PCI_EXP_DEVCAP2_ATOMIC_COMP128))
949 		cap.flags |= VFIO_PCI_ATOMIC_COMP128;
950 
951 	if (!cap.flags)
952 		return -ENODEV;
953 
954 	return vfio_info_add_capability(caps, &cap.header, sizeof(cap));
955 }
956 
vfio_pci_ioctl_get_info(struct vfio_pci_core_device * vdev,struct vfio_device_info __user * arg)957 static int vfio_pci_ioctl_get_info(struct vfio_pci_core_device *vdev,
958 				   struct vfio_device_info __user *arg)
959 {
960 	unsigned long minsz = offsetofend(struct vfio_device_info, num_irqs);
961 	struct vfio_device_info info = {};
962 	struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
963 	int ret;
964 
965 	if (copy_from_user(&info, arg, minsz))
966 		return -EFAULT;
967 
968 	if (info.argsz < minsz)
969 		return -EINVAL;
970 
971 	minsz = min_t(size_t, info.argsz, sizeof(info));
972 
973 	info.flags = VFIO_DEVICE_FLAGS_PCI;
974 
975 	if (vdev->reset_works)
976 		info.flags |= VFIO_DEVICE_FLAGS_RESET;
977 
978 	info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
979 	info.num_irqs = VFIO_PCI_NUM_IRQS;
980 
981 	ret = vfio_pci_info_zdev_add_caps(vdev, &caps);
982 	if (ret && ret != -ENODEV) {
983 		pci_warn(vdev->pdev,
984 			 "Failed to setup zPCI info capabilities\n");
985 		return ret;
986 	}
987 
988 	ret = vfio_pci_info_atomic_cap(vdev, &caps);
989 	if (ret && ret != -ENODEV) {
990 		pci_warn(vdev->pdev,
991 			 "Failed to setup AtomicOps info capability\n");
992 		return ret;
993 	}
994 
995 	if (caps.size) {
996 		info.flags |= VFIO_DEVICE_FLAGS_CAPS;
997 		if (info.argsz < sizeof(info) + caps.size) {
998 			info.argsz = sizeof(info) + caps.size;
999 		} else {
1000 			vfio_info_cap_shift(&caps, sizeof(info));
1001 			if (copy_to_user(arg + 1, caps.buf, caps.size)) {
1002 				kfree(caps.buf);
1003 				return -EFAULT;
1004 			}
1005 			info.cap_offset = sizeof(*arg);
1006 		}
1007 
1008 		kfree(caps.buf);
1009 	}
1010 
1011 	return copy_to_user(arg, &info, minsz) ? -EFAULT : 0;
1012 }
1013 
vfio_pci_ioctl_get_region_info(struct vfio_pci_core_device * vdev,struct vfio_region_info __user * arg)1014 static int vfio_pci_ioctl_get_region_info(struct vfio_pci_core_device *vdev,
1015 					  struct vfio_region_info __user *arg)
1016 {
1017 	unsigned long minsz = offsetofend(struct vfio_region_info, offset);
1018 	struct pci_dev *pdev = vdev->pdev;
1019 	struct vfio_region_info info;
1020 	struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
1021 	int i, ret;
1022 
1023 	if (copy_from_user(&info, arg, minsz))
1024 		return -EFAULT;
1025 
1026 	if (info.argsz < minsz)
1027 		return -EINVAL;
1028 
1029 	switch (info.index) {
1030 	case VFIO_PCI_CONFIG_REGION_INDEX:
1031 		info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1032 		info.size = pdev->cfg_size;
1033 		info.flags = VFIO_REGION_INFO_FLAG_READ |
1034 			     VFIO_REGION_INFO_FLAG_WRITE;
1035 		break;
1036 	case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1037 		info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1038 		info.size = pci_resource_len(pdev, info.index);
1039 		if (!info.size) {
1040 			info.flags = 0;
1041 			break;
1042 		}
1043 
1044 		info.flags = VFIO_REGION_INFO_FLAG_READ |
1045 			     VFIO_REGION_INFO_FLAG_WRITE;
1046 		if (vdev->bar_mmap_supported[info.index]) {
1047 			info.flags |= VFIO_REGION_INFO_FLAG_MMAP;
1048 			if (info.index == vdev->msix_bar) {
1049 				ret = msix_mmappable_cap(vdev, &caps);
1050 				if (ret)
1051 					return ret;
1052 			}
1053 		}
1054 
1055 		break;
1056 	case VFIO_PCI_ROM_REGION_INDEX: {
1057 		void __iomem *io;
1058 		size_t size;
1059 		u16 cmd;
1060 
1061 		info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1062 		info.flags = 0;
1063 
1064 		/* Report the BAR size, not the ROM size */
1065 		info.size = pci_resource_len(pdev, info.index);
1066 		if (!info.size) {
1067 			/* Shadow ROMs appear as PCI option ROMs */
1068 			if (pdev->resource[PCI_ROM_RESOURCE].flags &
1069 			    IORESOURCE_ROM_SHADOW)
1070 				info.size = 0x20000;
1071 			else
1072 				break;
1073 		}
1074 
1075 		/*
1076 		 * Is it really there?  Enable memory decode for implicit access
1077 		 * in pci_map_rom().
1078 		 */
1079 		cmd = vfio_pci_memory_lock_and_enable(vdev);
1080 		io = pci_map_rom(pdev, &size);
1081 		if (io) {
1082 			info.flags = VFIO_REGION_INFO_FLAG_READ;
1083 			pci_unmap_rom(pdev, io);
1084 		} else {
1085 			info.size = 0;
1086 		}
1087 		vfio_pci_memory_unlock_and_restore(vdev, cmd);
1088 
1089 		break;
1090 	}
1091 	case VFIO_PCI_VGA_REGION_INDEX:
1092 		if (!vdev->has_vga)
1093 			return -EINVAL;
1094 
1095 		info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1096 		info.size = 0xc0000;
1097 		info.flags = VFIO_REGION_INFO_FLAG_READ |
1098 			     VFIO_REGION_INFO_FLAG_WRITE;
1099 
1100 		break;
1101 	default: {
1102 		struct vfio_region_info_cap_type cap_type = {
1103 			.header.id = VFIO_REGION_INFO_CAP_TYPE,
1104 			.header.version = 1
1105 		};
1106 
1107 		if (info.index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1108 			return -EINVAL;
1109 		info.index = array_index_nospec(
1110 			info.index, VFIO_PCI_NUM_REGIONS + vdev->num_regions);
1111 
1112 		i = info.index - VFIO_PCI_NUM_REGIONS;
1113 
1114 		info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1115 		info.size = vdev->region[i].size;
1116 		info.flags = vdev->region[i].flags;
1117 
1118 		cap_type.type = vdev->region[i].type;
1119 		cap_type.subtype = vdev->region[i].subtype;
1120 
1121 		ret = vfio_info_add_capability(&caps, &cap_type.header,
1122 					       sizeof(cap_type));
1123 		if (ret)
1124 			return ret;
1125 
1126 		if (vdev->region[i].ops->add_capability) {
1127 			ret = vdev->region[i].ops->add_capability(
1128 				vdev, &vdev->region[i], &caps);
1129 			if (ret)
1130 				return ret;
1131 		}
1132 	}
1133 	}
1134 
1135 	if (caps.size) {
1136 		info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
1137 		if (info.argsz < sizeof(info) + caps.size) {
1138 			info.argsz = sizeof(info) + caps.size;
1139 			info.cap_offset = 0;
1140 		} else {
1141 			vfio_info_cap_shift(&caps, sizeof(info));
1142 			if (copy_to_user(arg + 1, caps.buf, caps.size)) {
1143 				kfree(caps.buf);
1144 				return -EFAULT;
1145 			}
1146 			info.cap_offset = sizeof(*arg);
1147 		}
1148 
1149 		kfree(caps.buf);
1150 	}
1151 
1152 	return copy_to_user(arg, &info, minsz) ? -EFAULT : 0;
1153 }
1154 
vfio_pci_ioctl_get_irq_info(struct vfio_pci_core_device * vdev,struct vfio_irq_info __user * arg)1155 static int vfio_pci_ioctl_get_irq_info(struct vfio_pci_core_device *vdev,
1156 				       struct vfio_irq_info __user *arg)
1157 {
1158 	unsigned long minsz = offsetofend(struct vfio_irq_info, count);
1159 	struct vfio_irq_info info;
1160 
1161 	if (copy_from_user(&info, arg, minsz))
1162 		return -EFAULT;
1163 
1164 	if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
1165 		return -EINVAL;
1166 
1167 	switch (info.index) {
1168 	case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:
1169 	case VFIO_PCI_REQ_IRQ_INDEX:
1170 		break;
1171 	case VFIO_PCI_ERR_IRQ_INDEX:
1172 		if (pci_is_pcie(vdev->pdev))
1173 			break;
1174 		fallthrough;
1175 	default:
1176 		return -EINVAL;
1177 	}
1178 
1179 	info.flags = VFIO_IRQ_INFO_EVENTFD;
1180 
1181 	info.count = vfio_pci_get_irq_count(vdev, info.index);
1182 
1183 	if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
1184 		info.flags |=
1185 			(VFIO_IRQ_INFO_MASKABLE | VFIO_IRQ_INFO_AUTOMASKED);
1186 	else if (info.index != VFIO_PCI_MSIX_IRQ_INDEX || !vdev->has_dyn_msix)
1187 		info.flags |= VFIO_IRQ_INFO_NORESIZE;
1188 
1189 	return copy_to_user(arg, &info, minsz) ? -EFAULT : 0;
1190 }
1191 
vfio_pci_ioctl_set_irqs(struct vfio_pci_core_device * vdev,struct vfio_irq_set __user * arg)1192 static int vfio_pci_ioctl_set_irqs(struct vfio_pci_core_device *vdev,
1193 				   struct vfio_irq_set __user *arg)
1194 {
1195 	unsigned long minsz = offsetofend(struct vfio_irq_set, count);
1196 	struct vfio_irq_set hdr;
1197 	u8 *data = NULL;
1198 	int max, ret = 0;
1199 	size_t data_size = 0;
1200 
1201 	if (copy_from_user(&hdr, arg, minsz))
1202 		return -EFAULT;
1203 
1204 	max = vfio_pci_get_irq_count(vdev, hdr.index);
1205 
1206 	ret = vfio_set_irqs_validate_and_prepare(&hdr, max, VFIO_PCI_NUM_IRQS,
1207 						 &data_size);
1208 	if (ret)
1209 		return ret;
1210 
1211 	if (data_size) {
1212 		data = memdup_user(&arg->data, data_size);
1213 		if (IS_ERR(data))
1214 			return PTR_ERR(data);
1215 	}
1216 
1217 	mutex_lock(&vdev->igate);
1218 
1219 	ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index, hdr.start,
1220 				      hdr.count, data);
1221 
1222 	mutex_unlock(&vdev->igate);
1223 	kfree(data);
1224 
1225 	return ret;
1226 }
1227 
vfio_pci_ioctl_reset(struct vfio_pci_core_device * vdev,void __user * arg)1228 static int vfio_pci_ioctl_reset(struct vfio_pci_core_device *vdev,
1229 				void __user *arg)
1230 {
1231 	int ret;
1232 
1233 	if (!vdev->reset_works)
1234 		return -EINVAL;
1235 
1236 	vfio_pci_zap_and_down_write_memory_lock(vdev);
1237 
1238 	/*
1239 	 * This function can be invoked while the power state is non-D0. If
1240 	 * pci_try_reset_function() has been called while the power state is
1241 	 * non-D0, then pci_try_reset_function() will internally set the power
1242 	 * state to D0 without vfio driver involvement. For the devices which
1243 	 * have NoSoftRst-, the reset function can cause the PCI config space
1244 	 * reset without restoring the original state (saved locally in
1245 	 * 'vdev->pm_save').
1246 	 */
1247 	vfio_pci_set_power_state(vdev, PCI_D0);
1248 
1249 	ret = pci_try_reset_function(vdev->pdev);
1250 	up_write(&vdev->memory_lock);
1251 
1252 	return ret;
1253 }
1254 
vfio_pci_ioctl_get_pci_hot_reset_info(struct vfio_pci_core_device * vdev,struct vfio_pci_hot_reset_info __user * arg)1255 static int vfio_pci_ioctl_get_pci_hot_reset_info(
1256 	struct vfio_pci_core_device *vdev,
1257 	struct vfio_pci_hot_reset_info __user *arg)
1258 {
1259 	unsigned long minsz =
1260 		offsetofend(struct vfio_pci_hot_reset_info, count);
1261 	struct vfio_pci_hot_reset_info hdr;
1262 	struct vfio_pci_fill_info fill = {};
1263 	bool slot = false;
1264 	int ret = 0;
1265 
1266 	if (copy_from_user(&hdr, arg, minsz))
1267 		return -EFAULT;
1268 
1269 	if (hdr.argsz < minsz)
1270 		return -EINVAL;
1271 
1272 	hdr.flags = 0;
1273 
1274 	/* Can we do a slot or bus reset or neither? */
1275 	if (!pci_probe_reset_slot(vdev->pdev->slot))
1276 		slot = true;
1277 	else if (pci_probe_reset_bus(vdev->pdev->bus))
1278 		return -ENODEV;
1279 
1280 	fill.devices = arg->devices;
1281 	fill.devices_end = arg->devices +
1282 			   (hdr.argsz - sizeof(hdr)) / sizeof(arg->devices[0]);
1283 	fill.vdev = &vdev->vdev;
1284 
1285 	if (vfio_device_cdev_opened(&vdev->vdev))
1286 		fill.flags |= VFIO_PCI_HOT_RESET_FLAG_DEV_ID |
1287 			     VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED;
1288 
1289 	mutex_lock(&vdev->vdev.dev_set->lock);
1290 	ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_fill_devs,
1291 					    &fill, slot);
1292 	mutex_unlock(&vdev->vdev.dev_set->lock);
1293 	if (ret)
1294 		return ret;
1295 
1296 	hdr.count = fill.count;
1297 	hdr.flags = fill.flags;
1298 	if (copy_to_user(arg, &hdr, minsz))
1299 		return -EFAULT;
1300 
1301 	if (fill.count > fill.devices - arg->devices)
1302 		return -ENOSPC;
1303 	return 0;
1304 }
1305 
1306 static int
vfio_pci_ioctl_pci_hot_reset_groups(struct vfio_pci_core_device * vdev,int array_count,bool slot,struct vfio_pci_hot_reset __user * arg)1307 vfio_pci_ioctl_pci_hot_reset_groups(struct vfio_pci_core_device *vdev,
1308 				    int array_count, bool slot,
1309 				    struct vfio_pci_hot_reset __user *arg)
1310 {
1311 	int32_t *group_fds;
1312 	struct file **files;
1313 	struct vfio_pci_group_info info;
1314 	int file_idx, count = 0, ret = 0;
1315 
1316 	/*
1317 	 * We can't let userspace give us an arbitrarily large buffer to copy,
1318 	 * so verify how many we think there could be.  Note groups can have
1319 	 * multiple devices so one group per device is the max.
1320 	 */
1321 	ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,
1322 					    &count, slot);
1323 	if (ret)
1324 		return ret;
1325 
1326 	if (array_count > count)
1327 		return -EINVAL;
1328 
1329 	group_fds = kcalloc(array_count, sizeof(*group_fds), GFP_KERNEL);
1330 	files = kcalloc(array_count, sizeof(*files), GFP_KERNEL);
1331 	if (!group_fds || !files) {
1332 		kfree(group_fds);
1333 		kfree(files);
1334 		return -ENOMEM;
1335 	}
1336 
1337 	if (copy_from_user(group_fds, arg->group_fds,
1338 			   array_count * sizeof(*group_fds))) {
1339 		kfree(group_fds);
1340 		kfree(files);
1341 		return -EFAULT;
1342 	}
1343 
1344 	/*
1345 	 * Get the group file for each fd to ensure the group is held across
1346 	 * the reset
1347 	 */
1348 	for (file_idx = 0; file_idx < array_count; file_idx++) {
1349 		struct file *file = fget(group_fds[file_idx]);
1350 
1351 		if (!file) {
1352 			ret = -EBADF;
1353 			break;
1354 		}
1355 
1356 		/* Ensure the FD is a vfio group FD.*/
1357 		if (!vfio_file_is_group(file)) {
1358 			fput(file);
1359 			ret = -EINVAL;
1360 			break;
1361 		}
1362 
1363 		files[file_idx] = file;
1364 	}
1365 
1366 	kfree(group_fds);
1367 
1368 	/* release reference to groups on error */
1369 	if (ret)
1370 		goto hot_reset_release;
1371 
1372 	info.count = array_count;
1373 	info.files = files;
1374 
1375 	ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info, NULL);
1376 
1377 hot_reset_release:
1378 	for (file_idx--; file_idx >= 0; file_idx--)
1379 		fput(files[file_idx]);
1380 
1381 	kfree(files);
1382 	return ret;
1383 }
1384 
vfio_pci_ioctl_pci_hot_reset(struct vfio_pci_core_device * vdev,struct vfio_pci_hot_reset __user * arg)1385 static int vfio_pci_ioctl_pci_hot_reset(struct vfio_pci_core_device *vdev,
1386 					struct vfio_pci_hot_reset __user *arg)
1387 {
1388 	unsigned long minsz = offsetofend(struct vfio_pci_hot_reset, count);
1389 	struct vfio_pci_hot_reset hdr;
1390 	bool slot = false;
1391 
1392 	if (copy_from_user(&hdr, arg, minsz))
1393 		return -EFAULT;
1394 
1395 	if (hdr.argsz < minsz || hdr.flags)
1396 		return -EINVAL;
1397 
1398 	/* zero-length array is only for cdev opened devices */
1399 	if (!!hdr.count == vfio_device_cdev_opened(&vdev->vdev))
1400 		return -EINVAL;
1401 
1402 	/* Can we do a slot or bus reset or neither? */
1403 	if (!pci_probe_reset_slot(vdev->pdev->slot))
1404 		slot = true;
1405 	else if (pci_probe_reset_bus(vdev->pdev->bus))
1406 		return -ENODEV;
1407 
1408 	if (hdr.count)
1409 		return vfio_pci_ioctl_pci_hot_reset_groups(vdev, hdr.count, slot, arg);
1410 
1411 	return vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, NULL,
1412 					  vfio_iommufd_device_ictx(&vdev->vdev));
1413 }
1414 
vfio_pci_ioctl_ioeventfd(struct vfio_pci_core_device * vdev,struct vfio_device_ioeventfd __user * arg)1415 static int vfio_pci_ioctl_ioeventfd(struct vfio_pci_core_device *vdev,
1416 				    struct vfio_device_ioeventfd __user *arg)
1417 {
1418 	unsigned long minsz = offsetofend(struct vfio_device_ioeventfd, fd);
1419 	struct vfio_device_ioeventfd ioeventfd;
1420 	int count;
1421 
1422 	if (copy_from_user(&ioeventfd, arg, minsz))
1423 		return -EFAULT;
1424 
1425 	if (ioeventfd.argsz < minsz)
1426 		return -EINVAL;
1427 
1428 	if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK)
1429 		return -EINVAL;
1430 
1431 	count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK;
1432 
1433 	if (hweight8(count) != 1 || ioeventfd.fd < -1)
1434 		return -EINVAL;
1435 
1436 	return vfio_pci_ioeventfd(vdev, ioeventfd.offset, ioeventfd.data, count,
1437 				  ioeventfd.fd);
1438 }
1439 
vfio_pci_core_ioctl(struct vfio_device * core_vdev,unsigned int cmd,unsigned long arg)1440 long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
1441 			 unsigned long arg)
1442 {
1443 	struct vfio_pci_core_device *vdev =
1444 		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1445 	void __user *uarg = (void __user *)arg;
1446 
1447 	switch (cmd) {
1448 	case VFIO_DEVICE_GET_INFO:
1449 		return vfio_pci_ioctl_get_info(vdev, uarg);
1450 	case VFIO_DEVICE_GET_IRQ_INFO:
1451 		return vfio_pci_ioctl_get_irq_info(vdev, uarg);
1452 	case VFIO_DEVICE_GET_PCI_HOT_RESET_INFO:
1453 		return vfio_pci_ioctl_get_pci_hot_reset_info(vdev, uarg);
1454 	case VFIO_DEVICE_GET_REGION_INFO:
1455 		return vfio_pci_ioctl_get_region_info(vdev, uarg);
1456 	case VFIO_DEVICE_IOEVENTFD:
1457 		return vfio_pci_ioctl_ioeventfd(vdev, uarg);
1458 	case VFIO_DEVICE_PCI_HOT_RESET:
1459 		return vfio_pci_ioctl_pci_hot_reset(vdev, uarg);
1460 	case VFIO_DEVICE_RESET:
1461 		return vfio_pci_ioctl_reset(vdev, uarg);
1462 	case VFIO_DEVICE_SET_IRQS:
1463 		return vfio_pci_ioctl_set_irqs(vdev, uarg);
1464 	default:
1465 		return -ENOTTY;
1466 	}
1467 }
1468 EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl);
1469 
vfio_pci_core_feature_token(struct vfio_device * device,u32 flags,uuid_t __user * arg,size_t argsz)1470 static int vfio_pci_core_feature_token(struct vfio_device *device, u32 flags,
1471 				       uuid_t __user *arg, size_t argsz)
1472 {
1473 	struct vfio_pci_core_device *vdev =
1474 		container_of(device, struct vfio_pci_core_device, vdev);
1475 	uuid_t uuid;
1476 	int ret;
1477 
1478 	if (!vdev->vf_token)
1479 		return -ENOTTY;
1480 	/*
1481 	 * We do not support GET of the VF Token UUID as this could
1482 	 * expose the token of the previous device user.
1483 	 */
1484 	ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_SET,
1485 				 sizeof(uuid));
1486 	if (ret != 1)
1487 		return ret;
1488 
1489 	if (copy_from_user(&uuid, arg, sizeof(uuid)))
1490 		return -EFAULT;
1491 
1492 	mutex_lock(&vdev->vf_token->lock);
1493 	uuid_copy(&vdev->vf_token->uuid, &uuid);
1494 	mutex_unlock(&vdev->vf_token->lock);
1495 	return 0;
1496 }
1497 
vfio_pci_core_ioctl_feature(struct vfio_device * device,u32 flags,void __user * arg,size_t argsz)1498 int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags,
1499 				void __user *arg, size_t argsz)
1500 {
1501 	switch (flags & VFIO_DEVICE_FEATURE_MASK) {
1502 	case VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY:
1503 		return vfio_pci_core_pm_entry(device, flags, arg, argsz);
1504 	case VFIO_DEVICE_FEATURE_LOW_POWER_ENTRY_WITH_WAKEUP:
1505 		return vfio_pci_core_pm_entry_with_wakeup(device, flags,
1506 							  arg, argsz);
1507 	case VFIO_DEVICE_FEATURE_LOW_POWER_EXIT:
1508 		return vfio_pci_core_pm_exit(device, flags, arg, argsz);
1509 	case VFIO_DEVICE_FEATURE_PCI_VF_TOKEN:
1510 		return vfio_pci_core_feature_token(device, flags, arg, argsz);
1511 	default:
1512 		return -ENOTTY;
1513 	}
1514 }
1515 EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl_feature);
1516 
vfio_pci_rw(struct vfio_pci_core_device * vdev,char __user * buf,size_t count,loff_t * ppos,bool iswrite)1517 static ssize_t vfio_pci_rw(struct vfio_pci_core_device *vdev, char __user *buf,
1518 			   size_t count, loff_t *ppos, bool iswrite)
1519 {
1520 	unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
1521 	int ret;
1522 
1523 	if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1524 		return -EINVAL;
1525 
1526 	ret = pm_runtime_resume_and_get(&vdev->pdev->dev);
1527 	if (ret) {
1528 		pci_info_ratelimited(vdev->pdev, "runtime resume failed %d\n",
1529 				     ret);
1530 		return -EIO;
1531 	}
1532 
1533 	switch (index) {
1534 	case VFIO_PCI_CONFIG_REGION_INDEX:
1535 		ret = vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
1536 		break;
1537 
1538 	case VFIO_PCI_ROM_REGION_INDEX:
1539 		if (iswrite)
1540 			ret = -EINVAL;
1541 		else
1542 			ret = vfio_pci_bar_rw(vdev, buf, count, ppos, false);
1543 		break;
1544 
1545 	case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1546 		ret = vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
1547 		break;
1548 
1549 	case VFIO_PCI_VGA_REGION_INDEX:
1550 		ret = vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
1551 		break;
1552 
1553 	default:
1554 		index -= VFIO_PCI_NUM_REGIONS;
1555 		ret = vdev->region[index].ops->rw(vdev, buf,
1556 						   count, ppos, iswrite);
1557 		break;
1558 	}
1559 
1560 	pm_runtime_put(&vdev->pdev->dev);
1561 	return ret;
1562 }
1563 
vfio_pci_core_read(struct vfio_device * core_vdev,char __user * buf,size_t count,loff_t * ppos)1564 ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf,
1565 		size_t count, loff_t *ppos)
1566 {
1567 	struct vfio_pci_core_device *vdev =
1568 		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1569 
1570 	if (!count)
1571 		return 0;
1572 
1573 	return vfio_pci_rw(vdev, buf, count, ppos, false);
1574 }
1575 EXPORT_SYMBOL_GPL(vfio_pci_core_read);
1576 
vfio_pci_core_write(struct vfio_device * core_vdev,const char __user * buf,size_t count,loff_t * ppos)1577 ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf,
1578 		size_t count, loff_t *ppos)
1579 {
1580 	struct vfio_pci_core_device *vdev =
1581 		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1582 
1583 	if (!count)
1584 		return 0;
1585 
1586 	return vfio_pci_rw(vdev, (char __user *)buf, count, ppos, true);
1587 }
1588 EXPORT_SYMBOL_GPL(vfio_pci_core_write);
1589 
1590 /* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */
vfio_pci_zap_and_vma_lock(struct vfio_pci_core_device * vdev,bool try)1591 static int vfio_pci_zap_and_vma_lock(struct vfio_pci_core_device *vdev, bool try)
1592 {
1593 	struct vfio_pci_mmap_vma *mmap_vma, *tmp;
1594 
1595 	/*
1596 	 * Lock ordering:
1597 	 * vma_lock is nested under mmap_lock for vm_ops callback paths.
1598 	 * The memory_lock semaphore is used by both code paths calling
1599 	 * into this function to zap vmas and the vm_ops.fault callback
1600 	 * to protect the memory enable state of the device.
1601 	 *
1602 	 * When zapping vmas we need to maintain the mmap_lock => vma_lock
1603 	 * ordering, which requires using vma_lock to walk vma_list to
1604 	 * acquire an mm, then dropping vma_lock to get the mmap_lock and
1605 	 * reacquiring vma_lock.  This logic is derived from similar
1606 	 * requirements in uverbs_user_mmap_disassociate().
1607 	 *
1608 	 * mmap_lock must always be the top-level lock when it is taken.
1609 	 * Therefore we can only hold the memory_lock write lock when
1610 	 * vma_list is empty, as we'd need to take mmap_lock to clear
1611 	 * entries.  vma_list can only be guaranteed empty when holding
1612 	 * vma_lock, thus memory_lock is nested under vma_lock.
1613 	 *
1614 	 * This enables the vm_ops.fault callback to acquire vma_lock,
1615 	 * followed by memory_lock read lock, while already holding
1616 	 * mmap_lock without risk of deadlock.
1617 	 */
1618 	while (1) {
1619 		struct mm_struct *mm = NULL;
1620 
1621 		if (try) {
1622 			if (!mutex_trylock(&vdev->vma_lock))
1623 				return 0;
1624 		} else {
1625 			mutex_lock(&vdev->vma_lock);
1626 		}
1627 		while (!list_empty(&vdev->vma_list)) {
1628 			mmap_vma = list_first_entry(&vdev->vma_list,
1629 						    struct vfio_pci_mmap_vma,
1630 						    vma_next);
1631 			mm = mmap_vma->vma->vm_mm;
1632 			if (mmget_not_zero(mm))
1633 				break;
1634 
1635 			list_del(&mmap_vma->vma_next);
1636 			kfree(mmap_vma);
1637 			mm = NULL;
1638 		}
1639 		if (!mm)
1640 			return 1;
1641 		mutex_unlock(&vdev->vma_lock);
1642 
1643 		if (try) {
1644 			if (!mmap_read_trylock(mm)) {
1645 				mmput(mm);
1646 				return 0;
1647 			}
1648 		} else {
1649 			mmap_read_lock(mm);
1650 		}
1651 		if (try) {
1652 			if (!mutex_trylock(&vdev->vma_lock)) {
1653 				mmap_read_unlock(mm);
1654 				mmput(mm);
1655 				return 0;
1656 			}
1657 		} else {
1658 			mutex_lock(&vdev->vma_lock);
1659 		}
1660 		list_for_each_entry_safe(mmap_vma, tmp,
1661 					 &vdev->vma_list, vma_next) {
1662 			struct vm_area_struct *vma = mmap_vma->vma;
1663 
1664 			if (vma->vm_mm != mm)
1665 				continue;
1666 
1667 			list_del(&mmap_vma->vma_next);
1668 			kfree(mmap_vma);
1669 
1670 			zap_vma_ptes(vma, vma->vm_start,
1671 				     vma->vm_end - vma->vm_start);
1672 		}
1673 		mutex_unlock(&vdev->vma_lock);
1674 		mmap_read_unlock(mm);
1675 		mmput(mm);
1676 	}
1677 }
1678 
vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device * vdev)1679 void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev)
1680 {
1681 	vfio_pci_zap_and_vma_lock(vdev, false);
1682 	down_write(&vdev->memory_lock);
1683 	mutex_unlock(&vdev->vma_lock);
1684 }
1685 
vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device * vdev)1686 u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev)
1687 {
1688 	u16 cmd;
1689 
1690 	down_write(&vdev->memory_lock);
1691 	pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd);
1692 	if (!(cmd & PCI_COMMAND_MEMORY))
1693 		pci_write_config_word(vdev->pdev, PCI_COMMAND,
1694 				      cmd | PCI_COMMAND_MEMORY);
1695 
1696 	return cmd;
1697 }
1698 
vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device * vdev,u16 cmd)1699 void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, u16 cmd)
1700 {
1701 	pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd);
1702 	up_write(&vdev->memory_lock);
1703 }
1704 
1705 /* Caller holds vma_lock */
__vfio_pci_add_vma(struct vfio_pci_core_device * vdev,struct vm_area_struct * vma)1706 static int __vfio_pci_add_vma(struct vfio_pci_core_device *vdev,
1707 			      struct vm_area_struct *vma)
1708 {
1709 	struct vfio_pci_mmap_vma *mmap_vma;
1710 
1711 	mmap_vma = kmalloc(sizeof(*mmap_vma), GFP_KERNEL_ACCOUNT);
1712 	if (!mmap_vma)
1713 		return -ENOMEM;
1714 
1715 	mmap_vma->vma = vma;
1716 	list_add(&mmap_vma->vma_next, &vdev->vma_list);
1717 
1718 	return 0;
1719 }
1720 
1721 /*
1722  * Zap mmaps on open so that we can fault them in on access and therefore
1723  * our vma_list only tracks mappings accessed since last zap.
1724  */
vfio_pci_mmap_open(struct vm_area_struct * vma)1725 static void vfio_pci_mmap_open(struct vm_area_struct *vma)
1726 {
1727 	zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
1728 }
1729 
vfio_pci_mmap_close(struct vm_area_struct * vma)1730 static void vfio_pci_mmap_close(struct vm_area_struct *vma)
1731 {
1732 	struct vfio_pci_core_device *vdev = vma->vm_private_data;
1733 	struct vfio_pci_mmap_vma *mmap_vma;
1734 
1735 	mutex_lock(&vdev->vma_lock);
1736 	list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) {
1737 		if (mmap_vma->vma == vma) {
1738 			list_del(&mmap_vma->vma_next);
1739 			kfree(mmap_vma);
1740 			break;
1741 		}
1742 	}
1743 	mutex_unlock(&vdev->vma_lock);
1744 }
1745 
vfio_pci_mmap_fault(struct vm_fault * vmf)1746 static vm_fault_t vfio_pci_mmap_fault(struct vm_fault *vmf)
1747 {
1748 	struct vm_area_struct *vma = vmf->vma;
1749 	struct vfio_pci_core_device *vdev = vma->vm_private_data;
1750 	struct vfio_pci_mmap_vma *mmap_vma;
1751 	vm_fault_t ret = VM_FAULT_NOPAGE;
1752 
1753 	mutex_lock(&vdev->vma_lock);
1754 	down_read(&vdev->memory_lock);
1755 
1756 	/*
1757 	 * Memory region cannot be accessed if the low power feature is engaged
1758 	 * or memory access is disabled.
1759 	 */
1760 	if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev)) {
1761 		ret = VM_FAULT_SIGBUS;
1762 		goto up_out;
1763 	}
1764 
1765 	/*
1766 	 * We populate the whole vma on fault, so we need to test whether
1767 	 * the vma has already been mapped, such as for concurrent faults
1768 	 * to the same vma.  io_remap_pfn_range() will trigger a BUG_ON if
1769 	 * we ask it to fill the same range again.
1770 	 */
1771 	list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) {
1772 		if (mmap_vma->vma == vma)
1773 			goto up_out;
1774 	}
1775 
1776 	if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
1777 			       vma->vm_end - vma->vm_start,
1778 			       vma->vm_page_prot)) {
1779 		ret = VM_FAULT_SIGBUS;
1780 		zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
1781 		goto up_out;
1782 	}
1783 
1784 	if (__vfio_pci_add_vma(vdev, vma)) {
1785 		ret = VM_FAULT_OOM;
1786 		zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
1787 	}
1788 
1789 up_out:
1790 	up_read(&vdev->memory_lock);
1791 	mutex_unlock(&vdev->vma_lock);
1792 	return ret;
1793 }
1794 
1795 static const struct vm_operations_struct vfio_pci_mmap_ops = {
1796 	.open = vfio_pci_mmap_open,
1797 	.close = vfio_pci_mmap_close,
1798 	.fault = vfio_pci_mmap_fault,
1799 };
1800 
vfio_pci_core_mmap(struct vfio_device * core_vdev,struct vm_area_struct * vma)1801 int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma)
1802 {
1803 	struct vfio_pci_core_device *vdev =
1804 		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1805 	struct pci_dev *pdev = vdev->pdev;
1806 	unsigned int index;
1807 	u64 phys_len, req_len, pgoff, req_start;
1808 	int ret;
1809 
1810 	index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1811 
1812 	if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1813 		return -EINVAL;
1814 	if (vma->vm_end < vma->vm_start)
1815 		return -EINVAL;
1816 	if ((vma->vm_flags & VM_SHARED) == 0)
1817 		return -EINVAL;
1818 	if (index >= VFIO_PCI_NUM_REGIONS) {
1819 		int regnum = index - VFIO_PCI_NUM_REGIONS;
1820 		struct vfio_pci_region *region = vdev->region + regnum;
1821 
1822 		if (region->ops && region->ops->mmap &&
1823 		    (region->flags & VFIO_REGION_INFO_FLAG_MMAP))
1824 			return region->ops->mmap(vdev, region, vma);
1825 		return -EINVAL;
1826 	}
1827 	if (index >= VFIO_PCI_ROM_REGION_INDEX)
1828 		return -EINVAL;
1829 	if (!vdev->bar_mmap_supported[index])
1830 		return -EINVAL;
1831 
1832 	phys_len = PAGE_ALIGN(pci_resource_len(pdev, index));
1833 	req_len = vma->vm_end - vma->vm_start;
1834 	pgoff = vma->vm_pgoff &
1835 		((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1836 	req_start = pgoff << PAGE_SHIFT;
1837 
1838 	if (req_start + req_len > phys_len)
1839 		return -EINVAL;
1840 
1841 	/*
1842 	 * Even though we don't make use of the barmap for the mmap,
1843 	 * we need to request the region and the barmap tracks that.
1844 	 */
1845 	if (!vdev->barmap[index]) {
1846 		ret = pci_request_selected_regions(pdev,
1847 						   1 << index, "vfio-pci");
1848 		if (ret)
1849 			return ret;
1850 
1851 		vdev->barmap[index] = pci_iomap(pdev, index, 0);
1852 		if (!vdev->barmap[index]) {
1853 			pci_release_selected_regions(pdev, 1 << index);
1854 			return -ENOMEM;
1855 		}
1856 	}
1857 
1858 	vma->vm_private_data = vdev;
1859 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1860 	vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff;
1861 
1862 	/*
1863 	 * See remap_pfn_range(), called from vfio_pci_fault() but we can't
1864 	 * change vm_flags within the fault handler.  Set them now.
1865 	 */
1866 	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
1867 	vma->vm_ops = &vfio_pci_mmap_ops;
1868 
1869 	return 0;
1870 }
1871 EXPORT_SYMBOL_GPL(vfio_pci_core_mmap);
1872 
vfio_pci_core_request(struct vfio_device * core_vdev,unsigned int count)1873 void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count)
1874 {
1875 	struct vfio_pci_core_device *vdev =
1876 		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1877 	struct pci_dev *pdev = vdev->pdev;
1878 
1879 	mutex_lock(&vdev->igate);
1880 
1881 	if (vdev->req_trigger) {
1882 		if (!(count % 10))
1883 			pci_notice_ratelimited(pdev,
1884 				"Relaying device request to user (#%u)\n",
1885 				count);
1886 		eventfd_signal(vdev->req_trigger, 1);
1887 	} else if (count == 0) {
1888 		pci_warn(pdev,
1889 			"No device request channel registered, blocked until released by user\n");
1890 	}
1891 
1892 	mutex_unlock(&vdev->igate);
1893 }
1894 EXPORT_SYMBOL_GPL(vfio_pci_core_request);
1895 
vfio_pci_validate_vf_token(struct vfio_pci_core_device * vdev,bool vf_token,uuid_t * uuid)1896 static int vfio_pci_validate_vf_token(struct vfio_pci_core_device *vdev,
1897 				      bool vf_token, uuid_t *uuid)
1898 {
1899 	/*
1900 	 * There's always some degree of trust or collaboration between SR-IOV
1901 	 * PF and VFs, even if just that the PF hosts the SR-IOV capability and
1902 	 * can disrupt VFs with a reset, but often the PF has more explicit
1903 	 * access to deny service to the VF or access data passed through the
1904 	 * VF.  We therefore require an opt-in via a shared VF token (UUID) to
1905 	 * represent this trust.  This both prevents that a VF driver might
1906 	 * assume the PF driver is a trusted, in-kernel driver, and also that
1907 	 * a PF driver might be replaced with a rogue driver, unknown to in-use
1908 	 * VF drivers.
1909 	 *
1910 	 * Therefore when presented with a VF, if the PF is a vfio device and
1911 	 * it is bound to the vfio-pci driver, the user needs to provide a VF
1912 	 * token to access the device, in the form of appending a vf_token to
1913 	 * the device name, for example:
1914 	 *
1915 	 * "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
1916 	 *
1917 	 * When presented with a PF which has VFs in use, the user must also
1918 	 * provide the current VF token to prove collaboration with existing
1919 	 * VF users.  If VFs are not in use, the VF token provided for the PF
1920 	 * device will act to set the VF token.
1921 	 *
1922 	 * If the VF token is provided but unused, an error is generated.
1923 	 */
1924 	if (vdev->pdev->is_virtfn) {
1925 		struct vfio_pci_core_device *pf_vdev = vdev->sriov_pf_core_dev;
1926 		bool match;
1927 
1928 		if (!pf_vdev) {
1929 			if (!vf_token)
1930 				return 0; /* PF is not vfio-pci, no VF token */
1931 
1932 			pci_info_ratelimited(vdev->pdev,
1933 				"VF token incorrectly provided, PF not bound to vfio-pci\n");
1934 			return -EINVAL;
1935 		}
1936 
1937 		if (!vf_token) {
1938 			pci_info_ratelimited(vdev->pdev,
1939 				"VF token required to access device\n");
1940 			return -EACCES;
1941 		}
1942 
1943 		mutex_lock(&pf_vdev->vf_token->lock);
1944 		match = uuid_equal(uuid, &pf_vdev->vf_token->uuid);
1945 		mutex_unlock(&pf_vdev->vf_token->lock);
1946 
1947 		if (!match) {
1948 			pci_info_ratelimited(vdev->pdev,
1949 				"Incorrect VF token provided for device\n");
1950 			return -EACCES;
1951 		}
1952 	} else if (vdev->vf_token) {
1953 		mutex_lock(&vdev->vf_token->lock);
1954 		if (vdev->vf_token->users) {
1955 			if (!vf_token) {
1956 				mutex_unlock(&vdev->vf_token->lock);
1957 				pci_info_ratelimited(vdev->pdev,
1958 					"VF token required to access device\n");
1959 				return -EACCES;
1960 			}
1961 
1962 			if (!uuid_equal(uuid, &vdev->vf_token->uuid)) {
1963 				mutex_unlock(&vdev->vf_token->lock);
1964 				pci_info_ratelimited(vdev->pdev,
1965 					"Incorrect VF token provided for device\n");
1966 				return -EACCES;
1967 			}
1968 		} else if (vf_token) {
1969 			uuid_copy(&vdev->vf_token->uuid, uuid);
1970 		}
1971 
1972 		mutex_unlock(&vdev->vf_token->lock);
1973 	} else if (vf_token) {
1974 		pci_info_ratelimited(vdev->pdev,
1975 			"VF token incorrectly provided, not a PF or VF\n");
1976 		return -EINVAL;
1977 	}
1978 
1979 	return 0;
1980 }
1981 
1982 #define VF_TOKEN_ARG "vf_token="
1983 
vfio_pci_core_match(struct vfio_device * core_vdev,char * buf)1984 int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf)
1985 {
1986 	struct vfio_pci_core_device *vdev =
1987 		container_of(core_vdev, struct vfio_pci_core_device, vdev);
1988 	bool vf_token = false;
1989 	uuid_t uuid;
1990 	int ret;
1991 
1992 	if (strncmp(pci_name(vdev->pdev), buf, strlen(pci_name(vdev->pdev))))
1993 		return 0; /* No match */
1994 
1995 	if (strlen(buf) > strlen(pci_name(vdev->pdev))) {
1996 		buf += strlen(pci_name(vdev->pdev));
1997 
1998 		if (*buf != ' ')
1999 			return 0; /* No match: non-whitespace after name */
2000 
2001 		while (*buf) {
2002 			if (*buf == ' ') {
2003 				buf++;
2004 				continue;
2005 			}
2006 
2007 			if (!vf_token && !strncmp(buf, VF_TOKEN_ARG,
2008 						  strlen(VF_TOKEN_ARG))) {
2009 				buf += strlen(VF_TOKEN_ARG);
2010 
2011 				if (strlen(buf) < UUID_STRING_LEN)
2012 					return -EINVAL;
2013 
2014 				ret = uuid_parse(buf, &uuid);
2015 				if (ret)
2016 					return ret;
2017 
2018 				vf_token = true;
2019 				buf += UUID_STRING_LEN;
2020 			} else {
2021 				/* Unknown/duplicate option */
2022 				return -EINVAL;
2023 			}
2024 		}
2025 	}
2026 
2027 	ret = vfio_pci_validate_vf_token(vdev, vf_token, &uuid);
2028 	if (ret)
2029 		return ret;
2030 
2031 	return 1; /* Match */
2032 }
2033 EXPORT_SYMBOL_GPL(vfio_pci_core_match);
2034 
vfio_pci_bus_notifier(struct notifier_block * nb,unsigned long action,void * data)2035 static int vfio_pci_bus_notifier(struct notifier_block *nb,
2036 				 unsigned long action, void *data)
2037 {
2038 	struct vfio_pci_core_device *vdev = container_of(nb,
2039 						    struct vfio_pci_core_device, nb);
2040 	struct device *dev = data;
2041 	struct pci_dev *pdev = to_pci_dev(dev);
2042 	struct pci_dev *physfn = pci_physfn(pdev);
2043 
2044 	if (action == BUS_NOTIFY_ADD_DEVICE &&
2045 	    pdev->is_virtfn && physfn == vdev->pdev) {
2046 		pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n",
2047 			 pci_name(pdev));
2048 		pdev->driver_override = kasprintf(GFP_KERNEL, "%s",
2049 						  vdev->vdev.ops->name);
2050 	} else if (action == BUS_NOTIFY_BOUND_DRIVER &&
2051 		   pdev->is_virtfn && physfn == vdev->pdev) {
2052 		struct pci_driver *drv = pci_dev_driver(pdev);
2053 
2054 		if (drv && drv != pci_dev_driver(vdev->pdev))
2055 			pci_warn(vdev->pdev,
2056 				 "VF %s bound to driver %s while PF bound to driver %s\n",
2057 				 pci_name(pdev), drv->name,
2058 				 pci_dev_driver(vdev->pdev)->name);
2059 	}
2060 
2061 	return 0;
2062 }
2063 
vfio_pci_vf_init(struct vfio_pci_core_device * vdev)2064 static int vfio_pci_vf_init(struct vfio_pci_core_device *vdev)
2065 {
2066 	struct pci_dev *pdev = vdev->pdev;
2067 	struct vfio_pci_core_device *cur;
2068 	struct pci_dev *physfn;
2069 	int ret;
2070 
2071 	if (pdev->is_virtfn) {
2072 		/*
2073 		 * If this VF was created by our vfio_pci_core_sriov_configure()
2074 		 * then we can find the PF vfio_pci_core_device now, and due to
2075 		 * the locking in pci_disable_sriov() it cannot change until
2076 		 * this VF device driver is removed.
2077 		 */
2078 		physfn = pci_physfn(vdev->pdev);
2079 		mutex_lock(&vfio_pci_sriov_pfs_mutex);
2080 		list_for_each_entry(cur, &vfio_pci_sriov_pfs, sriov_pfs_item) {
2081 			if (cur->pdev == physfn) {
2082 				vdev->sriov_pf_core_dev = cur;
2083 				break;
2084 			}
2085 		}
2086 		mutex_unlock(&vfio_pci_sriov_pfs_mutex);
2087 		return 0;
2088 	}
2089 
2090 	/* Not a SRIOV PF */
2091 	if (!pdev->is_physfn)
2092 		return 0;
2093 
2094 	vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL);
2095 	if (!vdev->vf_token)
2096 		return -ENOMEM;
2097 
2098 	mutex_init(&vdev->vf_token->lock);
2099 	uuid_gen(&vdev->vf_token->uuid);
2100 
2101 	vdev->nb.notifier_call = vfio_pci_bus_notifier;
2102 	ret = bus_register_notifier(&pci_bus_type, &vdev->nb);
2103 	if (ret) {
2104 		kfree(vdev->vf_token);
2105 		return ret;
2106 	}
2107 	return 0;
2108 }
2109 
vfio_pci_vf_uninit(struct vfio_pci_core_device * vdev)2110 static void vfio_pci_vf_uninit(struct vfio_pci_core_device *vdev)
2111 {
2112 	if (!vdev->vf_token)
2113 		return;
2114 
2115 	bus_unregister_notifier(&pci_bus_type, &vdev->nb);
2116 	WARN_ON(vdev->vf_token->users);
2117 	mutex_destroy(&vdev->vf_token->lock);
2118 	kfree(vdev->vf_token);
2119 }
2120 
vfio_pci_vga_init(struct vfio_pci_core_device * vdev)2121 static int vfio_pci_vga_init(struct vfio_pci_core_device *vdev)
2122 {
2123 	struct pci_dev *pdev = vdev->pdev;
2124 	int ret;
2125 
2126 	if (!vfio_pci_is_vga(pdev))
2127 		return 0;
2128 
2129 	ret = aperture_remove_conflicting_pci_devices(pdev, vdev->vdev.ops->name);
2130 	if (ret)
2131 		return ret;
2132 
2133 	ret = vga_client_register(pdev, vfio_pci_set_decode);
2134 	if (ret)
2135 		return ret;
2136 	vga_set_legacy_decoding(pdev, vfio_pci_set_decode(pdev, false));
2137 	return 0;
2138 }
2139 
vfio_pci_vga_uninit(struct vfio_pci_core_device * vdev)2140 static void vfio_pci_vga_uninit(struct vfio_pci_core_device *vdev)
2141 {
2142 	struct pci_dev *pdev = vdev->pdev;
2143 
2144 	if (!vfio_pci_is_vga(pdev))
2145 		return;
2146 	vga_client_unregister(pdev);
2147 	vga_set_legacy_decoding(pdev, VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |
2148 					      VGA_RSRC_LEGACY_IO |
2149 					      VGA_RSRC_LEGACY_MEM);
2150 }
2151 
vfio_pci_core_init_dev(struct vfio_device * core_vdev)2152 int vfio_pci_core_init_dev(struct vfio_device *core_vdev)
2153 {
2154 	struct vfio_pci_core_device *vdev =
2155 		container_of(core_vdev, struct vfio_pci_core_device, vdev);
2156 
2157 	vdev->pdev = to_pci_dev(core_vdev->dev);
2158 	vdev->irq_type = VFIO_PCI_NUM_IRQS;
2159 	mutex_init(&vdev->igate);
2160 	spin_lock_init(&vdev->irqlock);
2161 	mutex_init(&vdev->ioeventfds_lock);
2162 	INIT_LIST_HEAD(&vdev->dummy_resources_list);
2163 	INIT_LIST_HEAD(&vdev->ioeventfds_list);
2164 	mutex_init(&vdev->vma_lock);
2165 	INIT_LIST_HEAD(&vdev->vma_list);
2166 	INIT_LIST_HEAD(&vdev->sriov_pfs_item);
2167 	init_rwsem(&vdev->memory_lock);
2168 	xa_init(&vdev->ctx);
2169 
2170 	return 0;
2171 }
2172 EXPORT_SYMBOL_GPL(vfio_pci_core_init_dev);
2173 
vfio_pci_core_release_dev(struct vfio_device * core_vdev)2174 void vfio_pci_core_release_dev(struct vfio_device *core_vdev)
2175 {
2176 	struct vfio_pci_core_device *vdev =
2177 		container_of(core_vdev, struct vfio_pci_core_device, vdev);
2178 
2179 	mutex_destroy(&vdev->igate);
2180 	mutex_destroy(&vdev->ioeventfds_lock);
2181 	mutex_destroy(&vdev->vma_lock);
2182 	kfree(vdev->region);
2183 	kfree(vdev->pm_save);
2184 }
2185 EXPORT_SYMBOL_GPL(vfio_pci_core_release_dev);
2186 
vfio_pci_core_register_device(struct vfio_pci_core_device * vdev)2187 int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev)
2188 {
2189 	struct pci_dev *pdev = vdev->pdev;
2190 	struct device *dev = &pdev->dev;
2191 	int ret;
2192 
2193 	/* Drivers must set the vfio_pci_core_device to their drvdata */
2194 	if (WARN_ON(vdev != dev_get_drvdata(dev)))
2195 		return -EINVAL;
2196 
2197 	if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL)
2198 		return -EINVAL;
2199 
2200 	if (vdev->vdev.mig_ops) {
2201 		if (!(vdev->vdev.mig_ops->migration_get_state &&
2202 		      vdev->vdev.mig_ops->migration_set_state &&
2203 		      vdev->vdev.mig_ops->migration_get_data_size) ||
2204 		    !(vdev->vdev.migration_flags & VFIO_MIGRATION_STOP_COPY))
2205 			return -EINVAL;
2206 	}
2207 
2208 	if (vdev->vdev.log_ops && !(vdev->vdev.log_ops->log_start &&
2209 	    vdev->vdev.log_ops->log_stop &&
2210 	    vdev->vdev.log_ops->log_read_and_clear))
2211 		return -EINVAL;
2212 
2213 	/*
2214 	 * Prevent binding to PFs with VFs enabled, the VFs might be in use
2215 	 * by the host or other users.  We cannot capture the VFs if they
2216 	 * already exist, nor can we track VF users.  Disabling SR-IOV here
2217 	 * would initiate removing the VFs, which would unbind the driver,
2218 	 * which is prone to blocking if that VF is also in use by vfio-pci.
2219 	 * Just reject these PFs and let the user sort it out.
2220 	 */
2221 	if (pci_num_vf(pdev)) {
2222 		pci_warn(pdev, "Cannot bind to PF with SR-IOV enabled\n");
2223 		return -EBUSY;
2224 	}
2225 
2226 	if (pci_is_root_bus(pdev->bus)) {
2227 		ret = vfio_assign_device_set(&vdev->vdev, vdev);
2228 	} else if (!pci_probe_reset_slot(pdev->slot)) {
2229 		ret = vfio_assign_device_set(&vdev->vdev, pdev->slot);
2230 	} else {
2231 		/*
2232 		 * If there is no slot reset support for this device, the whole
2233 		 * bus needs to be grouped together to support bus-wide resets.
2234 		 */
2235 		ret = vfio_assign_device_set(&vdev->vdev, pdev->bus);
2236 	}
2237 
2238 	if (ret)
2239 		return ret;
2240 	ret = vfio_pci_vf_init(vdev);
2241 	if (ret)
2242 		return ret;
2243 	ret = vfio_pci_vga_init(vdev);
2244 	if (ret)
2245 		goto out_vf;
2246 
2247 	vfio_pci_probe_power_state(vdev);
2248 
2249 	/*
2250 	 * pci-core sets the device power state to an unknown value at
2251 	 * bootup and after being removed from a driver.  The only
2252 	 * transition it allows from this unknown state is to D0, which
2253 	 * typically happens when a driver calls pci_enable_device().
2254 	 * We're not ready to enable the device yet, but we do want to
2255 	 * be able to get to D3.  Therefore first do a D0 transition
2256 	 * before enabling runtime PM.
2257 	 */
2258 	vfio_pci_set_power_state(vdev, PCI_D0);
2259 
2260 	dev->driver->pm = &vfio_pci_core_pm_ops;
2261 	pm_runtime_allow(dev);
2262 	if (!disable_idle_d3)
2263 		pm_runtime_put(dev);
2264 
2265 	ret = vfio_register_group_dev(&vdev->vdev);
2266 	if (ret)
2267 		goto out_power;
2268 	return 0;
2269 
2270 out_power:
2271 	if (!disable_idle_d3)
2272 		pm_runtime_get_noresume(dev);
2273 
2274 	pm_runtime_forbid(dev);
2275 out_vf:
2276 	vfio_pci_vf_uninit(vdev);
2277 	return ret;
2278 }
2279 EXPORT_SYMBOL_GPL(vfio_pci_core_register_device);
2280 
vfio_pci_core_unregister_device(struct vfio_pci_core_device * vdev)2281 void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev)
2282 {
2283 	vfio_pci_core_sriov_configure(vdev, 0);
2284 
2285 	vfio_unregister_group_dev(&vdev->vdev);
2286 
2287 	vfio_pci_vf_uninit(vdev);
2288 	vfio_pci_vga_uninit(vdev);
2289 
2290 	if (!disable_idle_d3)
2291 		pm_runtime_get_noresume(&vdev->pdev->dev);
2292 
2293 	pm_runtime_forbid(&vdev->pdev->dev);
2294 }
2295 EXPORT_SYMBOL_GPL(vfio_pci_core_unregister_device);
2296 
vfio_pci_core_aer_err_detected(struct pci_dev * pdev,pci_channel_state_t state)2297 pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev,
2298 						pci_channel_state_t state)
2299 {
2300 	struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);
2301 
2302 	mutex_lock(&vdev->igate);
2303 
2304 	if (vdev->err_trigger)
2305 		eventfd_signal(vdev->err_trigger, 1);
2306 
2307 	mutex_unlock(&vdev->igate);
2308 
2309 	return PCI_ERS_RESULT_CAN_RECOVER;
2310 }
2311 EXPORT_SYMBOL_GPL(vfio_pci_core_aer_err_detected);
2312 
vfio_pci_core_sriov_configure(struct vfio_pci_core_device * vdev,int nr_virtfn)2313 int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev,
2314 				  int nr_virtfn)
2315 {
2316 	struct pci_dev *pdev = vdev->pdev;
2317 	int ret = 0;
2318 
2319 	device_lock_assert(&pdev->dev);
2320 
2321 	if (nr_virtfn) {
2322 		mutex_lock(&vfio_pci_sriov_pfs_mutex);
2323 		/*
2324 		 * The thread that adds the vdev to the list is the only thread
2325 		 * that gets to call pci_enable_sriov() and we will only allow
2326 		 * it to be called once without going through
2327 		 * pci_disable_sriov()
2328 		 */
2329 		if (!list_empty(&vdev->sriov_pfs_item)) {
2330 			ret = -EINVAL;
2331 			goto out_unlock;
2332 		}
2333 		list_add_tail(&vdev->sriov_pfs_item, &vfio_pci_sriov_pfs);
2334 		mutex_unlock(&vfio_pci_sriov_pfs_mutex);
2335 
2336 		/*
2337 		 * The PF power state should always be higher than the VF power
2338 		 * state. The PF can be in low power state either with runtime
2339 		 * power management (when there is no user) or PCI_PM_CTRL
2340 		 * register write by the user. If PF is in the low power state,
2341 		 * then change the power state to D0 first before enabling
2342 		 * SR-IOV. Also, this function can be called at any time, and
2343 		 * userspace PCI_PM_CTRL write can race against this code path,
2344 		 * so protect the same with 'memory_lock'.
2345 		 */
2346 		ret = pm_runtime_resume_and_get(&pdev->dev);
2347 		if (ret)
2348 			goto out_del;
2349 
2350 		down_write(&vdev->memory_lock);
2351 		vfio_pci_set_power_state(vdev, PCI_D0);
2352 		ret = pci_enable_sriov(pdev, nr_virtfn);
2353 		up_write(&vdev->memory_lock);
2354 		if (ret) {
2355 			pm_runtime_put(&pdev->dev);
2356 			goto out_del;
2357 		}
2358 		return nr_virtfn;
2359 	}
2360 
2361 	if (pci_num_vf(pdev)) {
2362 		pci_disable_sriov(pdev);
2363 		pm_runtime_put(&pdev->dev);
2364 	}
2365 
2366 out_del:
2367 	mutex_lock(&vfio_pci_sriov_pfs_mutex);
2368 	list_del_init(&vdev->sriov_pfs_item);
2369 out_unlock:
2370 	mutex_unlock(&vfio_pci_sriov_pfs_mutex);
2371 	return ret;
2372 }
2373 EXPORT_SYMBOL_GPL(vfio_pci_core_sriov_configure);
2374 
2375 const struct pci_error_handlers vfio_pci_core_err_handlers = {
2376 	.error_detected = vfio_pci_core_aer_err_detected,
2377 };
2378 EXPORT_SYMBOL_GPL(vfio_pci_core_err_handlers);
2379 
vfio_dev_in_groups(struct vfio_device * vdev,struct vfio_pci_group_info * groups)2380 static bool vfio_dev_in_groups(struct vfio_device *vdev,
2381 			       struct vfio_pci_group_info *groups)
2382 {
2383 	unsigned int i;
2384 
2385 	if (!groups)
2386 		return false;
2387 
2388 	for (i = 0; i < groups->count; i++)
2389 		if (vfio_file_has_dev(groups->files[i], vdev))
2390 			return true;
2391 	return false;
2392 }
2393 
vfio_pci_is_device_in_set(struct pci_dev * pdev,void * data)2394 static int vfio_pci_is_device_in_set(struct pci_dev *pdev, void *data)
2395 {
2396 	struct vfio_device_set *dev_set = data;
2397 
2398 	return vfio_find_device_in_devset(dev_set, &pdev->dev) ? 0 : -ENODEV;
2399 }
2400 
2401 /*
2402  * vfio-core considers a group to be viable and will create a vfio_device even
2403  * if some devices are bound to drivers like pci-stub or pcieport. Here we
2404  * require all PCI devices to be inside our dev_set since that ensures they stay
2405  * put and that every driver controlling the device can co-ordinate with the
2406  * device reset.
2407  *
2408  * Returns the pci_dev to pass to pci_reset_bus() if every PCI device to be
2409  * reset is inside the dev_set, and pci_reset_bus() can succeed. NULL otherwise.
2410  */
2411 static struct pci_dev *
vfio_pci_dev_set_resettable(struct vfio_device_set * dev_set)2412 vfio_pci_dev_set_resettable(struct vfio_device_set *dev_set)
2413 {
2414 	struct pci_dev *pdev;
2415 
2416 	lockdep_assert_held(&dev_set->lock);
2417 
2418 	/*
2419 	 * By definition all PCI devices in the dev_set share the same PCI
2420 	 * reset, so any pci_dev will have the same outcomes for
2421 	 * pci_probe_reset_*() and pci_reset_bus().
2422 	 */
2423 	pdev = list_first_entry(&dev_set->device_list,
2424 				struct vfio_pci_core_device,
2425 				vdev.dev_set_list)->pdev;
2426 
2427 	/* pci_reset_bus() is supported */
2428 	if (pci_probe_reset_slot(pdev->slot) && pci_probe_reset_bus(pdev->bus))
2429 		return NULL;
2430 
2431 	if (vfio_pci_for_each_slot_or_bus(pdev, vfio_pci_is_device_in_set,
2432 					  dev_set,
2433 					  !pci_probe_reset_slot(pdev->slot)))
2434 		return NULL;
2435 	return pdev;
2436 }
2437 
vfio_pci_dev_set_pm_runtime_get(struct vfio_device_set * dev_set)2438 static int vfio_pci_dev_set_pm_runtime_get(struct vfio_device_set *dev_set)
2439 {
2440 	struct vfio_pci_core_device *cur;
2441 	int ret;
2442 
2443 	list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
2444 		ret = pm_runtime_resume_and_get(&cur->pdev->dev);
2445 		if (ret)
2446 			goto unwind;
2447 	}
2448 
2449 	return 0;
2450 
2451 unwind:
2452 	list_for_each_entry_continue_reverse(cur, &dev_set->device_list,
2453 					     vdev.dev_set_list)
2454 		pm_runtime_put(&cur->pdev->dev);
2455 
2456 	return ret;
2457 }
2458 
2459 /*
2460  * We need to get memory_lock for each device, but devices can share mmap_lock,
2461  * therefore we need to zap and hold the vma_lock for each device, and only then
2462  * get each memory_lock.
2463  */
vfio_pci_dev_set_hot_reset(struct vfio_device_set * dev_set,struct vfio_pci_group_info * groups,struct iommufd_ctx * iommufd_ctx)2464 static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
2465 				      struct vfio_pci_group_info *groups,
2466 				      struct iommufd_ctx *iommufd_ctx)
2467 {
2468 	struct vfio_pci_core_device *cur_mem;
2469 	struct vfio_pci_core_device *cur_vma;
2470 	struct vfio_pci_core_device *cur;
2471 	struct pci_dev *pdev;
2472 	bool is_mem = true;
2473 	int ret;
2474 
2475 	mutex_lock(&dev_set->lock);
2476 	cur_mem = list_first_entry(&dev_set->device_list,
2477 				   struct vfio_pci_core_device,
2478 				   vdev.dev_set_list);
2479 
2480 	pdev = vfio_pci_dev_set_resettable(dev_set);
2481 	if (!pdev) {
2482 		ret = -EINVAL;
2483 		goto err_unlock;
2484 	}
2485 
2486 	/*
2487 	 * Some of the devices in the dev_set can be in the runtime suspended
2488 	 * state. Increment the usage count for all the devices in the dev_set
2489 	 * before reset and decrement the same after reset.
2490 	 */
2491 	ret = vfio_pci_dev_set_pm_runtime_get(dev_set);
2492 	if (ret)
2493 		goto err_unlock;
2494 
2495 	list_for_each_entry(cur_vma, &dev_set->device_list, vdev.dev_set_list) {
2496 		bool owned;
2497 
2498 		/*
2499 		 * Test whether all the affected devices can be reset by the
2500 		 * user.
2501 		 *
2502 		 * If called from a group opened device and the user provides
2503 		 * a set of groups, all the devices in the dev_set should be
2504 		 * contained by the set of groups provided by the user.
2505 		 *
2506 		 * If called from a cdev opened device and the user provides
2507 		 * a zero-length array, all the devices in the dev_set must
2508 		 * be bound to the same iommufd_ctx as the input iommufd_ctx.
2509 		 * If there is any device that has not been bound to any
2510 		 * iommufd_ctx yet, check if its iommu_group has any device
2511 		 * bound to the input iommufd_ctx.  Such devices can be
2512 		 * considered owned by the input iommufd_ctx as the device
2513 		 * cannot be owned by another iommufd_ctx when its iommu_group
2514 		 * is owned.
2515 		 *
2516 		 * Otherwise, reset is not allowed.
2517 		 */
2518 		if (iommufd_ctx) {
2519 			int devid = vfio_iommufd_get_dev_id(&cur_vma->vdev,
2520 							    iommufd_ctx);
2521 
2522 			owned = (devid > 0 || devid == -ENOENT);
2523 		} else {
2524 			owned = vfio_dev_in_groups(&cur_vma->vdev, groups);
2525 		}
2526 
2527 		if (!owned) {
2528 			ret = -EINVAL;
2529 			goto err_undo;
2530 		}
2531 
2532 		/*
2533 		 * Locking multiple devices is prone to deadlock, runaway and
2534 		 * unwind if we hit contention.
2535 		 */
2536 		if (!vfio_pci_zap_and_vma_lock(cur_vma, true)) {
2537 			ret = -EBUSY;
2538 			goto err_undo;
2539 		}
2540 	}
2541 	cur_vma = NULL;
2542 
2543 	list_for_each_entry(cur_mem, &dev_set->device_list, vdev.dev_set_list) {
2544 		if (!down_write_trylock(&cur_mem->memory_lock)) {
2545 			ret = -EBUSY;
2546 			goto err_undo;
2547 		}
2548 		mutex_unlock(&cur_mem->vma_lock);
2549 	}
2550 	cur_mem = NULL;
2551 
2552 	/*
2553 	 * The pci_reset_bus() will reset all the devices in the bus.
2554 	 * The power state can be non-D0 for some of the devices in the bus.
2555 	 * For these devices, the pci_reset_bus() will internally set
2556 	 * the power state to D0 without vfio driver involvement.
2557 	 * For the devices which have NoSoftRst-, the reset function can
2558 	 * cause the PCI config space reset without restoring the original
2559 	 * state (saved locally in 'vdev->pm_save').
2560 	 */
2561 	list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list)
2562 		vfio_pci_set_power_state(cur, PCI_D0);
2563 
2564 	ret = pci_reset_bus(pdev);
2565 
2566 err_undo:
2567 	list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
2568 		if (cur == cur_mem)
2569 			is_mem = false;
2570 		if (cur == cur_vma)
2571 			break;
2572 		if (is_mem)
2573 			up_write(&cur->memory_lock);
2574 		else
2575 			mutex_unlock(&cur->vma_lock);
2576 	}
2577 
2578 	list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list)
2579 		pm_runtime_put(&cur->pdev->dev);
2580 err_unlock:
2581 	mutex_unlock(&dev_set->lock);
2582 	return ret;
2583 }
2584 
vfio_pci_dev_set_needs_reset(struct vfio_device_set * dev_set)2585 static bool vfio_pci_dev_set_needs_reset(struct vfio_device_set *dev_set)
2586 {
2587 	struct vfio_pci_core_device *cur;
2588 	bool needs_reset = false;
2589 
2590 	/* No other VFIO device in the set can be open. */
2591 	if (vfio_device_set_open_count(dev_set) > 1)
2592 		return false;
2593 
2594 	list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list)
2595 		needs_reset |= cur->needs_reset;
2596 	return needs_reset;
2597 }
2598 
2599 /*
2600  * If a bus or slot reset is available for the provided dev_set and:
2601  *  - All of the devices affected by that bus or slot reset are unused
2602  *  - At least one of the affected devices is marked dirty via
2603  *    needs_reset (such as by lack of FLR support)
2604  * Then attempt to perform that bus or slot reset.
2605  */
vfio_pci_dev_set_try_reset(struct vfio_device_set * dev_set)2606 static void vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set)
2607 {
2608 	struct vfio_pci_core_device *cur;
2609 	struct pci_dev *pdev;
2610 	bool reset_done = false;
2611 
2612 	if (!vfio_pci_dev_set_needs_reset(dev_set))
2613 		return;
2614 
2615 	pdev = vfio_pci_dev_set_resettable(dev_set);
2616 	if (!pdev)
2617 		return;
2618 
2619 	/*
2620 	 * Some of the devices in the bus can be in the runtime suspended
2621 	 * state. Increment the usage count for all the devices in the dev_set
2622 	 * before reset and decrement the same after reset.
2623 	 */
2624 	if (!disable_idle_d3 && vfio_pci_dev_set_pm_runtime_get(dev_set))
2625 		return;
2626 
2627 	if (!pci_reset_bus(pdev))
2628 		reset_done = true;
2629 
2630 	list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
2631 		if (reset_done)
2632 			cur->needs_reset = false;
2633 
2634 		if (!disable_idle_d3)
2635 			pm_runtime_put(&cur->pdev->dev);
2636 	}
2637 }
2638 
vfio_pci_core_set_params(bool is_nointxmask,bool is_disable_vga,bool is_disable_idle_d3)2639 void vfio_pci_core_set_params(bool is_nointxmask, bool is_disable_vga,
2640 			      bool is_disable_idle_d3)
2641 {
2642 	nointxmask = is_nointxmask;
2643 	disable_vga = is_disable_vga;
2644 	disable_idle_d3 = is_disable_idle_d3;
2645 }
2646 EXPORT_SYMBOL_GPL(vfio_pci_core_set_params);
2647 
vfio_pci_core_cleanup(void)2648 static void vfio_pci_core_cleanup(void)
2649 {
2650 	vfio_pci_uninit_perm_bits();
2651 }
2652 
vfio_pci_core_init(void)2653 static int __init vfio_pci_core_init(void)
2654 {
2655 	/* Allocate shared config space permission data used by all devices */
2656 	return vfio_pci_init_perm_bits();
2657 }
2658 
2659 module_init(vfio_pci_core_init);
2660 module_exit(vfio_pci_core_cleanup);
2661 
2662 MODULE_LICENSE("GPL v2");
2663 MODULE_AUTHOR(DRIVER_AUTHOR);
2664 MODULE_DESCRIPTION(DRIVER_DESC);
2665