1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Adjunct processor matrix VFIO device driver callbacks.
4  *
5  * Copyright IBM Corp. 2018
6  *
7  * Author(s): Tony Krowiak <akrowiak@linux.ibm.com>
8  *	      Halil Pasic <pasic@linux.ibm.com>
9  *	      Pierre Morel <pmorel@linux.ibm.com>
10  */
11 #include <linux/string.h>
12 #include <linux/vfio.h>
13 #include <linux/device.h>
14 #include <linux/list.h>
15 #include <linux/ctype.h>
16 #include <linux/bitops.h>
17 #include <linux/kvm_host.h>
18 #include <linux/module.h>
19 #include <linux/uuid.h>
20 #include <asm/kvm.h>
21 #include <asm/zcrypt.h>
22 
23 #include "vfio_ap_private.h"
24 #include "vfio_ap_debug.h"
25 
26 #define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
27 #define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
28 
29 #define AP_QUEUE_ASSIGNED "assigned"
30 #define AP_QUEUE_UNASSIGNED "unassigned"
31 #define AP_QUEUE_IN_USE "in use"
32 
33 static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable);
34 static struct vfio_ap_queue *vfio_ap_find_queue(int apqn);
35 static const struct vfio_device_ops vfio_ap_matrix_dev_ops;
36 static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q, unsigned int retry);
37 
38 /**
39  * get_update_locks_for_kvm: Acquire the locks required to dynamically update a
40  *			     KVM guest's APCB in the proper order.
41  *
42  * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB.
43  *
44  * The proper locking order is:
45  * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
46  *			       guest's APCB.
47  * 2. kvm->lock:	       required to update a guest's APCB
48  * 3. matrix_dev->mdevs_lock:  required to access data stored in a matrix_mdev
49  *
50  * Note: If @kvm is NULL, the KVM lock will not be taken.
51  */
get_update_locks_for_kvm(struct kvm * kvm)52 static inline void get_update_locks_for_kvm(struct kvm *kvm)
53 {
54 	mutex_lock(&matrix_dev->guests_lock);
55 	if (kvm)
56 		mutex_lock(&kvm->lock);
57 	mutex_lock(&matrix_dev->mdevs_lock);
58 }
59 
60 /**
61  * release_update_locks_for_kvm: Release the locks used to dynamically update a
62  *				 KVM guest's APCB in the proper order.
63  *
64  * @kvm: a pointer to a struct kvm object containing the KVM guest's APCB.
65  *
66  * The proper unlocking order is:
67  * 1. matrix_dev->mdevs_lock
68  * 2. kvm->lock
69  * 3. matrix_dev->guests_lock
70  *
71  * Note: If @kvm is NULL, the KVM lock will not be released.
72  */
release_update_locks_for_kvm(struct kvm * kvm)73 static inline void release_update_locks_for_kvm(struct kvm *kvm)
74 {
75 	mutex_unlock(&matrix_dev->mdevs_lock);
76 	if (kvm)
77 		mutex_unlock(&kvm->lock);
78 	mutex_unlock(&matrix_dev->guests_lock);
79 }
80 
81 /**
82  * get_update_locks_for_mdev: Acquire the locks required to dynamically update a
83  *			      KVM guest's APCB in the proper order.
84  *
85  * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP
86  *		 configuration data to use to update a KVM guest's APCB.
87  *
88  * The proper locking order is:
89  * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
90  *			       guest's APCB.
91  * 2. matrix_mdev->kvm->lock:  required to update a guest's APCB
92  * 3. matrix_dev->mdevs_lock:  required to access data stored in a matrix_mdev
93  *
94  * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM
95  *	 lock will not be taken.
96  */
get_update_locks_for_mdev(struct ap_matrix_mdev * matrix_mdev)97 static inline void get_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev)
98 {
99 	mutex_lock(&matrix_dev->guests_lock);
100 	if (matrix_mdev && matrix_mdev->kvm)
101 		mutex_lock(&matrix_mdev->kvm->lock);
102 	mutex_lock(&matrix_dev->mdevs_lock);
103 }
104 
105 /**
106  * release_update_locks_for_mdev: Release the locks used to dynamically update a
107  *				  KVM guest's APCB in the proper order.
108  *
109  * @matrix_mdev: a pointer to a struct ap_matrix_mdev object containing the AP
110  *		 configuration data to use to update a KVM guest's APCB.
111  *
112  * The proper unlocking order is:
113  * 1. matrix_dev->mdevs_lock
114  * 2. matrix_mdev->kvm->lock
115  * 3. matrix_dev->guests_lock
116  *
117  * Note: If @matrix_mdev is NULL or is not attached to a KVM guest, the KVM
118  *	 lock will not be released.
119  */
release_update_locks_for_mdev(struct ap_matrix_mdev * matrix_mdev)120 static inline void release_update_locks_for_mdev(struct ap_matrix_mdev *matrix_mdev)
121 {
122 	mutex_unlock(&matrix_dev->mdevs_lock);
123 	if (matrix_mdev && matrix_mdev->kvm)
124 		mutex_unlock(&matrix_mdev->kvm->lock);
125 	mutex_unlock(&matrix_dev->guests_lock);
126 }
127 
128 /**
129  * get_update_locks_by_apqn: Find the mdev to which an APQN is assigned and
130  *			     acquire the locks required to update the APCB of
131  *			     the KVM guest to which the mdev is attached.
132  *
133  * @apqn: the APQN of a queue device.
134  *
135  * The proper locking order is:
136  * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM
137  *			       guest's APCB.
138  * 2. matrix_mdev->kvm->lock:  required to update a guest's APCB
139  * 3. matrix_dev->mdevs_lock:  required to access data stored in a matrix_mdev
140  *
141  * Note: If @apqn is not assigned to a matrix_mdev, the matrix_mdev->kvm->lock
142  *	 will not be taken.
143  *
144  * Return: the ap_matrix_mdev object to which @apqn is assigned or NULL if @apqn
145  *	   is not assigned to an ap_matrix_mdev.
146  */
get_update_locks_by_apqn(int apqn)147 static struct ap_matrix_mdev *get_update_locks_by_apqn(int apqn)
148 {
149 	struct ap_matrix_mdev *matrix_mdev;
150 
151 	mutex_lock(&matrix_dev->guests_lock);
152 
153 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
154 		if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) &&
155 		    test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) {
156 			if (matrix_mdev->kvm)
157 				mutex_lock(&matrix_mdev->kvm->lock);
158 
159 			mutex_lock(&matrix_dev->mdevs_lock);
160 
161 			return matrix_mdev;
162 		}
163 	}
164 
165 	mutex_lock(&matrix_dev->mdevs_lock);
166 
167 	return NULL;
168 }
169 
170 /**
171  * get_update_locks_for_queue: get the locks required to update the APCB of the
172  *			       KVM guest to which the matrix mdev linked to a
173  *			       vfio_ap_queue object is attached.
174  *
175  * @q: a pointer to a vfio_ap_queue object.
176  *
177  * The proper locking order is:
178  * 1. q->matrix_dev->guests_lock: required to use the KVM pointer to update a
179  *				  KVM guest's APCB.
180  * 2. q->matrix_mdev->kvm->lock:  required to update a guest's APCB
181  * 3. matrix_dev->mdevs_lock:	  required to access data stored in matrix_mdev
182  *
183  * Note: if @queue is not linked to an ap_matrix_mdev object, the KVM lock
184  *	  will not be taken.
185  */
get_update_locks_for_queue(struct vfio_ap_queue * q)186 static inline void get_update_locks_for_queue(struct vfio_ap_queue *q)
187 {
188 	mutex_lock(&matrix_dev->guests_lock);
189 	if (q->matrix_mdev && q->matrix_mdev->kvm)
190 		mutex_lock(&q->matrix_mdev->kvm->lock);
191 	mutex_lock(&matrix_dev->mdevs_lock);
192 }
193 
194 /**
195  * vfio_ap_mdev_get_queue - retrieve a queue with a specific APQN from a
196  *			    hash table of queues assigned to a matrix mdev
197  * @matrix_mdev: the matrix mdev
198  * @apqn: The APQN of a queue device
199  *
200  * Return: the pointer to the vfio_ap_queue struct representing the queue or
201  *	   NULL if the queue is not assigned to @matrix_mdev
202  */
vfio_ap_mdev_get_queue(struct ap_matrix_mdev * matrix_mdev,int apqn)203 static struct vfio_ap_queue *vfio_ap_mdev_get_queue(
204 					struct ap_matrix_mdev *matrix_mdev,
205 					int apqn)
206 {
207 	struct vfio_ap_queue *q;
208 
209 	hash_for_each_possible(matrix_mdev->qtable.queues, q, mdev_qnode,
210 			       apqn) {
211 		if (q && q->apqn == apqn)
212 			return q;
213 	}
214 
215 	return NULL;
216 }
217 
218 /**
219  * vfio_ap_wait_for_irqclear - clears the IR bit or gives up after 5 tries
220  * @apqn: The AP Queue number
221  *
222  * Checks the IRQ bit for the status of this APQN using ap_tapq.
223  * Returns if the ap_tapq function succeeded and the bit is clear.
224  * Returns if ap_tapq function failed with invalid, deconfigured or
225  * checkstopped AP.
226  * Otherwise retries up to 5 times after waiting 20ms.
227  */
vfio_ap_wait_for_irqclear(int apqn)228 static void vfio_ap_wait_for_irqclear(int apqn)
229 {
230 	struct ap_queue_status status;
231 	int retry = 5;
232 
233 	do {
234 		status = ap_tapq(apqn, NULL);
235 		switch (status.response_code) {
236 		case AP_RESPONSE_NORMAL:
237 		case AP_RESPONSE_RESET_IN_PROGRESS:
238 			if (!status.irq_enabled)
239 				return;
240 			fallthrough;
241 		case AP_RESPONSE_BUSY:
242 			msleep(20);
243 			break;
244 		case AP_RESPONSE_Q_NOT_AVAIL:
245 		case AP_RESPONSE_DECONFIGURED:
246 		case AP_RESPONSE_CHECKSTOPPED:
247 		default:
248 			WARN_ONCE(1, "%s: tapq rc %02x: %04x\n", __func__,
249 				  status.response_code, apqn);
250 			return;
251 		}
252 	} while (--retry);
253 
254 	WARN_ONCE(1, "%s: tapq rc %02x: %04x could not clear IR bit\n",
255 		  __func__, status.response_code, apqn);
256 }
257 
258 /**
259  * vfio_ap_free_aqic_resources - free vfio_ap_queue resources
260  * @q: The vfio_ap_queue
261  *
262  * Unregisters the ISC in the GIB when the saved ISC not invalid.
263  * Unpins the guest's page holding the NIB when it exists.
264  * Resets the saved_iova and saved_isc to invalid values.
265  */
vfio_ap_free_aqic_resources(struct vfio_ap_queue * q)266 static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q)
267 {
268 	if (!q)
269 		return;
270 	if (q->saved_isc != VFIO_AP_ISC_INVALID &&
271 	    !WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) {
272 		kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc);
273 		q->saved_isc = VFIO_AP_ISC_INVALID;
274 	}
275 	if (q->saved_iova && !WARN_ON(!q->matrix_mdev)) {
276 		vfio_unpin_pages(&q->matrix_mdev->vdev, q->saved_iova, 1);
277 		q->saved_iova = 0;
278 	}
279 }
280 
281 /**
282  * vfio_ap_irq_disable - disables and clears an ap_queue interrupt
283  * @q: The vfio_ap_queue
284  *
285  * Uses ap_aqic to disable the interruption and in case of success, reset
286  * in progress or IRQ disable command already proceeded: calls
287  * vfio_ap_wait_for_irqclear() to check for the IRQ bit to be clear
288  * and calls vfio_ap_free_aqic_resources() to free the resources associated
289  * with the AP interrupt handling.
290  *
291  * In the case the AP is busy, or a reset is in progress,
292  * retries after 20ms, up to 5 times.
293  *
294  * Returns if ap_aqic function failed with invalid, deconfigured or
295  * checkstopped AP.
296  *
297  * Return: &struct ap_queue_status
298  */
vfio_ap_irq_disable(struct vfio_ap_queue * q)299 static struct ap_queue_status vfio_ap_irq_disable(struct vfio_ap_queue *q)
300 {
301 	struct ap_qirq_ctrl aqic_gisa = {};
302 	struct ap_queue_status status;
303 	int retries = 5;
304 
305 	do {
306 		status = ap_aqic(q->apqn, aqic_gisa, 0);
307 		switch (status.response_code) {
308 		case AP_RESPONSE_OTHERWISE_CHANGED:
309 		case AP_RESPONSE_NORMAL:
310 			vfio_ap_wait_for_irqclear(q->apqn);
311 			goto end_free;
312 		case AP_RESPONSE_RESET_IN_PROGRESS:
313 		case AP_RESPONSE_BUSY:
314 			msleep(20);
315 			break;
316 		case AP_RESPONSE_Q_NOT_AVAIL:
317 		case AP_RESPONSE_DECONFIGURED:
318 		case AP_RESPONSE_CHECKSTOPPED:
319 		case AP_RESPONSE_INVALID_ADDRESS:
320 		default:
321 			/* All cases in default means AP not operational */
322 			WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__,
323 				  status.response_code);
324 			goto end_free;
325 		}
326 	} while (retries--);
327 
328 	WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__,
329 		  status.response_code);
330 end_free:
331 	vfio_ap_free_aqic_resources(q);
332 	return status;
333 }
334 
335 /**
336  * vfio_ap_validate_nib - validate a notification indicator byte (nib) address.
337  *
338  * @vcpu: the object representing the vcpu executing the PQAP(AQIC) instruction.
339  * @nib: the location for storing the nib address.
340  *
341  * When the PQAP(AQIC) instruction is executed, general register 2 contains the
342  * address of the notification indicator byte (nib) used for IRQ notification.
343  * This function parses and validates the nib from gr2.
344  *
345  * Return: returns zero if the nib address is a valid; otherwise, returns
346  *	   -EINVAL.
347  */
vfio_ap_validate_nib(struct kvm_vcpu * vcpu,dma_addr_t * nib)348 static int vfio_ap_validate_nib(struct kvm_vcpu *vcpu, dma_addr_t *nib)
349 {
350 	*nib = vcpu->run->s.regs.gprs[2];
351 
352 	if (kvm_is_error_hva(gfn_to_hva(vcpu->kvm, *nib >> PAGE_SHIFT)))
353 		return -EINVAL;
354 
355 	return 0;
356 }
357 
358 /**
359  * vfio_ap_irq_enable - Enable Interruption for a APQN
360  *
361  * @q:	 the vfio_ap_queue holding AQIC parameters
362  * @isc: the guest ISC to register with the GIB interface
363  * @vcpu: the vcpu object containing the registers specifying the parameters
364  *	  passed to the PQAP(AQIC) instruction.
365  *
366  * Pin the NIB saved in *q
367  * Register the guest ISC to GIB interface and retrieve the
368  * host ISC to issue the host side PQAP/AQIC
369  *
370  * Response.status may be set to AP_RESPONSE_INVALID_ADDRESS in case the
371  * vfio_pin_pages failed.
372  *
373  * Otherwise return the ap_queue_status returned by the ap_aqic(),
374  * all retry handling will be done by the guest.
375  *
376  * Return: &struct ap_queue_status
377  */
vfio_ap_irq_enable(struct vfio_ap_queue * q,int isc,struct kvm_vcpu * vcpu)378 static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q,
379 						 int isc,
380 						 struct kvm_vcpu *vcpu)
381 {
382 	struct ap_qirq_ctrl aqic_gisa = {};
383 	struct ap_queue_status status = {};
384 	struct kvm_s390_gisa *gisa;
385 	struct page *h_page;
386 	int nisc;
387 	struct kvm *kvm;
388 	phys_addr_t h_nib;
389 	dma_addr_t nib;
390 	int ret;
391 
392 	/* Verify that the notification indicator byte address is valid */
393 	if (vfio_ap_validate_nib(vcpu, &nib)) {
394 		VFIO_AP_DBF_WARN("%s: invalid NIB address: nib=%pad, apqn=%#04x\n",
395 				 __func__, &nib, q->apqn);
396 
397 		status.response_code = AP_RESPONSE_INVALID_ADDRESS;
398 		return status;
399 	}
400 
401 	ret = vfio_pin_pages(&q->matrix_mdev->vdev, nib, 1,
402 			     IOMMU_READ | IOMMU_WRITE, &h_page);
403 	switch (ret) {
404 	case 1:
405 		break;
406 	default:
407 		VFIO_AP_DBF_WARN("%s: vfio_pin_pages failed: rc=%d,"
408 				 "nib=%pad, apqn=%#04x\n",
409 				 __func__, ret, &nib, q->apqn);
410 
411 		status.response_code = AP_RESPONSE_INVALID_ADDRESS;
412 		return status;
413 	}
414 
415 	kvm = q->matrix_mdev->kvm;
416 	gisa = kvm->arch.gisa_int.origin;
417 
418 	h_nib = page_to_phys(h_page) | (nib & ~PAGE_MASK);
419 	aqic_gisa.gisc = isc;
420 
421 	nisc = kvm_s390_gisc_register(kvm, isc);
422 	if (nisc < 0) {
423 		VFIO_AP_DBF_WARN("%s: gisc registration failed: nisc=%d, isc=%d, apqn=%#04x\n",
424 				 __func__, nisc, isc, q->apqn);
425 
426 		status.response_code = AP_RESPONSE_INVALID_GISA;
427 		return status;
428 	}
429 
430 	aqic_gisa.isc = nisc;
431 	aqic_gisa.ir = 1;
432 	aqic_gisa.gisa = (uint64_t)gisa >> 4;
433 
434 	status = ap_aqic(q->apqn, aqic_gisa, h_nib);
435 	switch (status.response_code) {
436 	case AP_RESPONSE_NORMAL:
437 		/* See if we did clear older IRQ configuration */
438 		vfio_ap_free_aqic_resources(q);
439 		q->saved_iova = nib;
440 		q->saved_isc = isc;
441 		break;
442 	case AP_RESPONSE_OTHERWISE_CHANGED:
443 		/* We could not modify IRQ setings: clear new configuration */
444 		vfio_unpin_pages(&q->matrix_mdev->vdev, nib, 1);
445 		kvm_s390_gisc_unregister(kvm, isc);
446 		break;
447 	default:
448 		pr_warn("%s: apqn %04x: response: %02x\n", __func__, q->apqn,
449 			status.response_code);
450 		vfio_ap_irq_disable(q);
451 		break;
452 	}
453 
454 	if (status.response_code != AP_RESPONSE_NORMAL) {
455 		VFIO_AP_DBF_WARN("%s: PQAP(AQIC) failed with status=%#02x: "
456 				 "zone=%#x, ir=%#x, gisc=%#x, f=%#x,"
457 				 "gisa=%#x, isc=%#x, apqn=%#04x\n",
458 				 __func__, status.response_code,
459 				 aqic_gisa.zone, aqic_gisa.ir, aqic_gisa.gisc,
460 				 aqic_gisa.gf, aqic_gisa.gisa, aqic_gisa.isc,
461 				 q->apqn);
462 	}
463 
464 	return status;
465 }
466 
467 /**
468  * vfio_ap_le_guid_to_be_uuid - convert a little endian guid array into an array
469  *				of big endian elements that can be passed by
470  *				value to an s390dbf sprintf event function to
471  *				format a UUID string.
472  *
473  * @guid: the object containing the little endian guid
474  * @uuid: a six-element array of long values that can be passed by value as
475  *	  arguments for a formatting string specifying a UUID.
476  *
477  * The S390 Debug Feature (s390dbf) allows the use of "%s" in the sprintf
478  * event functions if the memory for the passed string is available as long as
479  * the debug feature exists. Since a mediated device can be removed at any
480  * time, it's name can not be used because %s passes the reference to the string
481  * in memory and the reference will go stale once the device is removed .
482  *
483  * The s390dbf string formatting function allows a maximum of 9 arguments for a
484  * message to be displayed in the 'sprintf' view. In order to use the bytes
485  * comprising the mediated device's UUID to display the mediated device name,
486  * they will have to be converted into an array whose elements can be passed by
487  * value to sprintf. For example:
488  *
489  * guid array: { 83, 78, 17, 62, bb, f1, f0, 47, 91, 4d, 32, a2, 2e, 3a, 88, 04 }
490  * mdev name: 62177883-f1bb-47f0-914d-32a22e3a8804
491  * array returned: { 62177883, f1bb, 47f0, 914d, 32a2, 2e3a8804 }
492  * formatting string: "%08lx-%04lx-%04lx-%04lx-%02lx%04lx"
493  */
vfio_ap_le_guid_to_be_uuid(guid_t * guid,unsigned long * uuid)494 static void vfio_ap_le_guid_to_be_uuid(guid_t *guid, unsigned long *uuid)
495 {
496 	/*
497 	 * The input guid is ordered in little endian, so it needs to be
498 	 * reordered for displaying a UUID as a string. This specifies the
499 	 * guid indices in proper order.
500 	 */
501 	uuid[0] = le32_to_cpup((__le32 *)guid);
502 	uuid[1] = le16_to_cpup((__le16 *)&guid->b[4]);
503 	uuid[2] = le16_to_cpup((__le16 *)&guid->b[6]);
504 	uuid[3] = *((__u16 *)&guid->b[8]);
505 	uuid[4] = *((__u16 *)&guid->b[10]);
506 	uuid[5] = *((__u32 *)&guid->b[12]);
507 }
508 
509 /**
510  * handle_pqap - PQAP instruction callback
511  *
512  * @vcpu: The vcpu on which we received the PQAP instruction
513  *
514  * Get the general register contents to initialize internal variables.
515  * REG[0]: APQN
516  * REG[1]: IR and ISC
517  * REG[2]: NIB
518  *
519  * Response.status may be set to following Response Code:
520  * - AP_RESPONSE_Q_NOT_AVAIL: if the queue is not available
521  * - AP_RESPONSE_DECONFIGURED: if the queue is not configured
522  * - AP_RESPONSE_NORMAL (0) : in case of successs
523  *   Check vfio_ap_setirq() and vfio_ap_clrirq() for other possible RC.
524  * We take the matrix_dev lock to ensure serialization on queues and
525  * mediated device access.
526  *
527  * Return: 0 if we could handle the request inside KVM.
528  * Otherwise, returns -EOPNOTSUPP to let QEMU handle the fault.
529  */
handle_pqap(struct kvm_vcpu * vcpu)530 static int handle_pqap(struct kvm_vcpu *vcpu)
531 {
532 	uint64_t status;
533 	uint16_t apqn;
534 	unsigned long uuid[6];
535 	struct vfio_ap_queue *q;
536 	struct ap_queue_status qstatus = {
537 			       .response_code = AP_RESPONSE_Q_NOT_AVAIL, };
538 	struct ap_matrix_mdev *matrix_mdev;
539 
540 	apqn = vcpu->run->s.regs.gprs[0] & 0xffff;
541 
542 	/* If we do not use the AIV facility just go to userland */
543 	if (!(vcpu->arch.sie_block->eca & ECA_AIV)) {
544 		VFIO_AP_DBF_WARN("%s: AIV facility not installed: apqn=0x%04x, eca=0x%04x\n",
545 				 __func__, apqn, vcpu->arch.sie_block->eca);
546 
547 		return -EOPNOTSUPP;
548 	}
549 
550 	mutex_lock(&matrix_dev->mdevs_lock);
551 
552 	if (!vcpu->kvm->arch.crypto.pqap_hook) {
553 		VFIO_AP_DBF_WARN("%s: PQAP(AQIC) hook not registered with the vfio_ap driver: apqn=0x%04x\n",
554 				 __func__, apqn);
555 
556 		goto out_unlock;
557 	}
558 
559 	matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook,
560 				   struct ap_matrix_mdev, pqap_hook);
561 
562 	/* If the there is no guest using the mdev, there is nothing to do */
563 	if (!matrix_mdev->kvm) {
564 		vfio_ap_le_guid_to_be_uuid(&matrix_mdev->mdev->uuid, uuid);
565 		VFIO_AP_DBF_WARN("%s: mdev %08lx-%04lx-%04lx-%04lx-%04lx%08lx not in use: apqn=0x%04x\n",
566 				 __func__, uuid[0],  uuid[1], uuid[2],
567 				 uuid[3], uuid[4], uuid[5], apqn);
568 		goto out_unlock;
569 	}
570 
571 	q = vfio_ap_mdev_get_queue(matrix_mdev, apqn);
572 	if (!q) {
573 		VFIO_AP_DBF_WARN("%s: Queue %02x.%04x not bound to the vfio_ap driver\n",
574 				 __func__, AP_QID_CARD(apqn),
575 				 AP_QID_QUEUE(apqn));
576 		goto out_unlock;
577 	}
578 
579 	status = vcpu->run->s.regs.gprs[1];
580 
581 	/* If IR bit(16) is set we enable the interrupt */
582 	if ((status >> (63 - 16)) & 0x01)
583 		qstatus = vfio_ap_irq_enable(q, status & 0x07, vcpu);
584 	else
585 		qstatus = vfio_ap_irq_disable(q);
586 
587 out_unlock:
588 	memcpy(&vcpu->run->s.regs.gprs[1], &qstatus, sizeof(qstatus));
589 	vcpu->run->s.regs.gprs[1] >>= 32;
590 	mutex_unlock(&matrix_dev->mdevs_lock);
591 	return 0;
592 }
593 
vfio_ap_matrix_init(struct ap_config_info * info,struct ap_matrix * matrix)594 static void vfio_ap_matrix_init(struct ap_config_info *info,
595 				struct ap_matrix *matrix)
596 {
597 	matrix->apm_max = info->apxa ? info->Na : 63;
598 	matrix->aqm_max = info->apxa ? info->Nd : 15;
599 	matrix->adm_max = info->apxa ? info->Nd : 15;
600 }
601 
vfio_ap_mdev_update_guest_apcb(struct ap_matrix_mdev * matrix_mdev)602 static void vfio_ap_mdev_update_guest_apcb(struct ap_matrix_mdev *matrix_mdev)
603 {
604 	if (matrix_mdev->kvm)
605 		kvm_arch_crypto_set_masks(matrix_mdev->kvm,
606 					  matrix_mdev->shadow_apcb.apm,
607 					  matrix_mdev->shadow_apcb.aqm,
608 					  matrix_mdev->shadow_apcb.adm);
609 }
610 
vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev * matrix_mdev)611 static bool vfio_ap_mdev_filter_cdoms(struct ap_matrix_mdev *matrix_mdev)
612 {
613 	DECLARE_BITMAP(prev_shadow_adm, AP_DOMAINS);
614 
615 	bitmap_copy(prev_shadow_adm, matrix_mdev->shadow_apcb.adm, AP_DOMAINS);
616 	bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm,
617 		   (unsigned long *)matrix_dev->info.adm, AP_DOMAINS);
618 
619 	return !bitmap_equal(prev_shadow_adm, matrix_mdev->shadow_apcb.adm,
620 			     AP_DOMAINS);
621 }
622 
623 /*
624  * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev
625  *				to ensure no queue devices are passed through to
626  *				the guest that are not bound to the vfio_ap
627  *				device driver.
628  *
629  * @matrix_mdev: the matrix mdev whose matrix is to be filtered.
630  *
631  * Note: If an APQN referencing a queue device that is not bound to the vfio_ap
632  *	 driver, its APID will be filtered from the guest's APCB. The matrix
633  *	 structure precludes filtering an individual APQN, so its APID will be
634  *	 filtered.
635  *
636  * Return: a boolean value indicating whether the KVM guest's APCB was changed
637  *	   by the filtering or not.
638  */
vfio_ap_mdev_filter_matrix(unsigned long * apm,unsigned long * aqm,struct ap_matrix_mdev * matrix_mdev)639 static bool vfio_ap_mdev_filter_matrix(unsigned long *apm, unsigned long *aqm,
640 				       struct ap_matrix_mdev *matrix_mdev)
641 {
642 	unsigned long apid, apqi, apqn;
643 	DECLARE_BITMAP(prev_shadow_apm, AP_DEVICES);
644 	DECLARE_BITMAP(prev_shadow_aqm, AP_DOMAINS);
645 	struct vfio_ap_queue *q;
646 
647 	bitmap_copy(prev_shadow_apm, matrix_mdev->shadow_apcb.apm, AP_DEVICES);
648 	bitmap_copy(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm, AP_DOMAINS);
649 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb);
650 
651 	/*
652 	 * Copy the adapters, domains and control domains to the shadow_apcb
653 	 * from the matrix mdev, but only those that are assigned to the host's
654 	 * AP configuration.
655 	 */
656 	bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm,
657 		   (unsigned long *)matrix_dev->info.apm, AP_DEVICES);
658 	bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm,
659 		   (unsigned long *)matrix_dev->info.aqm, AP_DOMAINS);
660 
661 	for_each_set_bit_inv(apid, apm, AP_DEVICES) {
662 		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) {
663 			/*
664 			 * If the APQN is not bound to the vfio_ap device
665 			 * driver, then we can't assign it to the guest's
666 			 * AP configuration. The AP architecture won't
667 			 * allow filtering of a single APQN, so let's filter
668 			 * the APID since an adapter represents a physical
669 			 * hardware device.
670 			 */
671 			apqn = AP_MKQID(apid, apqi);
672 			q = vfio_ap_mdev_get_queue(matrix_mdev, apqn);
673 			if (!q || q->reset_rc) {
674 				clear_bit_inv(apid,
675 					      matrix_mdev->shadow_apcb.apm);
676 				break;
677 			}
678 		}
679 	}
680 
681 	return !bitmap_equal(prev_shadow_apm, matrix_mdev->shadow_apcb.apm,
682 			     AP_DEVICES) ||
683 	       !bitmap_equal(prev_shadow_aqm, matrix_mdev->shadow_apcb.aqm,
684 			     AP_DOMAINS);
685 }
686 
vfio_ap_mdev_init_dev(struct vfio_device * vdev)687 static int vfio_ap_mdev_init_dev(struct vfio_device *vdev)
688 {
689 	struct ap_matrix_mdev *matrix_mdev =
690 		container_of(vdev, struct ap_matrix_mdev, vdev);
691 
692 	matrix_mdev->mdev = to_mdev_device(vdev->dev);
693 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);
694 	matrix_mdev->pqap_hook = handle_pqap;
695 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->shadow_apcb);
696 	hash_init(matrix_mdev->qtable.queues);
697 
698 	return 0;
699 }
700 
vfio_ap_mdev_probe(struct mdev_device * mdev)701 static int vfio_ap_mdev_probe(struct mdev_device *mdev)
702 {
703 	struct ap_matrix_mdev *matrix_mdev;
704 	int ret;
705 
706 	matrix_mdev = vfio_alloc_device(ap_matrix_mdev, vdev, &mdev->dev,
707 					&vfio_ap_matrix_dev_ops);
708 	if (IS_ERR(matrix_mdev))
709 		return PTR_ERR(matrix_mdev);
710 
711 	ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev);
712 	if (ret)
713 		goto err_put_vdev;
714 	dev_set_drvdata(&mdev->dev, matrix_mdev);
715 	mutex_lock(&matrix_dev->mdevs_lock);
716 	list_add(&matrix_mdev->node, &matrix_dev->mdev_list);
717 	mutex_unlock(&matrix_dev->mdevs_lock);
718 	return 0;
719 
720 err_put_vdev:
721 	vfio_put_device(&matrix_mdev->vdev);
722 	return ret;
723 }
724 
vfio_ap_mdev_link_queue(struct ap_matrix_mdev * matrix_mdev,struct vfio_ap_queue * q)725 static void vfio_ap_mdev_link_queue(struct ap_matrix_mdev *matrix_mdev,
726 				    struct vfio_ap_queue *q)
727 {
728 	if (q) {
729 		q->matrix_mdev = matrix_mdev;
730 		hash_add(matrix_mdev->qtable.queues, &q->mdev_qnode, q->apqn);
731 	}
732 }
733 
vfio_ap_mdev_link_apqn(struct ap_matrix_mdev * matrix_mdev,int apqn)734 static void vfio_ap_mdev_link_apqn(struct ap_matrix_mdev *matrix_mdev, int apqn)
735 {
736 	struct vfio_ap_queue *q;
737 
738 	q = vfio_ap_find_queue(apqn);
739 	vfio_ap_mdev_link_queue(matrix_mdev, q);
740 }
741 
vfio_ap_unlink_queue_fr_mdev(struct vfio_ap_queue * q)742 static void vfio_ap_unlink_queue_fr_mdev(struct vfio_ap_queue *q)
743 {
744 	hash_del(&q->mdev_qnode);
745 }
746 
vfio_ap_unlink_mdev_fr_queue(struct vfio_ap_queue * q)747 static void vfio_ap_unlink_mdev_fr_queue(struct vfio_ap_queue *q)
748 {
749 	q->matrix_mdev = NULL;
750 }
751 
vfio_ap_mdev_unlink_fr_queues(struct ap_matrix_mdev * matrix_mdev)752 static void vfio_ap_mdev_unlink_fr_queues(struct ap_matrix_mdev *matrix_mdev)
753 {
754 	struct vfio_ap_queue *q;
755 	unsigned long apid, apqi;
756 
757 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {
758 		for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,
759 				     AP_DOMAINS) {
760 			q = vfio_ap_mdev_get_queue(matrix_mdev,
761 						   AP_MKQID(apid, apqi));
762 			if (q)
763 				q->matrix_mdev = NULL;
764 		}
765 	}
766 }
767 
vfio_ap_mdev_release_dev(struct vfio_device * vdev)768 static void vfio_ap_mdev_release_dev(struct vfio_device *vdev)
769 {
770 	vfio_free_device(vdev);
771 }
772 
vfio_ap_mdev_remove(struct mdev_device * mdev)773 static void vfio_ap_mdev_remove(struct mdev_device *mdev)
774 {
775 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(&mdev->dev);
776 
777 	vfio_unregister_group_dev(&matrix_mdev->vdev);
778 
779 	mutex_lock(&matrix_dev->guests_lock);
780 	mutex_lock(&matrix_dev->mdevs_lock);
781 	vfio_ap_mdev_reset_queues(&matrix_mdev->qtable);
782 	vfio_ap_mdev_unlink_fr_queues(matrix_mdev);
783 	list_del(&matrix_mdev->node);
784 	mutex_unlock(&matrix_dev->mdevs_lock);
785 	mutex_unlock(&matrix_dev->guests_lock);
786 	vfio_put_device(&matrix_mdev->vdev);
787 }
788 
789 #define MDEV_SHARING_ERR "Userspace may not re-assign queue %02lx.%04lx " \
790 			 "already assigned to %s"
791 
vfio_ap_mdev_log_sharing_err(struct ap_matrix_mdev * matrix_mdev,unsigned long * apm,unsigned long * aqm)792 static void vfio_ap_mdev_log_sharing_err(struct ap_matrix_mdev *matrix_mdev,
793 					 unsigned long *apm,
794 					 unsigned long *aqm)
795 {
796 	unsigned long apid, apqi;
797 	const struct device *dev = mdev_dev(matrix_mdev->mdev);
798 	const char *mdev_name = dev_name(dev);
799 
800 	for_each_set_bit_inv(apid, apm, AP_DEVICES)
801 		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS)
802 			dev_warn(dev, MDEV_SHARING_ERR, apid, apqi, mdev_name);
803 }
804 
805 /**
806  * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs
807  *
808  * @mdev_apm: mask indicating the APIDs of the APQNs to be verified
809  * @mdev_aqm: mask indicating the APQIs of the APQNs to be verified
810  *
811  * Verifies that each APQN derived from the Cartesian product of a bitmap of
812  * AP adapter IDs and AP queue indexes is not configured for any matrix
813  * mediated device. AP queue sharing is not allowed.
814  *
815  * Return: 0 if the APQNs are not shared; otherwise return -EADDRINUSE.
816  */
vfio_ap_mdev_verify_no_sharing(unsigned long * mdev_apm,unsigned long * mdev_aqm)817 static int vfio_ap_mdev_verify_no_sharing(unsigned long *mdev_apm,
818 					  unsigned long *mdev_aqm)
819 {
820 	struct ap_matrix_mdev *matrix_mdev;
821 	DECLARE_BITMAP(apm, AP_DEVICES);
822 	DECLARE_BITMAP(aqm, AP_DOMAINS);
823 
824 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
825 		/*
826 		 * If the input apm and aqm are fields of the matrix_mdev
827 		 * object, then move on to the next matrix_mdev.
828 		 */
829 		if (mdev_apm == matrix_mdev->matrix.apm &&
830 		    mdev_aqm == matrix_mdev->matrix.aqm)
831 			continue;
832 
833 		memset(apm, 0, sizeof(apm));
834 		memset(aqm, 0, sizeof(aqm));
835 
836 		/*
837 		 * We work on full longs, as we can only exclude the leftover
838 		 * bits in non-inverse order. The leftover is all zeros.
839 		 */
840 		if (!bitmap_and(apm, mdev_apm, matrix_mdev->matrix.apm,
841 				AP_DEVICES))
842 			continue;
843 
844 		if (!bitmap_and(aqm, mdev_aqm, matrix_mdev->matrix.aqm,
845 				AP_DOMAINS))
846 			continue;
847 
848 		vfio_ap_mdev_log_sharing_err(matrix_mdev, apm, aqm);
849 
850 		return -EADDRINUSE;
851 	}
852 
853 	return 0;
854 }
855 
856 /**
857  * vfio_ap_mdev_validate_masks - verify that the APQNs assigned to the mdev are
858  *				 not reserved for the default zcrypt driver and
859  *				 are not assigned to another mdev.
860  *
861  * @matrix_mdev: the mdev to which the APQNs being validated are assigned.
862  *
863  * Return: One of the following values:
864  * o the error returned from the ap_apqn_in_matrix_owned_by_def_drv() function,
865  *   most likely -EBUSY indicating the ap_perms_mutex lock is already held.
866  * o EADDRNOTAVAIL if an APQN assigned to @matrix_mdev is reserved for the
867  *		   zcrypt default driver.
868  * o EADDRINUSE if an APQN assigned to @matrix_mdev is assigned to another mdev
869  * o A zero indicating validation succeeded.
870  */
vfio_ap_mdev_validate_masks(struct ap_matrix_mdev * matrix_mdev)871 static int vfio_ap_mdev_validate_masks(struct ap_matrix_mdev *matrix_mdev)
872 {
873 	if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm,
874 					       matrix_mdev->matrix.aqm))
875 		return -EADDRNOTAVAIL;
876 
877 	return vfio_ap_mdev_verify_no_sharing(matrix_mdev->matrix.apm,
878 					      matrix_mdev->matrix.aqm);
879 }
880 
vfio_ap_mdev_link_adapter(struct ap_matrix_mdev * matrix_mdev,unsigned long apid)881 static void vfio_ap_mdev_link_adapter(struct ap_matrix_mdev *matrix_mdev,
882 				      unsigned long apid)
883 {
884 	unsigned long apqi;
885 
886 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS)
887 		vfio_ap_mdev_link_apqn(matrix_mdev,
888 				       AP_MKQID(apid, apqi));
889 }
890 
891 /**
892  * assign_adapter_store - parses the APID from @buf and sets the
893  * corresponding bit in the mediated matrix device's APM
894  *
895  * @dev:	the matrix device
896  * @attr:	the mediated matrix device's assign_adapter attribute
897  * @buf:	a buffer containing the AP adapter number (APID) to
898  *		be assigned
899  * @count:	the number of bytes in @buf
900  *
901  * Return: the number of bytes processed if the APID is valid; otherwise,
902  * returns one of the following errors:
903  *
904  *	1. -EINVAL
905  *	   The APID is not a valid number
906  *
907  *	2. -ENODEV
908  *	   The APID exceeds the maximum value configured for the system
909  *
910  *	3. -EADDRNOTAVAIL
911  *	   An APQN derived from the cross product of the APID being assigned
912  *	   and the APQIs previously assigned is not bound to the vfio_ap device
913  *	   driver; or, if no APQIs have yet been assigned, the APID is not
914  *	   contained in an APQN bound to the vfio_ap device driver.
915  *
916  *	4. -EADDRINUSE
917  *	   An APQN derived from the cross product of the APID being assigned
918  *	   and the APQIs previously assigned is being used by another mediated
919  *	   matrix device
920  *
921  *	5. -EAGAIN
922  *	   A lock required to validate the mdev's AP configuration could not
923  *	   be obtained.
924  */
assign_adapter_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)925 static ssize_t assign_adapter_store(struct device *dev,
926 				    struct device_attribute *attr,
927 				    const char *buf, size_t count)
928 {
929 	int ret;
930 	unsigned long apid;
931 	DECLARE_BITMAP(apm_delta, AP_DEVICES);
932 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
933 
934 	mutex_lock(&ap_perms_mutex);
935 	get_update_locks_for_mdev(matrix_mdev);
936 
937 	ret = kstrtoul(buf, 0, &apid);
938 	if (ret)
939 		goto done;
940 
941 	if (apid > matrix_mdev->matrix.apm_max) {
942 		ret = -ENODEV;
943 		goto done;
944 	}
945 
946 	if (test_bit_inv(apid, matrix_mdev->matrix.apm)) {
947 		ret = count;
948 		goto done;
949 	}
950 
951 	set_bit_inv(apid, matrix_mdev->matrix.apm);
952 
953 	ret = vfio_ap_mdev_validate_masks(matrix_mdev);
954 	if (ret) {
955 		clear_bit_inv(apid, matrix_mdev->matrix.apm);
956 		goto done;
957 	}
958 
959 	vfio_ap_mdev_link_adapter(matrix_mdev, apid);
960 	memset(apm_delta, 0, sizeof(apm_delta));
961 	set_bit_inv(apid, apm_delta);
962 
963 	if (vfio_ap_mdev_filter_matrix(apm_delta,
964 				       matrix_mdev->matrix.aqm, matrix_mdev))
965 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
966 
967 	ret = count;
968 done:
969 	release_update_locks_for_mdev(matrix_mdev);
970 	mutex_unlock(&ap_perms_mutex);
971 
972 	return ret;
973 }
974 static DEVICE_ATTR_WO(assign_adapter);
975 
976 static struct vfio_ap_queue
vfio_ap_unlink_apqn_fr_mdev(struct ap_matrix_mdev * matrix_mdev,unsigned long apid,unsigned long apqi)977 *vfio_ap_unlink_apqn_fr_mdev(struct ap_matrix_mdev *matrix_mdev,
978 			     unsigned long apid, unsigned long apqi)
979 {
980 	struct vfio_ap_queue *q = NULL;
981 
982 	q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi));
983 	/* If the queue is assigned to the matrix mdev, unlink it. */
984 	if (q)
985 		vfio_ap_unlink_queue_fr_mdev(q);
986 
987 	return q;
988 }
989 
990 /**
991  * vfio_ap_mdev_unlink_adapter - unlink all queues associated with unassigned
992  *				 adapter from the matrix mdev to which the
993  *				 adapter was assigned.
994  * @matrix_mdev: the matrix mediated device to which the adapter was assigned.
995  * @apid: the APID of the unassigned adapter.
996  * @qtable: table for storing queues associated with unassigned adapter.
997  */
vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev * matrix_mdev,unsigned long apid,struct ap_queue_table * qtable)998 static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev,
999 					unsigned long apid,
1000 					struct ap_queue_table *qtable)
1001 {
1002 	unsigned long apqi;
1003 	struct vfio_ap_queue *q;
1004 
1005 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) {
1006 		q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi);
1007 
1008 		if (q && qtable) {
1009 			if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
1010 			    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm))
1011 				hash_add(qtable->queues, &q->mdev_qnode,
1012 					 q->apqn);
1013 		}
1014 	}
1015 }
1016 
vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev * matrix_mdev,unsigned long apid)1017 static void vfio_ap_mdev_hot_unplug_adapter(struct ap_matrix_mdev *matrix_mdev,
1018 					    unsigned long apid)
1019 {
1020 	int loop_cursor;
1021 	struct vfio_ap_queue *q;
1022 	struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL);
1023 
1024 	hash_init(qtable->queues);
1025 	vfio_ap_mdev_unlink_adapter(matrix_mdev, apid, qtable);
1026 
1027 	if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm)) {
1028 		clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm);
1029 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1030 	}
1031 
1032 	vfio_ap_mdev_reset_queues(qtable);
1033 
1034 	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1035 		vfio_ap_unlink_mdev_fr_queue(q);
1036 		hash_del(&q->mdev_qnode);
1037 	}
1038 
1039 	kfree(qtable);
1040 }
1041 
1042 /**
1043  * unassign_adapter_store - parses the APID from @buf and clears the
1044  * corresponding bit in the mediated matrix device's APM
1045  *
1046  * @dev:	the matrix device
1047  * @attr:	the mediated matrix device's unassign_adapter attribute
1048  * @buf:	a buffer containing the adapter number (APID) to be unassigned
1049  * @count:	the number of bytes in @buf
1050  *
1051  * Return: the number of bytes processed if the APID is valid; otherwise,
1052  * returns one of the following errors:
1053  *	-EINVAL if the APID is not a number
1054  *	-ENODEV if the APID it exceeds the maximum value configured for the
1055  *		system
1056  */
unassign_adapter_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1057 static ssize_t unassign_adapter_store(struct device *dev,
1058 				      struct device_attribute *attr,
1059 				      const char *buf, size_t count)
1060 {
1061 	int ret;
1062 	unsigned long apid;
1063 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1064 
1065 	get_update_locks_for_mdev(matrix_mdev);
1066 
1067 	ret = kstrtoul(buf, 0, &apid);
1068 	if (ret)
1069 		goto done;
1070 
1071 	if (apid > matrix_mdev->matrix.apm_max) {
1072 		ret = -ENODEV;
1073 		goto done;
1074 	}
1075 
1076 	if (!test_bit_inv(apid, matrix_mdev->matrix.apm)) {
1077 		ret = count;
1078 		goto done;
1079 	}
1080 
1081 	clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);
1082 	vfio_ap_mdev_hot_unplug_adapter(matrix_mdev, apid);
1083 	ret = count;
1084 done:
1085 	release_update_locks_for_mdev(matrix_mdev);
1086 	return ret;
1087 }
1088 static DEVICE_ATTR_WO(unassign_adapter);
1089 
vfio_ap_mdev_link_domain(struct ap_matrix_mdev * matrix_mdev,unsigned long apqi)1090 static void vfio_ap_mdev_link_domain(struct ap_matrix_mdev *matrix_mdev,
1091 				     unsigned long apqi)
1092 {
1093 	unsigned long apid;
1094 
1095 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES)
1096 		vfio_ap_mdev_link_apqn(matrix_mdev,
1097 				       AP_MKQID(apid, apqi));
1098 }
1099 
1100 /**
1101  * assign_domain_store - parses the APQI from @buf and sets the
1102  * corresponding bit in the mediated matrix device's AQM
1103  *
1104  * @dev:	the matrix device
1105  * @attr:	the mediated matrix device's assign_domain attribute
1106  * @buf:	a buffer containing the AP queue index (APQI) of the domain to
1107  *		be assigned
1108  * @count:	the number of bytes in @buf
1109  *
1110  * Return: the number of bytes processed if the APQI is valid; otherwise returns
1111  * one of the following errors:
1112  *
1113  *	1. -EINVAL
1114  *	   The APQI is not a valid number
1115  *
1116  *	2. -ENODEV
1117  *	   The APQI exceeds the maximum value configured for the system
1118  *
1119  *	3. -EADDRNOTAVAIL
1120  *	   An APQN derived from the cross product of the APQI being assigned
1121  *	   and the APIDs previously assigned is not bound to the vfio_ap device
1122  *	   driver; or, if no APIDs have yet been assigned, the APQI is not
1123  *	   contained in an APQN bound to the vfio_ap device driver.
1124  *
1125  *	4. -EADDRINUSE
1126  *	   An APQN derived from the cross product of the APQI being assigned
1127  *	   and the APIDs previously assigned is being used by another mediated
1128  *	   matrix device
1129  *
1130  *	5. -EAGAIN
1131  *	   The lock required to validate the mdev's AP configuration could not
1132  *	   be obtained.
1133  */
assign_domain_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1134 static ssize_t assign_domain_store(struct device *dev,
1135 				   struct device_attribute *attr,
1136 				   const char *buf, size_t count)
1137 {
1138 	int ret;
1139 	unsigned long apqi;
1140 	DECLARE_BITMAP(aqm_delta, AP_DOMAINS);
1141 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1142 
1143 	mutex_lock(&ap_perms_mutex);
1144 	get_update_locks_for_mdev(matrix_mdev);
1145 
1146 	ret = kstrtoul(buf, 0, &apqi);
1147 	if (ret)
1148 		goto done;
1149 
1150 	if (apqi > matrix_mdev->matrix.aqm_max) {
1151 		ret = -ENODEV;
1152 		goto done;
1153 	}
1154 
1155 	if (test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {
1156 		ret = count;
1157 		goto done;
1158 	}
1159 
1160 	set_bit_inv(apqi, matrix_mdev->matrix.aqm);
1161 
1162 	ret = vfio_ap_mdev_validate_masks(matrix_mdev);
1163 	if (ret) {
1164 		clear_bit_inv(apqi, matrix_mdev->matrix.aqm);
1165 		goto done;
1166 	}
1167 
1168 	vfio_ap_mdev_link_domain(matrix_mdev, apqi);
1169 	memset(aqm_delta, 0, sizeof(aqm_delta));
1170 	set_bit_inv(apqi, aqm_delta);
1171 
1172 	if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, aqm_delta,
1173 				       matrix_mdev))
1174 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1175 
1176 	ret = count;
1177 done:
1178 	release_update_locks_for_mdev(matrix_mdev);
1179 	mutex_unlock(&ap_perms_mutex);
1180 
1181 	return ret;
1182 }
1183 static DEVICE_ATTR_WO(assign_domain);
1184 
vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev * matrix_mdev,unsigned long apqi,struct ap_queue_table * qtable)1185 static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev,
1186 				       unsigned long apqi,
1187 				       struct ap_queue_table *qtable)
1188 {
1189 	unsigned long apid;
1190 	struct vfio_ap_queue *q;
1191 
1192 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {
1193 		q = vfio_ap_unlink_apqn_fr_mdev(matrix_mdev, apid, apqi);
1194 
1195 		if (q && qtable) {
1196 			if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
1197 			    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm))
1198 				hash_add(qtable->queues, &q->mdev_qnode,
1199 					 q->apqn);
1200 		}
1201 	}
1202 }
1203 
vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev * matrix_mdev,unsigned long apqi)1204 static void vfio_ap_mdev_hot_unplug_domain(struct ap_matrix_mdev *matrix_mdev,
1205 					   unsigned long apqi)
1206 {
1207 	int loop_cursor;
1208 	struct vfio_ap_queue *q;
1209 	struct ap_queue_table *qtable = kzalloc(sizeof(*qtable), GFP_KERNEL);
1210 
1211 	hash_init(qtable->queues);
1212 	vfio_ap_mdev_unlink_domain(matrix_mdev, apqi, qtable);
1213 
1214 	if (test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) {
1215 		clear_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm);
1216 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1217 	}
1218 
1219 	vfio_ap_mdev_reset_queues(qtable);
1220 
1221 	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1222 		vfio_ap_unlink_mdev_fr_queue(q);
1223 		hash_del(&q->mdev_qnode);
1224 	}
1225 
1226 	kfree(qtable);
1227 }
1228 
1229 /**
1230  * unassign_domain_store - parses the APQI from @buf and clears the
1231  * corresponding bit in the mediated matrix device's AQM
1232  *
1233  * @dev:	the matrix device
1234  * @attr:	the mediated matrix device's unassign_domain attribute
1235  * @buf:	a buffer containing the AP queue index (APQI) of the domain to
1236  *		be unassigned
1237  * @count:	the number of bytes in @buf
1238  *
1239  * Return: the number of bytes processed if the APQI is valid; otherwise,
1240  * returns one of the following errors:
1241  *	-EINVAL if the APQI is not a number
1242  *	-ENODEV if the APQI exceeds the maximum value configured for the system
1243  */
unassign_domain_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1244 static ssize_t unassign_domain_store(struct device *dev,
1245 				     struct device_attribute *attr,
1246 				     const char *buf, size_t count)
1247 {
1248 	int ret;
1249 	unsigned long apqi;
1250 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1251 
1252 	get_update_locks_for_mdev(matrix_mdev);
1253 
1254 	ret = kstrtoul(buf, 0, &apqi);
1255 	if (ret)
1256 		goto done;
1257 
1258 	if (apqi > matrix_mdev->matrix.aqm_max) {
1259 		ret = -ENODEV;
1260 		goto done;
1261 	}
1262 
1263 	if (!test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {
1264 		ret = count;
1265 		goto done;
1266 	}
1267 
1268 	clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);
1269 	vfio_ap_mdev_hot_unplug_domain(matrix_mdev, apqi);
1270 	ret = count;
1271 
1272 done:
1273 	release_update_locks_for_mdev(matrix_mdev);
1274 	return ret;
1275 }
1276 static DEVICE_ATTR_WO(unassign_domain);
1277 
1278 /**
1279  * assign_control_domain_store - parses the domain ID from @buf and sets
1280  * the corresponding bit in the mediated matrix device's ADM
1281  *
1282  * @dev:	the matrix device
1283  * @attr:	the mediated matrix device's assign_control_domain attribute
1284  * @buf:	a buffer containing the domain ID to be assigned
1285  * @count:	the number of bytes in @buf
1286  *
1287  * Return: the number of bytes processed if the domain ID is valid; otherwise,
1288  * returns one of the following errors:
1289  *	-EINVAL if the ID is not a number
1290  *	-ENODEV if the ID exceeds the maximum value configured for the system
1291  */
assign_control_domain_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1292 static ssize_t assign_control_domain_store(struct device *dev,
1293 					   struct device_attribute *attr,
1294 					   const char *buf, size_t count)
1295 {
1296 	int ret;
1297 	unsigned long id;
1298 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1299 
1300 	get_update_locks_for_mdev(matrix_mdev);
1301 
1302 	ret = kstrtoul(buf, 0, &id);
1303 	if (ret)
1304 		goto done;
1305 
1306 	if (id > matrix_mdev->matrix.adm_max) {
1307 		ret = -ENODEV;
1308 		goto done;
1309 	}
1310 
1311 	if (test_bit_inv(id, matrix_mdev->matrix.adm)) {
1312 		ret = count;
1313 		goto done;
1314 	}
1315 
1316 	/* Set the bit in the ADM (bitmask) corresponding to the AP control
1317 	 * domain number (id). The bits in the mask, from most significant to
1318 	 * least significant, correspond to IDs 0 up to the one less than the
1319 	 * number of control domains that can be assigned.
1320 	 */
1321 	set_bit_inv(id, matrix_mdev->matrix.adm);
1322 	if (vfio_ap_mdev_filter_cdoms(matrix_mdev))
1323 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1324 
1325 	ret = count;
1326 done:
1327 	release_update_locks_for_mdev(matrix_mdev);
1328 	return ret;
1329 }
1330 static DEVICE_ATTR_WO(assign_control_domain);
1331 
1332 /**
1333  * unassign_control_domain_store - parses the domain ID from @buf and
1334  * clears the corresponding bit in the mediated matrix device's ADM
1335  *
1336  * @dev:	the matrix device
1337  * @attr:	the mediated matrix device's unassign_control_domain attribute
1338  * @buf:	a buffer containing the domain ID to be unassigned
1339  * @count:	the number of bytes in @buf
1340  *
1341  * Return: the number of bytes processed if the domain ID is valid; otherwise,
1342  * returns one of the following errors:
1343  *	-EINVAL if the ID is not a number
1344  *	-ENODEV if the ID exceeds the maximum value configured for the system
1345  */
unassign_control_domain_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1346 static ssize_t unassign_control_domain_store(struct device *dev,
1347 					     struct device_attribute *attr,
1348 					     const char *buf, size_t count)
1349 {
1350 	int ret;
1351 	unsigned long domid;
1352 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1353 
1354 	get_update_locks_for_mdev(matrix_mdev);
1355 
1356 	ret = kstrtoul(buf, 0, &domid);
1357 	if (ret)
1358 		goto done;
1359 
1360 	if (domid > matrix_mdev->matrix.adm_max) {
1361 		ret = -ENODEV;
1362 		goto done;
1363 	}
1364 
1365 	if (!test_bit_inv(domid, matrix_mdev->matrix.adm)) {
1366 		ret = count;
1367 		goto done;
1368 	}
1369 
1370 	clear_bit_inv(domid, matrix_mdev->matrix.adm);
1371 
1372 	if (test_bit_inv(domid, matrix_mdev->shadow_apcb.adm)) {
1373 		clear_bit_inv(domid, matrix_mdev->shadow_apcb.adm);
1374 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1375 	}
1376 
1377 	ret = count;
1378 done:
1379 	release_update_locks_for_mdev(matrix_mdev);
1380 	return ret;
1381 }
1382 static DEVICE_ATTR_WO(unassign_control_domain);
1383 
control_domains_show(struct device * dev,struct device_attribute * dev_attr,char * buf)1384 static ssize_t control_domains_show(struct device *dev,
1385 				    struct device_attribute *dev_attr,
1386 				    char *buf)
1387 {
1388 	unsigned long id;
1389 	int nchars = 0;
1390 	int n;
1391 	char *bufpos = buf;
1392 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1393 	unsigned long max_domid = matrix_mdev->matrix.adm_max;
1394 
1395 	mutex_lock(&matrix_dev->mdevs_lock);
1396 	for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) {
1397 		n = sprintf(bufpos, "%04lx\n", id);
1398 		bufpos += n;
1399 		nchars += n;
1400 	}
1401 	mutex_unlock(&matrix_dev->mdevs_lock);
1402 
1403 	return nchars;
1404 }
1405 static DEVICE_ATTR_RO(control_domains);
1406 
vfio_ap_mdev_matrix_show(struct ap_matrix * matrix,char * buf)1407 static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf)
1408 {
1409 	char *bufpos = buf;
1410 	unsigned long apid;
1411 	unsigned long apqi;
1412 	unsigned long apid1;
1413 	unsigned long apqi1;
1414 	unsigned long napm_bits = matrix->apm_max + 1;
1415 	unsigned long naqm_bits = matrix->aqm_max + 1;
1416 	int nchars = 0;
1417 	int n;
1418 
1419 	apid1 = find_first_bit_inv(matrix->apm, napm_bits);
1420 	apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits);
1421 
1422 	if ((apid1 < napm_bits) && (apqi1 < naqm_bits)) {
1423 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {
1424 			for_each_set_bit_inv(apqi, matrix->aqm,
1425 					     naqm_bits) {
1426 				n = sprintf(bufpos, "%02lx.%04lx\n", apid,
1427 					    apqi);
1428 				bufpos += n;
1429 				nchars += n;
1430 			}
1431 		}
1432 	} else if (apid1 < napm_bits) {
1433 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {
1434 			n = sprintf(bufpos, "%02lx.\n", apid);
1435 			bufpos += n;
1436 			nchars += n;
1437 		}
1438 	} else if (apqi1 < naqm_bits) {
1439 		for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) {
1440 			n = sprintf(bufpos, ".%04lx\n", apqi);
1441 			bufpos += n;
1442 			nchars += n;
1443 		}
1444 	}
1445 
1446 	return nchars;
1447 }
1448 
matrix_show(struct device * dev,struct device_attribute * attr,char * buf)1449 static ssize_t matrix_show(struct device *dev, struct device_attribute *attr,
1450 			   char *buf)
1451 {
1452 	ssize_t nchars;
1453 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1454 
1455 	mutex_lock(&matrix_dev->mdevs_lock);
1456 	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf);
1457 	mutex_unlock(&matrix_dev->mdevs_lock);
1458 
1459 	return nchars;
1460 }
1461 static DEVICE_ATTR_RO(matrix);
1462 
guest_matrix_show(struct device * dev,struct device_attribute * attr,char * buf)1463 static ssize_t guest_matrix_show(struct device *dev,
1464 				 struct device_attribute *attr, char *buf)
1465 {
1466 	ssize_t nchars;
1467 	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1468 
1469 	mutex_lock(&matrix_dev->mdevs_lock);
1470 	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->shadow_apcb, buf);
1471 	mutex_unlock(&matrix_dev->mdevs_lock);
1472 
1473 	return nchars;
1474 }
1475 static DEVICE_ATTR_RO(guest_matrix);
1476 
1477 static struct attribute *vfio_ap_mdev_attrs[] = {
1478 	&dev_attr_assign_adapter.attr,
1479 	&dev_attr_unassign_adapter.attr,
1480 	&dev_attr_assign_domain.attr,
1481 	&dev_attr_unassign_domain.attr,
1482 	&dev_attr_assign_control_domain.attr,
1483 	&dev_attr_unassign_control_domain.attr,
1484 	&dev_attr_control_domains.attr,
1485 	&dev_attr_matrix.attr,
1486 	&dev_attr_guest_matrix.attr,
1487 	NULL,
1488 };
1489 
1490 static struct attribute_group vfio_ap_mdev_attr_group = {
1491 	.attrs = vfio_ap_mdev_attrs
1492 };
1493 
1494 static const struct attribute_group *vfio_ap_mdev_attr_groups[] = {
1495 	&vfio_ap_mdev_attr_group,
1496 	NULL
1497 };
1498 
1499 /**
1500  * vfio_ap_mdev_set_kvm - sets all data for @matrix_mdev that are needed
1501  * to manage AP resources for the guest whose state is represented by @kvm
1502  *
1503  * @matrix_mdev: a mediated matrix device
1504  * @kvm: reference to KVM instance
1505  *
1506  * Return: 0 if no other mediated matrix device has a reference to @kvm;
1507  * otherwise, returns an -EPERM.
1508  */
vfio_ap_mdev_set_kvm(struct ap_matrix_mdev * matrix_mdev,struct kvm * kvm)1509 static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev,
1510 				struct kvm *kvm)
1511 {
1512 	struct ap_matrix_mdev *m;
1513 
1514 	if (kvm->arch.crypto.crycbd) {
1515 		down_write(&kvm->arch.crypto.pqap_hook_rwsem);
1516 		kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;
1517 		up_write(&kvm->arch.crypto.pqap_hook_rwsem);
1518 
1519 		get_update_locks_for_kvm(kvm);
1520 
1521 		list_for_each_entry(m, &matrix_dev->mdev_list, node) {
1522 			if (m != matrix_mdev && m->kvm == kvm) {
1523 				release_update_locks_for_kvm(kvm);
1524 				return -EPERM;
1525 			}
1526 		}
1527 
1528 		kvm_get_kvm(kvm);
1529 		matrix_mdev->kvm = kvm;
1530 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1531 
1532 		release_update_locks_for_kvm(kvm);
1533 	}
1534 
1535 	return 0;
1536 }
1537 
vfio_ap_mdev_dma_unmap(struct vfio_device * vdev,u64 iova,u64 length)1538 static void vfio_ap_mdev_dma_unmap(struct vfio_device *vdev, u64 iova,
1539 				   u64 length)
1540 {
1541 	struct ap_matrix_mdev *matrix_mdev =
1542 		container_of(vdev, struct ap_matrix_mdev, vdev);
1543 
1544 	vfio_unpin_pages(&matrix_mdev->vdev, iova, 1);
1545 }
1546 
1547 /**
1548  * vfio_ap_mdev_unset_kvm - performs clean-up of resources no longer needed
1549  * by @matrix_mdev.
1550  *
1551  * @matrix_mdev: a matrix mediated device
1552  */
vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev * matrix_mdev)1553 static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev)
1554 {
1555 	struct kvm *kvm = matrix_mdev->kvm;
1556 
1557 	if (kvm && kvm->arch.crypto.crycbd) {
1558 		down_write(&kvm->arch.crypto.pqap_hook_rwsem);
1559 		kvm->arch.crypto.pqap_hook = NULL;
1560 		up_write(&kvm->arch.crypto.pqap_hook_rwsem);
1561 
1562 		get_update_locks_for_kvm(kvm);
1563 
1564 		kvm_arch_crypto_clear_masks(kvm);
1565 		vfio_ap_mdev_reset_queues(&matrix_mdev->qtable);
1566 		kvm_put_kvm(kvm);
1567 		matrix_mdev->kvm = NULL;
1568 
1569 		release_update_locks_for_kvm(kvm);
1570 	}
1571 }
1572 
vfio_ap_find_queue(int apqn)1573 static struct vfio_ap_queue *vfio_ap_find_queue(int apqn)
1574 {
1575 	struct ap_queue *queue;
1576 	struct vfio_ap_queue *q = NULL;
1577 
1578 	queue = ap_get_qdev(apqn);
1579 	if (!queue)
1580 		return NULL;
1581 
1582 	if (queue->ap_dev.device.driver == &matrix_dev->vfio_ap_drv->driver)
1583 		q = dev_get_drvdata(&queue->ap_dev.device);
1584 
1585 	put_device(&queue->ap_dev.device);
1586 
1587 	return q;
1588 }
1589 
vfio_ap_mdev_reset_queue(struct vfio_ap_queue * q,unsigned int retry)1590 static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q,
1591 				    unsigned int retry)
1592 {
1593 	struct ap_queue_status status;
1594 	int ret;
1595 	int retry2 = 2;
1596 
1597 	if (!q)
1598 		return 0;
1599 retry_zapq:
1600 	status = ap_zapq(q->apqn);
1601 	q->reset_rc = status.response_code;
1602 	switch (status.response_code) {
1603 	case AP_RESPONSE_NORMAL:
1604 		ret = 0;
1605 		break;
1606 	case AP_RESPONSE_RESET_IN_PROGRESS:
1607 		if (retry--) {
1608 			msleep(20);
1609 			goto retry_zapq;
1610 		}
1611 		ret = -EBUSY;
1612 		break;
1613 	case AP_RESPONSE_Q_NOT_AVAIL:
1614 	case AP_RESPONSE_DECONFIGURED:
1615 	case AP_RESPONSE_CHECKSTOPPED:
1616 		WARN_ONCE(status.irq_enabled,
1617 			  "PQAP/ZAPQ for %02x.%04x failed with rc=%u while IRQ enabled",
1618 			  AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn),
1619 			  status.response_code);
1620 		ret = -EBUSY;
1621 		goto free_resources;
1622 	default:
1623 		/* things are really broken, give up */
1624 		WARN(true,
1625 		     "PQAP/ZAPQ for %02x.%04x failed with invalid rc=%u\n",
1626 		     AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn),
1627 		     status.response_code);
1628 		return -EIO;
1629 	}
1630 
1631 	/* wait for the reset to take effect */
1632 	while (retry2--) {
1633 		if (status.queue_empty && !status.irq_enabled)
1634 			break;
1635 		msleep(20);
1636 		status = ap_tapq(q->apqn, NULL);
1637 	}
1638 	WARN_ONCE(retry2 <= 0, "unable to verify reset of queue %02x.%04x",
1639 		  AP_QID_CARD(q->apqn), AP_QID_QUEUE(q->apqn));
1640 
1641 free_resources:
1642 	vfio_ap_free_aqic_resources(q);
1643 
1644 	return ret;
1645 }
1646 
vfio_ap_mdev_reset_queues(struct ap_queue_table * qtable)1647 static int vfio_ap_mdev_reset_queues(struct ap_queue_table *qtable)
1648 {
1649 	int ret, loop_cursor, rc = 0;
1650 	struct vfio_ap_queue *q;
1651 
1652 	hash_for_each(qtable->queues, loop_cursor, q, mdev_qnode) {
1653 		ret = vfio_ap_mdev_reset_queue(q, 1);
1654 		/*
1655 		 * Regardless whether a queue turns out to be busy, or
1656 		 * is not operational, we need to continue resetting
1657 		 * the remaining queues.
1658 		 */
1659 		if (ret)
1660 			rc = ret;
1661 	}
1662 
1663 	return rc;
1664 }
1665 
vfio_ap_mdev_open_device(struct vfio_device * vdev)1666 static int vfio_ap_mdev_open_device(struct vfio_device *vdev)
1667 {
1668 	struct ap_matrix_mdev *matrix_mdev =
1669 		container_of(vdev, struct ap_matrix_mdev, vdev);
1670 
1671 	if (!vdev->kvm)
1672 		return -EINVAL;
1673 
1674 	return vfio_ap_mdev_set_kvm(matrix_mdev, vdev->kvm);
1675 }
1676 
vfio_ap_mdev_close_device(struct vfio_device * vdev)1677 static void vfio_ap_mdev_close_device(struct vfio_device *vdev)
1678 {
1679 	struct ap_matrix_mdev *matrix_mdev =
1680 		container_of(vdev, struct ap_matrix_mdev, vdev);
1681 
1682 	vfio_ap_mdev_unset_kvm(matrix_mdev);
1683 }
1684 
vfio_ap_mdev_get_device_info(unsigned long arg)1685 static int vfio_ap_mdev_get_device_info(unsigned long arg)
1686 {
1687 	unsigned long minsz;
1688 	struct vfio_device_info info;
1689 
1690 	minsz = offsetofend(struct vfio_device_info, num_irqs);
1691 
1692 	if (copy_from_user(&info, (void __user *)arg, minsz))
1693 		return -EFAULT;
1694 
1695 	if (info.argsz < minsz)
1696 		return -EINVAL;
1697 
1698 	info.flags = VFIO_DEVICE_FLAGS_AP | VFIO_DEVICE_FLAGS_RESET;
1699 	info.num_regions = 0;
1700 	info.num_irqs = 0;
1701 
1702 	return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
1703 }
1704 
vfio_ap_mdev_ioctl(struct vfio_device * vdev,unsigned int cmd,unsigned long arg)1705 static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev,
1706 				    unsigned int cmd, unsigned long arg)
1707 {
1708 	struct ap_matrix_mdev *matrix_mdev =
1709 		container_of(vdev, struct ap_matrix_mdev, vdev);
1710 	int ret;
1711 
1712 	mutex_lock(&matrix_dev->mdevs_lock);
1713 	switch (cmd) {
1714 	case VFIO_DEVICE_GET_INFO:
1715 		ret = vfio_ap_mdev_get_device_info(arg);
1716 		break;
1717 	case VFIO_DEVICE_RESET:
1718 		ret = vfio_ap_mdev_reset_queues(&matrix_mdev->qtable);
1719 		break;
1720 	default:
1721 		ret = -EOPNOTSUPP;
1722 		break;
1723 	}
1724 	mutex_unlock(&matrix_dev->mdevs_lock);
1725 
1726 	return ret;
1727 }
1728 
vfio_ap_mdev_for_queue(struct vfio_ap_queue * q)1729 static struct ap_matrix_mdev *vfio_ap_mdev_for_queue(struct vfio_ap_queue *q)
1730 {
1731 	struct ap_matrix_mdev *matrix_mdev;
1732 	unsigned long apid = AP_QID_CARD(q->apqn);
1733 	unsigned long apqi = AP_QID_QUEUE(q->apqn);
1734 
1735 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
1736 		if (test_bit_inv(apid, matrix_mdev->matrix.apm) &&
1737 		    test_bit_inv(apqi, matrix_mdev->matrix.aqm))
1738 			return matrix_mdev;
1739 	}
1740 
1741 	return NULL;
1742 }
1743 
status_show(struct device * dev,struct device_attribute * attr,char * buf)1744 static ssize_t status_show(struct device *dev,
1745 			   struct device_attribute *attr,
1746 			   char *buf)
1747 {
1748 	ssize_t nchars = 0;
1749 	struct vfio_ap_queue *q;
1750 	struct ap_matrix_mdev *matrix_mdev;
1751 	struct ap_device *apdev = to_ap_dev(dev);
1752 
1753 	mutex_lock(&matrix_dev->mdevs_lock);
1754 	q = dev_get_drvdata(&apdev->device);
1755 	matrix_mdev = vfio_ap_mdev_for_queue(q);
1756 
1757 	if (matrix_mdev) {
1758 		if (matrix_mdev->kvm)
1759 			nchars = scnprintf(buf, PAGE_SIZE, "%s\n",
1760 					   AP_QUEUE_IN_USE);
1761 		else
1762 			nchars = scnprintf(buf, PAGE_SIZE, "%s\n",
1763 					   AP_QUEUE_ASSIGNED);
1764 	} else {
1765 		nchars = scnprintf(buf, PAGE_SIZE, "%s\n",
1766 				   AP_QUEUE_UNASSIGNED);
1767 	}
1768 
1769 	mutex_unlock(&matrix_dev->mdevs_lock);
1770 
1771 	return nchars;
1772 }
1773 
1774 static DEVICE_ATTR_RO(status);
1775 
1776 static struct attribute *vfio_queue_attrs[] = {
1777 	&dev_attr_status.attr,
1778 	NULL,
1779 };
1780 
1781 static const struct attribute_group vfio_queue_attr_group = {
1782 	.attrs = vfio_queue_attrs,
1783 };
1784 
1785 static const struct vfio_device_ops vfio_ap_matrix_dev_ops = {
1786 	.init = vfio_ap_mdev_init_dev,
1787 	.release = vfio_ap_mdev_release_dev,
1788 	.open_device = vfio_ap_mdev_open_device,
1789 	.close_device = vfio_ap_mdev_close_device,
1790 	.ioctl = vfio_ap_mdev_ioctl,
1791 	.dma_unmap = vfio_ap_mdev_dma_unmap,
1792 };
1793 
1794 static struct mdev_driver vfio_ap_matrix_driver = {
1795 	.device_api = VFIO_DEVICE_API_AP_STRING,
1796 	.max_instances = MAX_ZDEV_ENTRIES_EXT,
1797 	.driver = {
1798 		.name = "vfio_ap_mdev",
1799 		.owner = THIS_MODULE,
1800 		.mod_name = KBUILD_MODNAME,
1801 		.dev_groups = vfio_ap_mdev_attr_groups,
1802 	},
1803 	.probe = vfio_ap_mdev_probe,
1804 	.remove = vfio_ap_mdev_remove,
1805 };
1806 
vfio_ap_mdev_register(void)1807 int vfio_ap_mdev_register(void)
1808 {
1809 	int ret;
1810 
1811 	ret = mdev_register_driver(&vfio_ap_matrix_driver);
1812 	if (ret)
1813 		return ret;
1814 
1815 	matrix_dev->mdev_type.sysfs_name = VFIO_AP_MDEV_TYPE_HWVIRT;
1816 	matrix_dev->mdev_type.pretty_name = VFIO_AP_MDEV_NAME_HWVIRT;
1817 	matrix_dev->mdev_types[0] = &matrix_dev->mdev_type;
1818 	ret = mdev_register_parent(&matrix_dev->parent, &matrix_dev->device,
1819 				   &vfio_ap_matrix_driver,
1820 				   matrix_dev->mdev_types, 1);
1821 	if (ret)
1822 		goto err_driver;
1823 	return 0;
1824 
1825 err_driver:
1826 	mdev_unregister_driver(&vfio_ap_matrix_driver);
1827 	return ret;
1828 }
1829 
vfio_ap_mdev_unregister(void)1830 void vfio_ap_mdev_unregister(void)
1831 {
1832 	mdev_unregister_parent(&matrix_dev->parent);
1833 	mdev_unregister_driver(&vfio_ap_matrix_driver);
1834 }
1835 
vfio_ap_mdev_probe_queue(struct ap_device * apdev)1836 int vfio_ap_mdev_probe_queue(struct ap_device *apdev)
1837 {
1838 	int ret;
1839 	struct vfio_ap_queue *q;
1840 	struct ap_matrix_mdev *matrix_mdev;
1841 
1842 	ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group);
1843 	if (ret)
1844 		return ret;
1845 
1846 	q = kzalloc(sizeof(*q), GFP_KERNEL);
1847 	if (!q)
1848 		return -ENOMEM;
1849 
1850 	q->apqn = to_ap_queue(&apdev->device)->qid;
1851 	q->saved_isc = VFIO_AP_ISC_INVALID;
1852 	matrix_mdev = get_update_locks_by_apqn(q->apqn);
1853 
1854 	if (matrix_mdev) {
1855 		vfio_ap_mdev_link_queue(matrix_mdev, q);
1856 
1857 		if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm,
1858 					       matrix_mdev->matrix.aqm,
1859 					       matrix_mdev))
1860 			vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1861 	}
1862 	dev_set_drvdata(&apdev->device, q);
1863 	release_update_locks_for_mdev(matrix_mdev);
1864 
1865 	return 0;
1866 }
1867 
vfio_ap_mdev_remove_queue(struct ap_device * apdev)1868 void vfio_ap_mdev_remove_queue(struct ap_device *apdev)
1869 {
1870 	unsigned long apid, apqi;
1871 	struct vfio_ap_queue *q;
1872 	struct ap_matrix_mdev *matrix_mdev;
1873 
1874 	sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group);
1875 	q = dev_get_drvdata(&apdev->device);
1876 	get_update_locks_for_queue(q);
1877 	matrix_mdev = q->matrix_mdev;
1878 
1879 	if (matrix_mdev) {
1880 		vfio_ap_unlink_queue_fr_mdev(q);
1881 
1882 		apid = AP_QID_CARD(q->apqn);
1883 		apqi = AP_QID_QUEUE(q->apqn);
1884 
1885 		/*
1886 		 * If the queue is assigned to the guest's APCB, then remove
1887 		 * the adapter's APID from the APCB and hot it into the guest.
1888 		 */
1889 		if (test_bit_inv(apid, matrix_mdev->shadow_apcb.apm) &&
1890 		    test_bit_inv(apqi, matrix_mdev->shadow_apcb.aqm)) {
1891 			clear_bit_inv(apid, matrix_mdev->shadow_apcb.apm);
1892 			vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1893 		}
1894 	}
1895 
1896 	vfio_ap_mdev_reset_queue(q, 1);
1897 	dev_set_drvdata(&apdev->device, NULL);
1898 	kfree(q);
1899 	release_update_locks_for_mdev(matrix_mdev);
1900 }
1901 
1902 /**
1903  * vfio_ap_mdev_resource_in_use: check whether any of a set of APQNs is
1904  *				 assigned to a mediated device under the control
1905  *				 of the vfio_ap device driver.
1906  *
1907  * @apm: a bitmap specifying a set of APIDs comprising the APQNs to check.
1908  * @aqm: a bitmap specifying a set of APQIs comprising the APQNs to check.
1909  *
1910  * Return:
1911  *	* -EADDRINUSE if one or more of the APQNs specified via @apm/@aqm are
1912  *	  assigned to a mediated device under the control of the vfio_ap
1913  *	  device driver.
1914  *	* Otherwise, return 0.
1915  */
vfio_ap_mdev_resource_in_use(unsigned long * apm,unsigned long * aqm)1916 int vfio_ap_mdev_resource_in_use(unsigned long *apm, unsigned long *aqm)
1917 {
1918 	int ret;
1919 
1920 	mutex_lock(&matrix_dev->guests_lock);
1921 	mutex_lock(&matrix_dev->mdevs_lock);
1922 	ret = vfio_ap_mdev_verify_no_sharing(apm, aqm);
1923 	mutex_unlock(&matrix_dev->mdevs_lock);
1924 	mutex_unlock(&matrix_dev->guests_lock);
1925 
1926 	return ret;
1927 }
1928 
1929 /**
1930  * vfio_ap_mdev_hot_unplug_cfg - hot unplug the adapters, domains and control
1931  *				 domains that have been removed from the host's
1932  *				 AP configuration from a guest.
1933  *
1934  * @matrix_mdev: an ap_matrix_mdev object attached to a KVM guest.
1935  * @aprem: the adapters that have been removed from the host's AP configuration
1936  * @aqrem: the domains that have been removed from the host's AP configuration
1937  * @cdrem: the control domains that have been removed from the host's AP
1938  *	   configuration.
1939  */
vfio_ap_mdev_hot_unplug_cfg(struct ap_matrix_mdev * matrix_mdev,unsigned long * aprem,unsigned long * aqrem,unsigned long * cdrem)1940 static void vfio_ap_mdev_hot_unplug_cfg(struct ap_matrix_mdev *matrix_mdev,
1941 					unsigned long *aprem,
1942 					unsigned long *aqrem,
1943 					unsigned long *cdrem)
1944 {
1945 	int do_hotplug = 0;
1946 
1947 	if (!bitmap_empty(aprem, AP_DEVICES)) {
1948 		do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.apm,
1949 					    matrix_mdev->shadow_apcb.apm,
1950 					    aprem, AP_DEVICES);
1951 	}
1952 
1953 	if (!bitmap_empty(aqrem, AP_DOMAINS)) {
1954 		do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.aqm,
1955 					    matrix_mdev->shadow_apcb.aqm,
1956 					    aqrem, AP_DEVICES);
1957 	}
1958 
1959 	if (!bitmap_empty(cdrem, AP_DOMAINS))
1960 		do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.adm,
1961 					    matrix_mdev->shadow_apcb.adm,
1962 					    cdrem, AP_DOMAINS);
1963 
1964 	if (do_hotplug)
1965 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
1966 }
1967 
1968 /**
1969  * vfio_ap_mdev_cfg_remove - determines which guests are using the adapters,
1970  *			     domains and control domains that have been removed
1971  *			     from the host AP configuration and unplugs them
1972  *			     from those guests.
1973  *
1974  * @ap_remove:	bitmap specifying which adapters have been removed from the host
1975  *		config.
1976  * @aq_remove:	bitmap specifying which domains have been removed from the host
1977  *		config.
1978  * @cd_remove:	bitmap specifying which control domains have been removed from
1979  *		the host config.
1980  */
vfio_ap_mdev_cfg_remove(unsigned long * ap_remove,unsigned long * aq_remove,unsigned long * cd_remove)1981 static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove,
1982 				    unsigned long *aq_remove,
1983 				    unsigned long *cd_remove)
1984 {
1985 	struct ap_matrix_mdev *matrix_mdev;
1986 	DECLARE_BITMAP(aprem, AP_DEVICES);
1987 	DECLARE_BITMAP(aqrem, AP_DOMAINS);
1988 	DECLARE_BITMAP(cdrem, AP_DOMAINS);
1989 	int do_remove = 0;
1990 
1991 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
1992 		mutex_lock(&matrix_mdev->kvm->lock);
1993 		mutex_lock(&matrix_dev->mdevs_lock);
1994 
1995 		do_remove |= bitmap_and(aprem, ap_remove,
1996 					  matrix_mdev->matrix.apm,
1997 					  AP_DEVICES);
1998 		do_remove |= bitmap_and(aqrem, aq_remove,
1999 					  matrix_mdev->matrix.aqm,
2000 					  AP_DOMAINS);
2001 		do_remove |= bitmap_andnot(cdrem, cd_remove,
2002 					     matrix_mdev->matrix.adm,
2003 					     AP_DOMAINS);
2004 
2005 		if (do_remove)
2006 			vfio_ap_mdev_hot_unplug_cfg(matrix_mdev, aprem, aqrem,
2007 						    cdrem);
2008 
2009 		mutex_unlock(&matrix_dev->mdevs_lock);
2010 		mutex_unlock(&matrix_mdev->kvm->lock);
2011 	}
2012 }
2013 
2014 /**
2015  * vfio_ap_mdev_on_cfg_remove - responds to the removal of adapters, domains and
2016  *				control domains from the host AP configuration
2017  *				by unplugging them from the guests that are
2018  *				using them.
2019  * @cur_config_info: the current host AP configuration information
2020  * @prev_config_info: the previous host AP configuration information
2021  */
vfio_ap_mdev_on_cfg_remove(struct ap_config_info * cur_config_info,struct ap_config_info * prev_config_info)2022 static void vfio_ap_mdev_on_cfg_remove(struct ap_config_info *cur_config_info,
2023 				       struct ap_config_info *prev_config_info)
2024 {
2025 	int do_remove;
2026 	DECLARE_BITMAP(aprem, AP_DEVICES);
2027 	DECLARE_BITMAP(aqrem, AP_DOMAINS);
2028 	DECLARE_BITMAP(cdrem, AP_DOMAINS);
2029 
2030 	do_remove = bitmap_andnot(aprem,
2031 				  (unsigned long *)prev_config_info->apm,
2032 				  (unsigned long *)cur_config_info->apm,
2033 				  AP_DEVICES);
2034 	do_remove |= bitmap_andnot(aqrem,
2035 				   (unsigned long *)prev_config_info->aqm,
2036 				   (unsigned long *)cur_config_info->aqm,
2037 				   AP_DEVICES);
2038 	do_remove |= bitmap_andnot(cdrem,
2039 				   (unsigned long *)prev_config_info->adm,
2040 				   (unsigned long *)cur_config_info->adm,
2041 				   AP_DEVICES);
2042 
2043 	if (do_remove)
2044 		vfio_ap_mdev_cfg_remove(aprem, aqrem, cdrem);
2045 }
2046 
2047 /**
2048  * vfio_ap_filter_apid_by_qtype: filter APIDs from an AP mask for adapters that
2049  *				 are older than AP type 10 (CEX4).
2050  * @apm: a bitmap of the APIDs to examine
2051  * @aqm: a bitmap of the APQIs of the queues to query for the AP type.
2052  */
vfio_ap_filter_apid_by_qtype(unsigned long * apm,unsigned long * aqm)2053 static void vfio_ap_filter_apid_by_qtype(unsigned long *apm, unsigned long *aqm)
2054 {
2055 	bool apid_cleared;
2056 	struct ap_queue_status status;
2057 	unsigned long apid, apqi, info;
2058 	int qtype, qtype_mask = 0xff000000;
2059 
2060 	for_each_set_bit_inv(apid, apm, AP_DEVICES) {
2061 		apid_cleared = false;
2062 
2063 		for_each_set_bit_inv(apqi, aqm, AP_DOMAINS) {
2064 			status = ap_test_queue(AP_MKQID(apid, apqi), 1, &info);
2065 			switch (status.response_code) {
2066 			/*
2067 			 * According to the architecture in each case
2068 			 * below, the queue's info should be filled.
2069 			 */
2070 			case AP_RESPONSE_NORMAL:
2071 			case AP_RESPONSE_RESET_IN_PROGRESS:
2072 			case AP_RESPONSE_DECONFIGURED:
2073 			case AP_RESPONSE_CHECKSTOPPED:
2074 			case AP_RESPONSE_BUSY:
2075 				qtype = info & qtype_mask;
2076 
2077 				/*
2078 				 * The vfio_ap device driver only
2079 				 * supports CEX4 and newer adapters, so
2080 				 * remove the APID if the adapter is
2081 				 * older than a CEX4.
2082 				 */
2083 				if (qtype < AP_DEVICE_TYPE_CEX4) {
2084 					clear_bit_inv(apid, apm);
2085 					apid_cleared = true;
2086 				}
2087 
2088 				break;
2089 
2090 			default:
2091 				/*
2092 				 * If we don't know the adapter type,
2093 				 * clear its APID since it can't be
2094 				 * determined whether the vfio_ap
2095 				 * device driver supports it.
2096 				 */
2097 				clear_bit_inv(apid, apm);
2098 				apid_cleared = true;
2099 				break;
2100 			}
2101 
2102 			/*
2103 			 * If we've already cleared the APID from the apm, there
2104 			 * is no need to continue examining the remainin AP
2105 			 * queues to determine the type of the adapter.
2106 			 */
2107 			if (apid_cleared)
2108 				continue;
2109 		}
2110 	}
2111 }
2112 
2113 /**
2114  * vfio_ap_mdev_cfg_add - store bitmaps specifying the adapters, domains and
2115  *			  control domains that have been added to the host's
2116  *			  AP configuration for each matrix mdev to which they
2117  *			  are assigned.
2118  *
2119  * @apm_add: a bitmap specifying the adapters that have been added to the AP
2120  *	     configuration.
2121  * @aqm_add: a bitmap specifying the domains that have been added to the AP
2122  *	     configuration.
2123  * @adm_add: a bitmap specifying the control domains that have been added to the
2124  *	     AP configuration.
2125  */
vfio_ap_mdev_cfg_add(unsigned long * apm_add,unsigned long * aqm_add,unsigned long * adm_add)2126 static void vfio_ap_mdev_cfg_add(unsigned long *apm_add, unsigned long *aqm_add,
2127 				 unsigned long *adm_add)
2128 {
2129 	struct ap_matrix_mdev *matrix_mdev;
2130 
2131 	if (list_empty(&matrix_dev->mdev_list))
2132 		return;
2133 
2134 	vfio_ap_filter_apid_by_qtype(apm_add, aqm_add);
2135 
2136 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
2137 		bitmap_and(matrix_mdev->apm_add,
2138 			   matrix_mdev->matrix.apm, apm_add, AP_DEVICES);
2139 		bitmap_and(matrix_mdev->aqm_add,
2140 			   matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS);
2141 		bitmap_and(matrix_mdev->adm_add,
2142 			   matrix_mdev->matrix.adm, adm_add, AP_DEVICES);
2143 	}
2144 }
2145 
2146 /**
2147  * vfio_ap_mdev_on_cfg_add - responds to the addition of adapters, domains and
2148  *			     control domains to the host AP configuration
2149  *			     by updating the bitmaps that specify what adapters,
2150  *			     domains and control domains have been added so they
2151  *			     can be hot plugged into the guest when the AP bus
2152  *			     scan completes (see vfio_ap_on_scan_complete
2153  *			     function).
2154  * @cur_config_info: the current AP configuration information
2155  * @prev_config_info: the previous AP configuration information
2156  */
vfio_ap_mdev_on_cfg_add(struct ap_config_info * cur_config_info,struct ap_config_info * prev_config_info)2157 static void vfio_ap_mdev_on_cfg_add(struct ap_config_info *cur_config_info,
2158 				    struct ap_config_info *prev_config_info)
2159 {
2160 	bool do_add;
2161 	DECLARE_BITMAP(apm_add, AP_DEVICES);
2162 	DECLARE_BITMAP(aqm_add, AP_DOMAINS);
2163 	DECLARE_BITMAP(adm_add, AP_DOMAINS);
2164 
2165 	do_add = bitmap_andnot(apm_add,
2166 			       (unsigned long *)cur_config_info->apm,
2167 			       (unsigned long *)prev_config_info->apm,
2168 			       AP_DEVICES);
2169 	do_add |= bitmap_andnot(aqm_add,
2170 				(unsigned long *)cur_config_info->aqm,
2171 				(unsigned long *)prev_config_info->aqm,
2172 				AP_DOMAINS);
2173 	do_add |= bitmap_andnot(adm_add,
2174 				(unsigned long *)cur_config_info->adm,
2175 				(unsigned long *)prev_config_info->adm,
2176 				AP_DOMAINS);
2177 
2178 	if (do_add)
2179 		vfio_ap_mdev_cfg_add(apm_add, aqm_add, adm_add);
2180 }
2181 
2182 /**
2183  * vfio_ap_on_cfg_changed - handles notification of changes to the host AP
2184  *			    configuration.
2185  *
2186  * @cur_cfg_info: the current host AP configuration
2187  * @prev_cfg_info: the previous host AP configuration
2188  */
vfio_ap_on_cfg_changed(struct ap_config_info * cur_cfg_info,struct ap_config_info * prev_cfg_info)2189 void vfio_ap_on_cfg_changed(struct ap_config_info *cur_cfg_info,
2190 			    struct ap_config_info *prev_cfg_info)
2191 {
2192 	if (!cur_cfg_info || !prev_cfg_info)
2193 		return;
2194 
2195 	mutex_lock(&matrix_dev->guests_lock);
2196 
2197 	vfio_ap_mdev_on_cfg_remove(cur_cfg_info, prev_cfg_info);
2198 	vfio_ap_mdev_on_cfg_add(cur_cfg_info, prev_cfg_info);
2199 	memcpy(&matrix_dev->info, cur_cfg_info, sizeof(*cur_cfg_info));
2200 
2201 	mutex_unlock(&matrix_dev->guests_lock);
2202 }
2203 
vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev * matrix_mdev)2204 static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev)
2205 {
2206 	bool do_hotplug = false;
2207 	int filter_domains = 0;
2208 	int filter_adapters = 0;
2209 	DECLARE_BITMAP(apm, AP_DEVICES);
2210 	DECLARE_BITMAP(aqm, AP_DOMAINS);
2211 
2212 	mutex_lock(&matrix_mdev->kvm->lock);
2213 	mutex_lock(&matrix_dev->mdevs_lock);
2214 
2215 	filter_adapters = bitmap_and(apm, matrix_mdev->matrix.apm,
2216 				     matrix_mdev->apm_add, AP_DEVICES);
2217 	filter_domains = bitmap_and(aqm, matrix_mdev->matrix.aqm,
2218 				    matrix_mdev->aqm_add, AP_DOMAINS);
2219 
2220 	if (filter_adapters && filter_domains)
2221 		do_hotplug |= vfio_ap_mdev_filter_matrix(apm, aqm, matrix_mdev);
2222 	else if (filter_adapters)
2223 		do_hotplug |=
2224 			vfio_ap_mdev_filter_matrix(apm,
2225 						   matrix_mdev->shadow_apcb.aqm,
2226 						   matrix_mdev);
2227 	else
2228 		do_hotplug |=
2229 			vfio_ap_mdev_filter_matrix(matrix_mdev->shadow_apcb.apm,
2230 						   aqm, matrix_mdev);
2231 
2232 	if (bitmap_intersects(matrix_mdev->matrix.adm, matrix_mdev->adm_add,
2233 			      AP_DOMAINS))
2234 		do_hotplug |= vfio_ap_mdev_filter_cdoms(matrix_mdev);
2235 
2236 	if (do_hotplug)
2237 		vfio_ap_mdev_update_guest_apcb(matrix_mdev);
2238 
2239 	mutex_unlock(&matrix_dev->mdevs_lock);
2240 	mutex_unlock(&matrix_mdev->kvm->lock);
2241 }
2242 
vfio_ap_on_scan_complete(struct ap_config_info * new_config_info,struct ap_config_info * old_config_info)2243 void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info,
2244 			      struct ap_config_info *old_config_info)
2245 {
2246 	struct ap_matrix_mdev *matrix_mdev;
2247 
2248 	mutex_lock(&matrix_dev->guests_lock);
2249 
2250 	list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
2251 		if (bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) &&
2252 		    bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS) &&
2253 		    bitmap_empty(matrix_mdev->adm_add, AP_DOMAINS))
2254 			continue;
2255 
2256 		vfio_ap_mdev_hot_plug_cfg(matrix_mdev);
2257 		bitmap_clear(matrix_mdev->apm_add, 0, AP_DEVICES);
2258 		bitmap_clear(matrix_mdev->aqm_add, 0, AP_DOMAINS);
2259 		bitmap_clear(matrix_mdev->adm_add, 0, AP_DOMAINS);
2260 	}
2261 
2262 	mutex_unlock(&matrix_dev->guests_lock);
2263 }
2264