1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright 2015 IBM Corp.
4  */
5 
6 #include <linux/spinlock.h>
7 #include <linux/uaccess.h>
8 #include <linux/delay.h>
9 #include <linux/irqdomain.h>
10 #include <linux/platform_device.h>
11 
12 #include "cxl.h"
13 #include "hcalls.h"
14 #include "trace.h"
15 
16 #define CXL_ERROR_DETECTED_EVENT	1
17 #define CXL_SLOT_RESET_EVENT		2
18 #define CXL_RESUME_EVENT		3
19 
pci_error_handlers(struct cxl_afu * afu,int bus_error_event,pci_channel_state_t state)20 static void pci_error_handlers(struct cxl_afu *afu,
21 				int bus_error_event,
22 				pci_channel_state_t state)
23 {
24 	struct pci_dev *afu_dev;
25 	struct pci_driver *afu_drv;
26 	const struct pci_error_handlers *err_handler;
27 
28 	if (afu->phb == NULL)
29 		return;
30 
31 	list_for_each_entry(afu_dev, &afu->phb->bus->devices, bus_list) {
32 		afu_drv = to_pci_driver(afu_dev->dev.driver);
33 		if (!afu_drv)
34 			continue;
35 
36 		err_handler = afu_drv->err_handler;
37 		switch (bus_error_event) {
38 		case CXL_ERROR_DETECTED_EVENT:
39 			afu_dev->error_state = state;
40 
41 			if (err_handler &&
42 			    err_handler->error_detected)
43 				err_handler->error_detected(afu_dev, state);
44 			break;
45 		case CXL_SLOT_RESET_EVENT:
46 			afu_dev->error_state = state;
47 
48 			if (err_handler &&
49 			    err_handler->slot_reset)
50 				err_handler->slot_reset(afu_dev);
51 			break;
52 		case CXL_RESUME_EVENT:
53 			if (err_handler &&
54 			    err_handler->resume)
55 				err_handler->resume(afu_dev);
56 			break;
57 		}
58 	}
59 }
60 
guest_handle_psl_slice_error(struct cxl_context * ctx,u64 dsisr,u64 errstat)61 static irqreturn_t guest_handle_psl_slice_error(struct cxl_context *ctx, u64 dsisr,
62 					u64 errstat)
63 {
64 	pr_devel("in %s\n", __func__);
65 	dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%.16llx\n", errstat);
66 
67 	return cxl_ops->ack_irq(ctx, 0, errstat);
68 }
69 
guest_collect_vpd(struct cxl * adapter,struct cxl_afu * afu,void * buf,size_t len)70 static ssize_t guest_collect_vpd(struct cxl *adapter, struct cxl_afu *afu,
71 			void *buf, size_t len)
72 {
73 	unsigned int entries, mod;
74 	unsigned long **vpd_buf = NULL;
75 	struct sg_list *le;
76 	int rc = 0, i, tocopy;
77 	u64 out = 0;
78 
79 	if (buf == NULL)
80 		return -EINVAL;
81 
82 	/* number of entries in the list */
83 	entries = len / SG_BUFFER_SIZE;
84 	mod = len % SG_BUFFER_SIZE;
85 	if (mod)
86 		entries++;
87 
88 	if (entries > SG_MAX_ENTRIES) {
89 		entries = SG_MAX_ENTRIES;
90 		len = SG_MAX_ENTRIES * SG_BUFFER_SIZE;
91 		mod = 0;
92 	}
93 
94 	vpd_buf = kcalloc(entries, sizeof(unsigned long *), GFP_KERNEL);
95 	if (!vpd_buf)
96 		return -ENOMEM;
97 
98 	le = (struct sg_list *)get_zeroed_page(GFP_KERNEL);
99 	if (!le) {
100 		rc = -ENOMEM;
101 		goto err1;
102 	}
103 
104 	for (i = 0; i < entries; i++) {
105 		vpd_buf[i] = (unsigned long *)get_zeroed_page(GFP_KERNEL);
106 		if (!vpd_buf[i]) {
107 			rc = -ENOMEM;
108 			goto err2;
109 		}
110 		le[i].phys_addr = cpu_to_be64(virt_to_phys(vpd_buf[i]));
111 		le[i].len = cpu_to_be64(SG_BUFFER_SIZE);
112 		if ((i == (entries - 1)) && mod)
113 			le[i].len = cpu_to_be64(mod);
114 	}
115 
116 	if (adapter)
117 		rc = cxl_h_collect_vpd_adapter(adapter->guest->handle,
118 					virt_to_phys(le), entries, &out);
119 	else
120 		rc = cxl_h_collect_vpd(afu->guest->handle, 0,
121 				virt_to_phys(le), entries, &out);
122 	pr_devel("length of available (entries: %i), vpd: %#llx\n",
123 		entries, out);
124 
125 	if (!rc) {
126 		/*
127 		 * hcall returns in 'out' the size of available VPDs.
128 		 * It fills the buffer with as much data as possible.
129 		 */
130 		if (out < len)
131 			len = out;
132 		rc = len;
133 		if (out) {
134 			for (i = 0; i < entries; i++) {
135 				if (len < SG_BUFFER_SIZE)
136 					tocopy = len;
137 				else
138 					tocopy = SG_BUFFER_SIZE;
139 				memcpy(buf, vpd_buf[i], tocopy);
140 				buf += tocopy;
141 				len -= tocopy;
142 			}
143 		}
144 	}
145 err2:
146 	for (i = 0; i < entries; i++) {
147 		if (vpd_buf[i])
148 			free_page((unsigned long) vpd_buf[i]);
149 	}
150 	free_page((unsigned long) le);
151 err1:
152 	kfree(vpd_buf);
153 	return rc;
154 }
155 
guest_get_irq_info(struct cxl_context * ctx,struct cxl_irq_info * info)156 static int guest_get_irq_info(struct cxl_context *ctx, struct cxl_irq_info *info)
157 {
158 	return cxl_h_collect_int_info(ctx->afu->guest->handle, ctx->process_token, info);
159 }
160 
guest_psl_irq(int irq,void * data)161 static irqreturn_t guest_psl_irq(int irq, void *data)
162 {
163 	struct cxl_context *ctx = data;
164 	struct cxl_irq_info irq_info;
165 	int rc;
166 
167 	pr_devel("%d: received PSL interrupt %i\n", ctx->pe, irq);
168 	rc = guest_get_irq_info(ctx, &irq_info);
169 	if (rc) {
170 		WARN(1, "Unable to get IRQ info: %i\n", rc);
171 		return IRQ_HANDLED;
172 	}
173 
174 	rc = cxl_irq_psl8(irq, ctx, &irq_info);
175 	return rc;
176 }
177 
afu_read_error_state(struct cxl_afu * afu,int * state_out)178 static int afu_read_error_state(struct cxl_afu *afu, int *state_out)
179 {
180 	u64 state;
181 	int rc = 0;
182 
183 	if (!afu)
184 		return -EIO;
185 
186 	rc = cxl_h_read_error_state(afu->guest->handle, &state);
187 	if (!rc) {
188 		WARN_ON(state != H_STATE_NORMAL &&
189 			state != H_STATE_DISABLE &&
190 			state != H_STATE_TEMP_UNAVAILABLE &&
191 			state != H_STATE_PERM_UNAVAILABLE);
192 		*state_out = state & 0xffffffff;
193 	}
194 	return rc;
195 }
196 
guest_slice_irq_err(int irq,void * data)197 static irqreturn_t guest_slice_irq_err(int irq, void *data)
198 {
199 	struct cxl_afu *afu = data;
200 	int rc;
201 	u64 serr, afu_error, dsisr;
202 
203 	rc = cxl_h_get_fn_error_interrupt(afu->guest->handle, &serr);
204 	if (rc) {
205 		dev_crit(&afu->dev, "Couldn't read PSL_SERR_An: %d\n", rc);
206 		return IRQ_HANDLED;
207 	}
208 	afu_error = cxl_p2n_read(afu, CXL_AFU_ERR_An);
209 	dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
210 	cxl_afu_decode_psl_serr(afu, serr);
211 	dev_crit(&afu->dev, "AFU_ERR_An: 0x%.16llx\n", afu_error);
212 	dev_crit(&afu->dev, "PSL_DSISR_An: 0x%.16llx\n", dsisr);
213 
214 	rc = cxl_h_ack_fn_error_interrupt(afu->guest->handle, serr);
215 	if (rc)
216 		dev_crit(&afu->dev, "Couldn't ack slice error interrupt: %d\n",
217 			rc);
218 
219 	return IRQ_HANDLED;
220 }
221 
222 
irq_alloc_range(struct cxl * adapter,int len,int * irq)223 static int irq_alloc_range(struct cxl *adapter, int len, int *irq)
224 {
225 	int i, n;
226 	struct irq_avail *cur;
227 
228 	for (i = 0; i < adapter->guest->irq_nranges; i++) {
229 		cur = &adapter->guest->irq_avail[i];
230 		n = bitmap_find_next_zero_area(cur->bitmap, cur->range,
231 					0, len, 0);
232 		if (n < cur->range) {
233 			bitmap_set(cur->bitmap, n, len);
234 			*irq = cur->offset + n;
235 			pr_devel("guest: allocate IRQs %#x->%#x\n",
236 				*irq, *irq + len - 1);
237 
238 			return 0;
239 		}
240 	}
241 	return -ENOSPC;
242 }
243 
irq_free_range(struct cxl * adapter,int irq,int len)244 static int irq_free_range(struct cxl *adapter, int irq, int len)
245 {
246 	int i, n;
247 	struct irq_avail *cur;
248 
249 	if (len == 0)
250 		return -ENOENT;
251 
252 	for (i = 0; i < adapter->guest->irq_nranges; i++) {
253 		cur = &adapter->guest->irq_avail[i];
254 		if (irq >= cur->offset &&
255 			(irq + len) <= (cur->offset + cur->range)) {
256 			n = irq - cur->offset;
257 			bitmap_clear(cur->bitmap, n, len);
258 			pr_devel("guest: release IRQs %#x->%#x\n",
259 				irq, irq + len - 1);
260 			return 0;
261 		}
262 	}
263 	return -ENOENT;
264 }
265 
guest_reset(struct cxl * adapter)266 static int guest_reset(struct cxl *adapter)
267 {
268 	struct cxl_afu *afu = NULL;
269 	int i, rc;
270 
271 	pr_devel("Adapter reset request\n");
272 	spin_lock(&adapter->afu_list_lock);
273 	for (i = 0; i < adapter->slices; i++) {
274 		if ((afu = adapter->afu[i])) {
275 			pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT,
276 					pci_channel_io_frozen);
277 			cxl_context_detach_all(afu);
278 		}
279 	}
280 
281 	rc = cxl_h_reset_adapter(adapter->guest->handle);
282 	for (i = 0; i < adapter->slices; i++) {
283 		if (!rc && (afu = adapter->afu[i])) {
284 			pci_error_handlers(afu, CXL_SLOT_RESET_EVENT,
285 					pci_channel_io_normal);
286 			pci_error_handlers(afu, CXL_RESUME_EVENT, 0);
287 		}
288 	}
289 	spin_unlock(&adapter->afu_list_lock);
290 	return rc;
291 }
292 
guest_alloc_one_irq(struct cxl * adapter)293 static int guest_alloc_one_irq(struct cxl *adapter)
294 {
295 	int irq;
296 
297 	spin_lock(&adapter->guest->irq_alloc_lock);
298 	if (irq_alloc_range(adapter, 1, &irq))
299 		irq = -ENOSPC;
300 	spin_unlock(&adapter->guest->irq_alloc_lock);
301 	return irq;
302 }
303 
guest_release_one_irq(struct cxl * adapter,int irq)304 static void guest_release_one_irq(struct cxl *adapter, int irq)
305 {
306 	spin_lock(&adapter->guest->irq_alloc_lock);
307 	irq_free_range(adapter, irq, 1);
308 	spin_unlock(&adapter->guest->irq_alloc_lock);
309 }
310 
guest_alloc_irq_ranges(struct cxl_irq_ranges * irqs,struct cxl * adapter,unsigned int num)311 static int guest_alloc_irq_ranges(struct cxl_irq_ranges *irqs,
312 				struct cxl *adapter, unsigned int num)
313 {
314 	int i, try, irq;
315 
316 	memset(irqs, 0, sizeof(struct cxl_irq_ranges));
317 
318 	spin_lock(&adapter->guest->irq_alloc_lock);
319 	for (i = 0; i < CXL_IRQ_RANGES && num; i++) {
320 		try = num;
321 		while (try) {
322 			if (irq_alloc_range(adapter, try, &irq) == 0)
323 				break;
324 			try /= 2;
325 		}
326 		if (!try)
327 			goto error;
328 		irqs->offset[i] = irq;
329 		irqs->range[i] = try;
330 		num -= try;
331 	}
332 	if (num)
333 		goto error;
334 	spin_unlock(&adapter->guest->irq_alloc_lock);
335 	return 0;
336 
337 error:
338 	for (i = 0; i < CXL_IRQ_RANGES; i++)
339 		irq_free_range(adapter, irqs->offset[i], irqs->range[i]);
340 	spin_unlock(&adapter->guest->irq_alloc_lock);
341 	return -ENOSPC;
342 }
343 
guest_release_irq_ranges(struct cxl_irq_ranges * irqs,struct cxl * adapter)344 static void guest_release_irq_ranges(struct cxl_irq_ranges *irqs,
345 				struct cxl *adapter)
346 {
347 	int i;
348 
349 	spin_lock(&adapter->guest->irq_alloc_lock);
350 	for (i = 0; i < CXL_IRQ_RANGES; i++)
351 		irq_free_range(adapter, irqs->offset[i], irqs->range[i]);
352 	spin_unlock(&adapter->guest->irq_alloc_lock);
353 }
354 
guest_register_serr_irq(struct cxl_afu * afu)355 static int guest_register_serr_irq(struct cxl_afu *afu)
356 {
357 	afu->err_irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err",
358 				      dev_name(&afu->dev));
359 	if (!afu->err_irq_name)
360 		return -ENOMEM;
361 
362 	if (!(afu->serr_virq = cxl_map_irq(afu->adapter, afu->serr_hwirq,
363 				 guest_slice_irq_err, afu, afu->err_irq_name))) {
364 		kfree(afu->err_irq_name);
365 		afu->err_irq_name = NULL;
366 		return -ENOMEM;
367 	}
368 
369 	return 0;
370 }
371 
guest_release_serr_irq(struct cxl_afu * afu)372 static void guest_release_serr_irq(struct cxl_afu *afu)
373 {
374 	cxl_unmap_irq(afu->serr_virq, afu);
375 	cxl_ops->release_one_irq(afu->adapter, afu->serr_hwirq);
376 	kfree(afu->err_irq_name);
377 }
378 
guest_ack_irq(struct cxl_context * ctx,u64 tfc,u64 psl_reset_mask)379 static int guest_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask)
380 {
381 	return cxl_h_control_faults(ctx->afu->guest->handle, ctx->process_token,
382 				tfc >> 32, (psl_reset_mask != 0));
383 }
384 
disable_afu_irqs(struct cxl_context * ctx)385 static void disable_afu_irqs(struct cxl_context *ctx)
386 {
387 	irq_hw_number_t hwirq;
388 	unsigned int virq;
389 	int r, i;
390 
391 	pr_devel("Disabling AFU(%d) interrupts\n", ctx->afu->slice);
392 	for (r = 0; r < CXL_IRQ_RANGES; r++) {
393 		hwirq = ctx->irqs.offset[r];
394 		for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
395 			virq = irq_find_mapping(NULL, hwirq);
396 			disable_irq(virq);
397 		}
398 	}
399 }
400 
enable_afu_irqs(struct cxl_context * ctx)401 static void enable_afu_irqs(struct cxl_context *ctx)
402 {
403 	irq_hw_number_t hwirq;
404 	unsigned int virq;
405 	int r, i;
406 
407 	pr_devel("Enabling AFU(%d) interrupts\n", ctx->afu->slice);
408 	for (r = 0; r < CXL_IRQ_RANGES; r++) {
409 		hwirq = ctx->irqs.offset[r];
410 		for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
411 			virq = irq_find_mapping(NULL, hwirq);
412 			enable_irq(virq);
413 		}
414 	}
415 }
416 
_guest_afu_cr_readXX(int sz,struct cxl_afu * afu,int cr_idx,u64 offset,u64 * val)417 static int _guest_afu_cr_readXX(int sz, struct cxl_afu *afu, int cr_idx,
418 			u64 offset, u64 *val)
419 {
420 	unsigned long cr;
421 	char c;
422 	int rc = 0;
423 
424 	if (afu->crs_len < sz)
425 		return -ENOENT;
426 
427 	if (unlikely(offset >= afu->crs_len))
428 		return -ERANGE;
429 
430 	cr = get_zeroed_page(GFP_KERNEL);
431 	if (!cr)
432 		return -ENOMEM;
433 
434 	rc = cxl_h_get_config(afu->guest->handle, cr_idx, offset,
435 			virt_to_phys((void *)cr), sz);
436 	if (rc)
437 		goto err;
438 
439 	switch (sz) {
440 	case 1:
441 		c = *((char *) cr);
442 		*val = c;
443 		break;
444 	case 2:
445 		*val = in_le16((u16 *)cr);
446 		break;
447 	case 4:
448 		*val = in_le32((unsigned *)cr);
449 		break;
450 	case 8:
451 		*val = in_le64((u64 *)cr);
452 		break;
453 	default:
454 		WARN_ON(1);
455 	}
456 err:
457 	free_page(cr);
458 	return rc;
459 }
460 
guest_afu_cr_read32(struct cxl_afu * afu,int cr_idx,u64 offset,u32 * out)461 static int guest_afu_cr_read32(struct cxl_afu *afu, int cr_idx, u64 offset,
462 			u32 *out)
463 {
464 	int rc;
465 	u64 val;
466 
467 	rc = _guest_afu_cr_readXX(4, afu, cr_idx, offset, &val);
468 	if (!rc)
469 		*out = (u32) val;
470 	return rc;
471 }
472 
guest_afu_cr_read16(struct cxl_afu * afu,int cr_idx,u64 offset,u16 * out)473 static int guest_afu_cr_read16(struct cxl_afu *afu, int cr_idx, u64 offset,
474 			u16 *out)
475 {
476 	int rc;
477 	u64 val;
478 
479 	rc = _guest_afu_cr_readXX(2, afu, cr_idx, offset, &val);
480 	if (!rc)
481 		*out = (u16) val;
482 	return rc;
483 }
484 
guest_afu_cr_read8(struct cxl_afu * afu,int cr_idx,u64 offset,u8 * out)485 static int guest_afu_cr_read8(struct cxl_afu *afu, int cr_idx, u64 offset,
486 			u8 *out)
487 {
488 	int rc;
489 	u64 val;
490 
491 	rc = _guest_afu_cr_readXX(1, afu, cr_idx, offset, &val);
492 	if (!rc)
493 		*out = (u8) val;
494 	return rc;
495 }
496 
guest_afu_cr_read64(struct cxl_afu * afu,int cr_idx,u64 offset,u64 * out)497 static int guest_afu_cr_read64(struct cxl_afu *afu, int cr_idx, u64 offset,
498 			u64 *out)
499 {
500 	return _guest_afu_cr_readXX(8, afu, cr_idx, offset, out);
501 }
502 
guest_afu_cr_write32(struct cxl_afu * afu,int cr,u64 off,u32 in)503 static int guest_afu_cr_write32(struct cxl_afu *afu, int cr, u64 off, u32 in)
504 {
505 	/* config record is not writable from guest */
506 	return -EPERM;
507 }
508 
guest_afu_cr_write16(struct cxl_afu * afu,int cr,u64 off,u16 in)509 static int guest_afu_cr_write16(struct cxl_afu *afu, int cr, u64 off, u16 in)
510 {
511 	/* config record is not writable from guest */
512 	return -EPERM;
513 }
514 
guest_afu_cr_write8(struct cxl_afu * afu,int cr,u64 off,u8 in)515 static int guest_afu_cr_write8(struct cxl_afu *afu, int cr, u64 off, u8 in)
516 {
517 	/* config record is not writable from guest */
518 	return -EPERM;
519 }
520 
attach_afu_directed(struct cxl_context * ctx,u64 wed,u64 amr)521 static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr)
522 {
523 	struct cxl_process_element_hcall *elem;
524 	struct cxl *adapter = ctx->afu->adapter;
525 	const struct cred *cred;
526 	u32 pid, idx;
527 	int rc, r, i;
528 	u64 mmio_addr, mmio_size;
529 	__be64 flags = 0;
530 
531 	/* Must be 8 byte aligned and cannot cross a 4096 byte boundary */
532 	if (!(elem = (struct cxl_process_element_hcall *)
533 			get_zeroed_page(GFP_KERNEL)))
534 		return -ENOMEM;
535 
536 	elem->version = cpu_to_be64(CXL_PROCESS_ELEMENT_VERSION);
537 	if (ctx->kernel) {
538 		pid = 0;
539 		flags |= CXL_PE_TRANSLATION_ENABLED;
540 		flags |= CXL_PE_PRIVILEGED_PROCESS;
541 		if (mfmsr() & MSR_SF)
542 			flags |= CXL_PE_64_BIT;
543 	} else {
544 		pid = current->pid;
545 		flags |= CXL_PE_PROBLEM_STATE;
546 		flags |= CXL_PE_TRANSLATION_ENABLED;
547 		if (!test_tsk_thread_flag(current, TIF_32BIT))
548 			flags |= CXL_PE_64_BIT;
549 		cred = get_current_cred();
550 		if (uid_eq(cred->euid, GLOBAL_ROOT_UID))
551 			flags |= CXL_PE_PRIVILEGED_PROCESS;
552 		put_cred(cred);
553 	}
554 	elem->flags         = cpu_to_be64(flags);
555 	elem->common.tid    = cpu_to_be32(0); /* Unused */
556 	elem->common.pid    = cpu_to_be32(pid);
557 	elem->common.csrp   = cpu_to_be64(0); /* disable */
558 	elem->common.u.psl8.aurp0  = cpu_to_be64(0); /* disable */
559 	elem->common.u.psl8.aurp1  = cpu_to_be64(0); /* disable */
560 
561 	cxl_prefault(ctx, wed);
562 
563 	elem->common.u.psl8.sstp0  = cpu_to_be64(ctx->sstp0);
564 	elem->common.u.psl8.sstp1  = cpu_to_be64(ctx->sstp1);
565 
566 	/*
567 	 * Ensure we have at least one interrupt allocated to take faults for
568 	 * kernel contexts that may not have allocated any AFU IRQs at all:
569 	 */
570 	if (ctx->irqs.range[0] == 0) {
571 		rc = afu_register_irqs(ctx, 0);
572 		if (rc)
573 			goto out_free;
574 	}
575 
576 	for (r = 0; r < CXL_IRQ_RANGES; r++) {
577 		for (i = 0; i < ctx->irqs.range[r]; i++) {
578 			if (r == 0 && i == 0) {
579 				elem->pslVirtualIsn = cpu_to_be32(ctx->irqs.offset[0]);
580 			} else {
581 				idx = ctx->irqs.offset[r] + i - adapter->guest->irq_base_offset;
582 				elem->applicationVirtualIsnBitmap[idx / 8] |= 0x80 >> (idx % 8);
583 			}
584 		}
585 	}
586 	elem->common.amr = cpu_to_be64(amr);
587 	elem->common.wed = cpu_to_be64(wed);
588 
589 	disable_afu_irqs(ctx);
590 
591 	rc = cxl_h_attach_process(ctx->afu->guest->handle, elem,
592 				&ctx->process_token, &mmio_addr, &mmio_size);
593 	if (rc == H_SUCCESS) {
594 		if (ctx->master || !ctx->afu->pp_psa) {
595 			ctx->psn_phys = ctx->afu->psn_phys;
596 			ctx->psn_size = ctx->afu->adapter->ps_size;
597 		} else {
598 			ctx->psn_phys = mmio_addr;
599 			ctx->psn_size = mmio_size;
600 		}
601 		if (ctx->afu->pp_psa && mmio_size &&
602 			ctx->afu->pp_size == 0) {
603 			/*
604 			 * There's no property in the device tree to read the
605 			 * pp_size. We only find out at the 1st attach.
606 			 * Compared to bare-metal, it is too late and we
607 			 * should really lock here. However, on powerVM,
608 			 * pp_size is really only used to display in /sys.
609 			 * Being discussed with pHyp for their next release.
610 			 */
611 			ctx->afu->pp_size = mmio_size;
612 		}
613 		/* from PAPR: process element is bytes 4-7 of process token */
614 		ctx->external_pe = ctx->process_token & 0xFFFFFFFF;
615 		pr_devel("CXL pe=%i is known as %i for pHyp, mmio_size=%#llx",
616 			ctx->pe, ctx->external_pe, ctx->psn_size);
617 		ctx->pe_inserted = true;
618 		enable_afu_irqs(ctx);
619 	}
620 
621 out_free:
622 	free_page((u64)elem);
623 	return rc;
624 }
625 
guest_attach_process(struct cxl_context * ctx,bool kernel,u64 wed,u64 amr)626 static int guest_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u64 amr)
627 {
628 	pr_devel("in %s\n", __func__);
629 
630 	ctx->kernel = kernel;
631 	if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
632 		return attach_afu_directed(ctx, wed, amr);
633 
634 	/* dedicated mode not supported on FW840 */
635 
636 	return -EINVAL;
637 }
638 
detach_afu_directed(struct cxl_context * ctx)639 static int detach_afu_directed(struct cxl_context *ctx)
640 {
641 	if (!ctx->pe_inserted)
642 		return 0;
643 	if (cxl_h_detach_process(ctx->afu->guest->handle, ctx->process_token))
644 		return -1;
645 	return 0;
646 }
647 
guest_detach_process(struct cxl_context * ctx)648 static int guest_detach_process(struct cxl_context *ctx)
649 {
650 	pr_devel("in %s\n", __func__);
651 	trace_cxl_detach(ctx);
652 
653 	if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu))
654 		return -EIO;
655 
656 	if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
657 		return detach_afu_directed(ctx);
658 
659 	return -EINVAL;
660 }
661 
guest_release_afu(struct device * dev)662 static void guest_release_afu(struct device *dev)
663 {
664 	struct cxl_afu *afu = to_cxl_afu(dev);
665 
666 	pr_devel("%s\n", __func__);
667 
668 	idr_destroy(&afu->contexts_idr);
669 
670 	kfree(afu->guest);
671 	kfree(afu);
672 }
673 
cxl_guest_read_afu_vpd(struct cxl_afu * afu,void * buf,size_t len)674 ssize_t cxl_guest_read_afu_vpd(struct cxl_afu *afu, void *buf, size_t len)
675 {
676 	return guest_collect_vpd(NULL, afu, buf, len);
677 }
678 
679 #define ERR_BUFF_MAX_COPY_SIZE PAGE_SIZE
guest_afu_read_err_buffer(struct cxl_afu * afu,char * buf,loff_t off,size_t count)680 static ssize_t guest_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
681 					loff_t off, size_t count)
682 {
683 	void *tbuf = NULL;
684 	int rc = 0;
685 
686 	tbuf = (void *) get_zeroed_page(GFP_KERNEL);
687 	if (!tbuf)
688 		return -ENOMEM;
689 
690 	rc = cxl_h_get_afu_err(afu->guest->handle,
691 			       off & 0x7,
692 			       virt_to_phys(tbuf),
693 			       count);
694 	if (rc)
695 		goto err;
696 
697 	if (count > ERR_BUFF_MAX_COPY_SIZE)
698 		count = ERR_BUFF_MAX_COPY_SIZE - (off & 0x7);
699 	memcpy(buf, tbuf, count);
700 err:
701 	free_page((u64)tbuf);
702 
703 	return rc;
704 }
705 
guest_afu_check_and_enable(struct cxl_afu * afu)706 static int guest_afu_check_and_enable(struct cxl_afu *afu)
707 {
708 	return 0;
709 }
710 
guest_support_attributes(const char * attr_name,enum cxl_attrs type)711 static bool guest_support_attributes(const char *attr_name,
712 				     enum cxl_attrs type)
713 {
714 	switch (type) {
715 	case CXL_ADAPTER_ATTRS:
716 		if ((strcmp(attr_name, "base_image") == 0) ||
717 			(strcmp(attr_name, "load_image_on_perst") == 0) ||
718 			(strcmp(attr_name, "perst_reloads_same_image") == 0) ||
719 			(strcmp(attr_name, "image_loaded") == 0))
720 			return false;
721 		break;
722 	case CXL_AFU_MASTER_ATTRS:
723 		if ((strcmp(attr_name, "pp_mmio_off") == 0))
724 			return false;
725 		break;
726 	case CXL_AFU_ATTRS:
727 		break;
728 	default:
729 		break;
730 	}
731 
732 	return true;
733 }
734 
activate_afu_directed(struct cxl_afu * afu)735 static int activate_afu_directed(struct cxl_afu *afu)
736 {
737 	int rc;
738 
739 	dev_info(&afu->dev, "Activating AFU(%d) directed mode\n", afu->slice);
740 
741 	afu->current_mode = CXL_MODE_DIRECTED;
742 
743 	afu->num_procs = afu->max_procs_virtualised;
744 
745 	if ((rc = cxl_chardev_m_afu_add(afu)))
746 		return rc;
747 
748 	if ((rc = cxl_sysfs_afu_m_add(afu)))
749 		goto err;
750 
751 	if ((rc = cxl_chardev_s_afu_add(afu)))
752 		goto err1;
753 
754 	return 0;
755 err1:
756 	cxl_sysfs_afu_m_remove(afu);
757 err:
758 	cxl_chardev_afu_remove(afu);
759 	return rc;
760 }
761 
guest_afu_activate_mode(struct cxl_afu * afu,int mode)762 static int guest_afu_activate_mode(struct cxl_afu *afu, int mode)
763 {
764 	if (!mode)
765 		return 0;
766 	if (!(mode & afu->modes_supported))
767 		return -EINVAL;
768 
769 	if (mode == CXL_MODE_DIRECTED)
770 		return activate_afu_directed(afu);
771 
772 	if (mode == CXL_MODE_DEDICATED)
773 		dev_err(&afu->dev, "Dedicated mode not supported\n");
774 
775 	return -EINVAL;
776 }
777 
deactivate_afu_directed(struct cxl_afu * afu)778 static int deactivate_afu_directed(struct cxl_afu *afu)
779 {
780 	dev_info(&afu->dev, "Deactivating AFU(%d) directed mode\n", afu->slice);
781 
782 	afu->current_mode = 0;
783 	afu->num_procs = 0;
784 
785 	cxl_sysfs_afu_m_remove(afu);
786 	cxl_chardev_afu_remove(afu);
787 
788 	cxl_ops->afu_reset(afu);
789 
790 	return 0;
791 }
792 
guest_afu_deactivate_mode(struct cxl_afu * afu,int mode)793 static int guest_afu_deactivate_mode(struct cxl_afu *afu, int mode)
794 {
795 	if (!mode)
796 		return 0;
797 	if (!(mode & afu->modes_supported))
798 		return -EINVAL;
799 
800 	if (mode == CXL_MODE_DIRECTED)
801 		return deactivate_afu_directed(afu);
802 	return 0;
803 }
804 
guest_afu_reset(struct cxl_afu * afu)805 static int guest_afu_reset(struct cxl_afu *afu)
806 {
807 	pr_devel("AFU(%d) reset request\n", afu->slice);
808 	return cxl_h_reset_afu(afu->guest->handle);
809 }
810 
guest_map_slice_regs(struct cxl_afu * afu)811 static int guest_map_slice_regs(struct cxl_afu *afu)
812 {
813 	if (!(afu->p2n_mmio = ioremap(afu->guest->p2n_phys, afu->guest->p2n_size))) {
814 		dev_err(&afu->dev, "Error mapping AFU(%d) MMIO regions\n",
815 			afu->slice);
816 		return -ENOMEM;
817 	}
818 	return 0;
819 }
820 
guest_unmap_slice_regs(struct cxl_afu * afu)821 static void guest_unmap_slice_regs(struct cxl_afu *afu)
822 {
823 	if (afu->p2n_mmio)
824 		iounmap(afu->p2n_mmio);
825 }
826 
afu_update_state(struct cxl_afu * afu)827 static int afu_update_state(struct cxl_afu *afu)
828 {
829 	int rc, cur_state;
830 
831 	rc = afu_read_error_state(afu, &cur_state);
832 	if (rc)
833 		return rc;
834 
835 	if (afu->guest->previous_state == cur_state)
836 		return 0;
837 
838 	pr_devel("AFU(%d) update state to %#x\n", afu->slice, cur_state);
839 
840 	switch (cur_state) {
841 	case H_STATE_NORMAL:
842 		afu->guest->previous_state = cur_state;
843 		break;
844 
845 	case H_STATE_DISABLE:
846 		pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT,
847 				pci_channel_io_frozen);
848 
849 		cxl_context_detach_all(afu);
850 		if ((rc = cxl_ops->afu_reset(afu)))
851 			pr_devel("reset hcall failed %d\n", rc);
852 
853 		rc = afu_read_error_state(afu, &cur_state);
854 		if (!rc && cur_state == H_STATE_NORMAL) {
855 			pci_error_handlers(afu, CXL_SLOT_RESET_EVENT,
856 					pci_channel_io_normal);
857 			pci_error_handlers(afu, CXL_RESUME_EVENT, 0);
858 		}
859 		afu->guest->previous_state = 0;
860 		break;
861 
862 	case H_STATE_TEMP_UNAVAILABLE:
863 		afu->guest->previous_state = cur_state;
864 		break;
865 
866 	case H_STATE_PERM_UNAVAILABLE:
867 		dev_err(&afu->dev, "AFU is in permanent error state\n");
868 		pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT,
869 				pci_channel_io_perm_failure);
870 		afu->guest->previous_state = cur_state;
871 		break;
872 
873 	default:
874 		pr_err("Unexpected AFU(%d) error state: %#x\n",
875 		       afu->slice, cur_state);
876 		return -EINVAL;
877 	}
878 
879 	return rc;
880 }
881 
afu_handle_errstate(struct work_struct * work)882 static void afu_handle_errstate(struct work_struct *work)
883 {
884 	struct cxl_afu_guest *afu_guest =
885 		container_of(to_delayed_work(work), struct cxl_afu_guest, work_err);
886 
887 	if (!afu_update_state(afu_guest->parent) &&
888 	    afu_guest->previous_state == H_STATE_PERM_UNAVAILABLE)
889 		return;
890 
891 	if (afu_guest->handle_err)
892 		schedule_delayed_work(&afu_guest->work_err,
893 				      msecs_to_jiffies(3000));
894 }
895 
guest_link_ok(struct cxl * cxl,struct cxl_afu * afu)896 static bool guest_link_ok(struct cxl *cxl, struct cxl_afu *afu)
897 {
898 	int state;
899 
900 	if (afu && (!afu_read_error_state(afu, &state))) {
901 		if (state == H_STATE_NORMAL)
902 			return true;
903 	}
904 
905 	return false;
906 }
907 
afu_properties_look_ok(struct cxl_afu * afu)908 static int afu_properties_look_ok(struct cxl_afu *afu)
909 {
910 	if (afu->pp_irqs < 0) {
911 		dev_err(&afu->dev, "Unexpected per-process minimum interrupt value\n");
912 		return -EINVAL;
913 	}
914 
915 	if (afu->max_procs_virtualised < 1) {
916 		dev_err(&afu->dev, "Unexpected max number of processes virtualised value\n");
917 		return -EINVAL;
918 	}
919 
920 	return 0;
921 }
922 
cxl_guest_init_afu(struct cxl * adapter,int slice,struct device_node * afu_np)923 int cxl_guest_init_afu(struct cxl *adapter, int slice, struct device_node *afu_np)
924 {
925 	struct cxl_afu *afu;
926 	bool free = true;
927 	int rc;
928 
929 	pr_devel("in %s - AFU(%d)\n", __func__, slice);
930 	if (!(afu = cxl_alloc_afu(adapter, slice)))
931 		return -ENOMEM;
932 
933 	if (!(afu->guest = kzalloc(sizeof(struct cxl_afu_guest), GFP_KERNEL))) {
934 		kfree(afu);
935 		return -ENOMEM;
936 	}
937 
938 	if ((rc = dev_set_name(&afu->dev, "afu%i.%i",
939 					  adapter->adapter_num,
940 					  slice)))
941 		goto err1;
942 
943 	adapter->slices++;
944 
945 	if ((rc = cxl_of_read_afu_handle(afu, afu_np)))
946 		goto err1;
947 
948 	if ((rc = cxl_ops->afu_reset(afu)))
949 		goto err1;
950 
951 	if ((rc = cxl_of_read_afu_properties(afu, afu_np)))
952 		goto err1;
953 
954 	if ((rc = afu_properties_look_ok(afu)))
955 		goto err1;
956 
957 	if ((rc = guest_map_slice_regs(afu)))
958 		goto err1;
959 
960 	if ((rc = guest_register_serr_irq(afu)))
961 		goto err2;
962 
963 	/*
964 	 * After we call this function we must not free the afu directly, even
965 	 * if it returns an error!
966 	 */
967 	if ((rc = cxl_register_afu(afu)))
968 		goto err_put_dev;
969 
970 	if ((rc = cxl_sysfs_afu_add(afu)))
971 		goto err_del_dev;
972 
973 	/*
974 	 * pHyp doesn't expose the programming models supported by the
975 	 * AFU. pHyp currently only supports directed mode. If it adds
976 	 * dedicated mode later, this version of cxl has no way to
977 	 * detect it. So we'll initialize the driver, but the first
978 	 * attach will fail.
979 	 * Being discussed with pHyp to do better (likely new property)
980 	 */
981 	if (afu->max_procs_virtualised == 1)
982 		afu->modes_supported = CXL_MODE_DEDICATED;
983 	else
984 		afu->modes_supported = CXL_MODE_DIRECTED;
985 
986 	if ((rc = cxl_afu_select_best_mode(afu)))
987 		goto err_remove_sysfs;
988 
989 	adapter->afu[afu->slice] = afu;
990 
991 	afu->enabled = true;
992 
993 	/*
994 	 * wake up the cpu periodically to check the state
995 	 * of the AFU using "afu" stored in the guest structure.
996 	 */
997 	afu->guest->parent = afu;
998 	afu->guest->handle_err = true;
999 	INIT_DELAYED_WORK(&afu->guest->work_err, afu_handle_errstate);
1000 	schedule_delayed_work(&afu->guest->work_err, msecs_to_jiffies(1000));
1001 
1002 	if ((rc = cxl_pci_vphb_add(afu)))
1003 		dev_info(&afu->dev, "Can't register vPHB\n");
1004 
1005 	return 0;
1006 
1007 err_remove_sysfs:
1008 	cxl_sysfs_afu_remove(afu);
1009 err_del_dev:
1010 	device_del(&afu->dev);
1011 err_put_dev:
1012 	put_device(&afu->dev);
1013 	free = false;
1014 	guest_release_serr_irq(afu);
1015 err2:
1016 	guest_unmap_slice_regs(afu);
1017 err1:
1018 	if (free) {
1019 		kfree(afu->guest);
1020 		kfree(afu);
1021 	}
1022 	return rc;
1023 }
1024 
cxl_guest_remove_afu(struct cxl_afu * afu)1025 void cxl_guest_remove_afu(struct cxl_afu *afu)
1026 {
1027 	if (!afu)
1028 		return;
1029 
1030 	/* flush and stop pending job */
1031 	afu->guest->handle_err = false;
1032 	flush_delayed_work(&afu->guest->work_err);
1033 
1034 	cxl_pci_vphb_remove(afu);
1035 	cxl_sysfs_afu_remove(afu);
1036 
1037 	spin_lock(&afu->adapter->afu_list_lock);
1038 	afu->adapter->afu[afu->slice] = NULL;
1039 	spin_unlock(&afu->adapter->afu_list_lock);
1040 
1041 	cxl_context_detach_all(afu);
1042 	cxl_ops->afu_deactivate_mode(afu, afu->current_mode);
1043 	guest_release_serr_irq(afu);
1044 	guest_unmap_slice_regs(afu);
1045 
1046 	device_unregister(&afu->dev);
1047 }
1048 
free_adapter(struct cxl * adapter)1049 static void free_adapter(struct cxl *adapter)
1050 {
1051 	struct irq_avail *cur;
1052 	int i;
1053 
1054 	if (adapter->guest) {
1055 		if (adapter->guest->irq_avail) {
1056 			for (i = 0; i < adapter->guest->irq_nranges; i++) {
1057 				cur = &adapter->guest->irq_avail[i];
1058 				bitmap_free(cur->bitmap);
1059 			}
1060 			kfree(adapter->guest->irq_avail);
1061 		}
1062 		kfree(adapter->guest->status);
1063 		kfree(adapter->guest);
1064 	}
1065 	cxl_remove_adapter_nr(adapter);
1066 	kfree(adapter);
1067 }
1068 
properties_look_ok(struct cxl * adapter)1069 static int properties_look_ok(struct cxl *adapter)
1070 {
1071 	/* The absence of this property means that the operational
1072 	 * status is unknown or okay
1073 	 */
1074 	if (strlen(adapter->guest->status) &&
1075 	    strcmp(adapter->guest->status, "okay")) {
1076 		pr_err("ABORTING:Bad operational status of the device\n");
1077 		return -EINVAL;
1078 	}
1079 
1080 	return 0;
1081 }
1082 
cxl_guest_read_adapter_vpd(struct cxl * adapter,void * buf,size_t len)1083 ssize_t cxl_guest_read_adapter_vpd(struct cxl *adapter, void *buf, size_t len)
1084 {
1085 	return guest_collect_vpd(adapter, NULL, buf, len);
1086 }
1087 
cxl_guest_remove_adapter(struct cxl * adapter)1088 void cxl_guest_remove_adapter(struct cxl *adapter)
1089 {
1090 	pr_devel("in %s\n", __func__);
1091 
1092 	cxl_sysfs_adapter_remove(adapter);
1093 
1094 	cxl_guest_remove_chardev(adapter);
1095 	device_unregister(&adapter->dev);
1096 }
1097 
release_adapter(struct device * dev)1098 static void release_adapter(struct device *dev)
1099 {
1100 	free_adapter(to_cxl_adapter(dev));
1101 }
1102 
cxl_guest_init_adapter(struct device_node * np,struct platform_device * pdev)1103 struct cxl *cxl_guest_init_adapter(struct device_node *np, struct platform_device *pdev)
1104 {
1105 	struct cxl *adapter;
1106 	bool free = true;
1107 	int rc;
1108 
1109 	if (!(adapter = cxl_alloc_adapter()))
1110 		return ERR_PTR(-ENOMEM);
1111 
1112 	if (!(adapter->guest = kzalloc(sizeof(struct cxl_guest), GFP_KERNEL))) {
1113 		free_adapter(adapter);
1114 		return ERR_PTR(-ENOMEM);
1115 	}
1116 
1117 	adapter->slices = 0;
1118 	adapter->guest->pdev = pdev;
1119 	adapter->dev.parent = &pdev->dev;
1120 	adapter->dev.release = release_adapter;
1121 	dev_set_drvdata(&pdev->dev, adapter);
1122 
1123 	/*
1124 	 * Hypervisor controls PSL timebase initialization (p1 register).
1125 	 * On FW840, PSL is initialized.
1126 	 */
1127 	adapter->psl_timebase_synced = true;
1128 
1129 	if ((rc = cxl_of_read_adapter_handle(adapter, np)))
1130 		goto err1;
1131 
1132 	if ((rc = cxl_of_read_adapter_properties(adapter, np)))
1133 		goto err1;
1134 
1135 	if ((rc = properties_look_ok(adapter)))
1136 		goto err1;
1137 
1138 	if ((rc = cxl_guest_add_chardev(adapter)))
1139 		goto err1;
1140 
1141 	/*
1142 	 * After we call this function we must not free the adapter directly,
1143 	 * even if it returns an error!
1144 	 */
1145 	if ((rc = cxl_register_adapter(adapter)))
1146 		goto err_put_dev;
1147 
1148 	if ((rc = cxl_sysfs_adapter_add(adapter)))
1149 		goto err_del_dev;
1150 
1151 	/* release the context lock as the adapter is configured */
1152 	cxl_adapter_context_unlock(adapter);
1153 
1154 	return adapter;
1155 
1156 err_del_dev:
1157 	device_del(&adapter->dev);
1158 err_put_dev:
1159 	put_device(&adapter->dev);
1160 	free = false;
1161 	cxl_guest_remove_chardev(adapter);
1162 err1:
1163 	if (free)
1164 		free_adapter(adapter);
1165 	return ERR_PTR(rc);
1166 }
1167 
cxl_guest_reload_module(struct cxl * adapter)1168 void cxl_guest_reload_module(struct cxl *adapter)
1169 {
1170 	struct platform_device *pdev;
1171 
1172 	pdev = adapter->guest->pdev;
1173 	cxl_guest_remove_adapter(adapter);
1174 
1175 	cxl_of_probe(pdev);
1176 }
1177 
1178 const struct cxl_backend_ops cxl_guest_ops = {
1179 	.module = THIS_MODULE,
1180 	.adapter_reset = guest_reset,
1181 	.alloc_one_irq = guest_alloc_one_irq,
1182 	.release_one_irq = guest_release_one_irq,
1183 	.alloc_irq_ranges = guest_alloc_irq_ranges,
1184 	.release_irq_ranges = guest_release_irq_ranges,
1185 	.setup_irq = NULL,
1186 	.handle_psl_slice_error = guest_handle_psl_slice_error,
1187 	.psl_interrupt = guest_psl_irq,
1188 	.ack_irq = guest_ack_irq,
1189 	.attach_process = guest_attach_process,
1190 	.detach_process = guest_detach_process,
1191 	.update_ivtes = NULL,
1192 	.support_attributes = guest_support_attributes,
1193 	.link_ok = guest_link_ok,
1194 	.release_afu = guest_release_afu,
1195 	.afu_read_err_buffer = guest_afu_read_err_buffer,
1196 	.afu_check_and_enable = guest_afu_check_and_enable,
1197 	.afu_activate_mode = guest_afu_activate_mode,
1198 	.afu_deactivate_mode = guest_afu_deactivate_mode,
1199 	.afu_reset = guest_afu_reset,
1200 	.afu_cr_read8 = guest_afu_cr_read8,
1201 	.afu_cr_read16 = guest_afu_cr_read16,
1202 	.afu_cr_read32 = guest_afu_cr_read32,
1203 	.afu_cr_read64 = guest_afu_cr_read64,
1204 	.afu_cr_write8 = guest_afu_cr_write8,
1205 	.afu_cr_write16 = guest_afu_cr_write16,
1206 	.afu_cr_write32 = guest_afu_cr_write32,
1207 	.read_adapter_vpd = cxl_guest_read_adapter_vpd,
1208 };
1209