1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright IBM Corp. 2012
4  *
5  * Author(s):
6  *   Jan Glauber <jang@linux.vnet.ibm.com>
7  *
8  * The System z PCI code is a rewrite from a prototype by
9  * the following people (Kudoz!):
10  *   Alexander Schmidt
11  *   Christoph Raisch
12  *   Hannes Hering
13  *   Hoang-Nam Nguyen
14  *   Jan-Bernd Themann
15  *   Stefan Roscher
16  *   Thomas Klein
17  */
18 
19 #define KMSG_COMPONENT "zpci"
20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
21 
22 #include <linux/kernel.h>
23 #include <linux/slab.h>
24 #include <linux/err.h>
25 #include <linux/export.h>
26 #include <linux/delay.h>
27 #include <linux/seq_file.h>
28 #include <linux/jump_label.h>
29 #include <linux/pci.h>
30 #include <linux/printk.h>
31 
32 #include <asm/isc.h>
33 #include <asm/airq.h>
34 #include <asm/facility.h>
35 #include <asm/pci_insn.h>
36 #include <asm/pci_clp.h>
37 #include <asm/pci_dma.h>
38 
39 #include "pci_bus.h"
40 #include "pci_iov.h"
41 
42 /* list of all detected zpci devices */
43 static LIST_HEAD(zpci_list);
44 static DEFINE_SPINLOCK(zpci_list_lock);
45 
46 static DECLARE_BITMAP(zpci_domain, ZPCI_DOMAIN_BITMAP_SIZE);
47 static DEFINE_SPINLOCK(zpci_domain_lock);
48 
49 #define ZPCI_IOMAP_ENTRIES						\
50 	min(((unsigned long) ZPCI_NR_DEVICES * PCI_STD_NUM_BARS / 2),	\
51 	    ZPCI_IOMAP_MAX_ENTRIES)
52 
53 unsigned int s390_pci_no_rid;
54 
55 static DEFINE_SPINLOCK(zpci_iomap_lock);
56 static unsigned long *zpci_iomap_bitmap;
57 struct zpci_iomap_entry *zpci_iomap_start;
58 EXPORT_SYMBOL_GPL(zpci_iomap_start);
59 
60 DEFINE_STATIC_KEY_FALSE(have_mio);
61 
62 static struct kmem_cache *zdev_fmb_cache;
63 
get_zdev_by_fid(u32 fid)64 struct zpci_dev *get_zdev_by_fid(u32 fid)
65 {
66 	struct zpci_dev *tmp, *zdev = NULL;
67 
68 	spin_lock(&zpci_list_lock);
69 	list_for_each_entry(tmp, &zpci_list, entry) {
70 		if (tmp->fid == fid) {
71 			zdev = tmp;
72 			zpci_zdev_get(zdev);
73 			break;
74 		}
75 	}
76 	spin_unlock(&zpci_list_lock);
77 	return zdev;
78 }
79 
zpci_remove_reserved_devices(void)80 void zpci_remove_reserved_devices(void)
81 {
82 	struct zpci_dev *tmp, *zdev;
83 	enum zpci_state state;
84 	LIST_HEAD(remove);
85 
86 	spin_lock(&zpci_list_lock);
87 	list_for_each_entry_safe(zdev, tmp, &zpci_list, entry) {
88 		if (zdev->state == ZPCI_FN_STATE_STANDBY &&
89 		    !clp_get_state(zdev->fid, &state) &&
90 		    state == ZPCI_FN_STATE_RESERVED)
91 			list_move_tail(&zdev->entry, &remove);
92 	}
93 	spin_unlock(&zpci_list_lock);
94 
95 	list_for_each_entry_safe(zdev, tmp, &remove, entry)
96 		zpci_device_reserved(zdev);
97 }
98 
pci_domain_nr(struct pci_bus * bus)99 int pci_domain_nr(struct pci_bus *bus)
100 {
101 	return ((struct zpci_bus *) bus->sysdata)->domain_nr;
102 }
103 EXPORT_SYMBOL_GPL(pci_domain_nr);
104 
pci_proc_domain(struct pci_bus * bus)105 int pci_proc_domain(struct pci_bus *bus)
106 {
107 	return pci_domain_nr(bus);
108 }
109 EXPORT_SYMBOL_GPL(pci_proc_domain);
110 
111 /* Modify PCI: Register I/O address translation parameters */
zpci_register_ioat(struct zpci_dev * zdev,u8 dmaas,u64 base,u64 limit,u64 iota)112 int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
113 		       u64 base, u64 limit, u64 iota)
114 {
115 	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT);
116 	struct zpci_fib fib = {0};
117 	u8 cc, status;
118 
119 	WARN_ON_ONCE(iota & 0x3fff);
120 	fib.pba = base;
121 	fib.pal = limit;
122 	fib.iota = iota | ZPCI_IOTA_RTTO_FLAG;
123 	cc = zpci_mod_fc(req, &fib, &status);
124 	if (cc)
125 		zpci_dbg(3, "reg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status);
126 	return cc;
127 }
128 
129 /* Modify PCI: Unregister I/O address translation parameters */
zpci_unregister_ioat(struct zpci_dev * zdev,u8 dmaas)130 int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas)
131 {
132 	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_DEREG_IOAT);
133 	struct zpci_fib fib = {0};
134 	u8 cc, status;
135 
136 	cc = zpci_mod_fc(req, &fib, &status);
137 	if (cc)
138 		zpci_dbg(3, "unreg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status);
139 	return cc;
140 }
141 
142 /* Modify PCI: Set PCI function measurement parameters */
zpci_fmb_enable_device(struct zpci_dev * zdev)143 int zpci_fmb_enable_device(struct zpci_dev *zdev)
144 {
145 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
146 	struct zpci_fib fib = {0};
147 	u8 cc, status;
148 
149 	if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length)
150 		return -EINVAL;
151 
152 	zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL);
153 	if (!zdev->fmb)
154 		return -ENOMEM;
155 	WARN_ON((u64) zdev->fmb & 0xf);
156 
157 	/* reset software counters */
158 	atomic64_set(&zdev->allocated_pages, 0);
159 	atomic64_set(&zdev->mapped_pages, 0);
160 	atomic64_set(&zdev->unmapped_pages, 0);
161 
162 	fib.fmb_addr = virt_to_phys(zdev->fmb);
163 	cc = zpci_mod_fc(req, &fib, &status);
164 	if (cc) {
165 		kmem_cache_free(zdev_fmb_cache, zdev->fmb);
166 		zdev->fmb = NULL;
167 	}
168 	return cc ? -EIO : 0;
169 }
170 
171 /* Modify PCI: Disable PCI function measurement */
zpci_fmb_disable_device(struct zpci_dev * zdev)172 int zpci_fmb_disable_device(struct zpci_dev *zdev)
173 {
174 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
175 	struct zpci_fib fib = {0};
176 	u8 cc, status;
177 
178 	if (!zdev->fmb)
179 		return -EINVAL;
180 
181 	/* Function measurement is disabled if fmb address is zero */
182 	cc = zpci_mod_fc(req, &fib, &status);
183 	if (cc == 3) /* Function already gone. */
184 		cc = 0;
185 
186 	if (!cc) {
187 		kmem_cache_free(zdev_fmb_cache, zdev->fmb);
188 		zdev->fmb = NULL;
189 	}
190 	return cc ? -EIO : 0;
191 }
192 
zpci_cfg_load(struct zpci_dev * zdev,int offset,u32 * val,u8 len)193 static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len)
194 {
195 	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
196 	u64 data;
197 	int rc;
198 
199 	rc = __zpci_load(&data, req, offset);
200 	if (!rc) {
201 		data = le64_to_cpu((__force __le64) data);
202 		data >>= (8 - len) * 8;
203 		*val = (u32) data;
204 	} else
205 		*val = 0xffffffff;
206 	return rc;
207 }
208 
zpci_cfg_store(struct zpci_dev * zdev,int offset,u32 val,u8 len)209 static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len)
210 {
211 	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
212 	u64 data = val;
213 	int rc;
214 
215 	data <<= (8 - len) * 8;
216 	data = (__force u64) cpu_to_le64(data);
217 	rc = __zpci_store(data, req, offset);
218 	return rc;
219 }
220 
pcibios_align_resource(void * data,const struct resource * res,resource_size_t size,resource_size_t align)221 resource_size_t pcibios_align_resource(void *data, const struct resource *res,
222 				       resource_size_t size,
223 				       resource_size_t align)
224 {
225 	return 0;
226 }
227 
228 /* combine single writes by using store-block insn */
__iowrite64_copy(void __iomem * to,const void * from,size_t count)229 void __iowrite64_copy(void __iomem *to, const void *from, size_t count)
230 {
231        zpci_memcpy_toio(to, from, count);
232 }
233 
__ioremap(phys_addr_t addr,size_t size,pgprot_t prot)234 static void __iomem *__ioremap(phys_addr_t addr, size_t size, pgprot_t prot)
235 {
236 	unsigned long offset, vaddr;
237 	struct vm_struct *area;
238 	phys_addr_t last_addr;
239 
240 	last_addr = addr + size - 1;
241 	if (!size || last_addr < addr)
242 		return NULL;
243 
244 	if (!static_branch_unlikely(&have_mio))
245 		return (void __iomem *) addr;
246 
247 	offset = addr & ~PAGE_MASK;
248 	addr &= PAGE_MASK;
249 	size = PAGE_ALIGN(size + offset);
250 	area = get_vm_area(size, VM_IOREMAP);
251 	if (!area)
252 		return NULL;
253 
254 	vaddr = (unsigned long) area->addr;
255 	if (ioremap_page_range(vaddr, vaddr + size, addr, prot)) {
256 		free_vm_area(area);
257 		return NULL;
258 	}
259 	return (void __iomem *) ((unsigned long) area->addr + offset);
260 }
261 
ioremap_prot(phys_addr_t addr,size_t size,unsigned long prot)262 void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long prot)
263 {
264 	return __ioremap(addr, size, __pgprot(prot));
265 }
266 EXPORT_SYMBOL(ioremap_prot);
267 
ioremap(phys_addr_t addr,size_t size)268 void __iomem *ioremap(phys_addr_t addr, size_t size)
269 {
270 	return __ioremap(addr, size, PAGE_KERNEL);
271 }
272 EXPORT_SYMBOL(ioremap);
273 
ioremap_wc(phys_addr_t addr,size_t size)274 void __iomem *ioremap_wc(phys_addr_t addr, size_t size)
275 {
276 	return __ioremap(addr, size, pgprot_writecombine(PAGE_KERNEL));
277 }
278 EXPORT_SYMBOL(ioremap_wc);
279 
ioremap_wt(phys_addr_t addr,size_t size)280 void __iomem *ioremap_wt(phys_addr_t addr, size_t size)
281 {
282 	return __ioremap(addr, size, pgprot_writethrough(PAGE_KERNEL));
283 }
284 EXPORT_SYMBOL(ioremap_wt);
285 
iounmap(volatile void __iomem * addr)286 void iounmap(volatile void __iomem *addr)
287 {
288 	if (static_branch_likely(&have_mio))
289 		vunmap((__force void *) ((unsigned long) addr & PAGE_MASK));
290 }
291 EXPORT_SYMBOL(iounmap);
292 
293 /* Create a virtual mapping cookie for a PCI BAR */
pci_iomap_range_fh(struct pci_dev * pdev,int bar,unsigned long offset,unsigned long max)294 static void __iomem *pci_iomap_range_fh(struct pci_dev *pdev, int bar,
295 					unsigned long offset, unsigned long max)
296 {
297 	struct zpci_dev *zdev =	to_zpci(pdev);
298 	int idx;
299 
300 	idx = zdev->bars[bar].map_idx;
301 	spin_lock(&zpci_iomap_lock);
302 	/* Detect overrun */
303 	WARN_ON(!++zpci_iomap_start[idx].count);
304 	zpci_iomap_start[idx].fh = zdev->fh;
305 	zpci_iomap_start[idx].bar = bar;
306 	spin_unlock(&zpci_iomap_lock);
307 
308 	return (void __iomem *) ZPCI_ADDR(idx) + offset;
309 }
310 
pci_iomap_range_mio(struct pci_dev * pdev,int bar,unsigned long offset,unsigned long max)311 static void __iomem *pci_iomap_range_mio(struct pci_dev *pdev, int bar,
312 					 unsigned long offset,
313 					 unsigned long max)
314 {
315 	unsigned long barsize = pci_resource_len(pdev, bar);
316 	struct zpci_dev *zdev = to_zpci(pdev);
317 	void __iomem *iova;
318 
319 	iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize);
320 	return iova ? iova + offset : iova;
321 }
322 
pci_iomap_range(struct pci_dev * pdev,int bar,unsigned long offset,unsigned long max)323 void __iomem *pci_iomap_range(struct pci_dev *pdev, int bar,
324 			      unsigned long offset, unsigned long max)
325 {
326 	if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar))
327 		return NULL;
328 
329 	if (static_branch_likely(&have_mio))
330 		return pci_iomap_range_mio(pdev, bar, offset, max);
331 	else
332 		return pci_iomap_range_fh(pdev, bar, offset, max);
333 }
334 EXPORT_SYMBOL(pci_iomap_range);
335 
pci_iomap(struct pci_dev * dev,int bar,unsigned long maxlen)336 void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen)
337 {
338 	return pci_iomap_range(dev, bar, 0, maxlen);
339 }
340 EXPORT_SYMBOL(pci_iomap);
341 
pci_iomap_wc_range_mio(struct pci_dev * pdev,int bar,unsigned long offset,unsigned long max)342 static void __iomem *pci_iomap_wc_range_mio(struct pci_dev *pdev, int bar,
343 					    unsigned long offset, unsigned long max)
344 {
345 	unsigned long barsize = pci_resource_len(pdev, bar);
346 	struct zpci_dev *zdev = to_zpci(pdev);
347 	void __iomem *iova;
348 
349 	iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize);
350 	return iova ? iova + offset : iova;
351 }
352 
pci_iomap_wc_range(struct pci_dev * pdev,int bar,unsigned long offset,unsigned long max)353 void __iomem *pci_iomap_wc_range(struct pci_dev *pdev, int bar,
354 				 unsigned long offset, unsigned long max)
355 {
356 	if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar))
357 		return NULL;
358 
359 	if (static_branch_likely(&have_mio))
360 		return pci_iomap_wc_range_mio(pdev, bar, offset, max);
361 	else
362 		return pci_iomap_range_fh(pdev, bar, offset, max);
363 }
364 EXPORT_SYMBOL(pci_iomap_wc_range);
365 
pci_iomap_wc(struct pci_dev * dev,int bar,unsigned long maxlen)366 void __iomem *pci_iomap_wc(struct pci_dev *dev, int bar, unsigned long maxlen)
367 {
368 	return pci_iomap_wc_range(dev, bar, 0, maxlen);
369 }
370 EXPORT_SYMBOL(pci_iomap_wc);
371 
pci_iounmap_fh(struct pci_dev * pdev,void __iomem * addr)372 static void pci_iounmap_fh(struct pci_dev *pdev, void __iomem *addr)
373 {
374 	unsigned int idx = ZPCI_IDX(addr);
375 
376 	spin_lock(&zpci_iomap_lock);
377 	/* Detect underrun */
378 	WARN_ON(!zpci_iomap_start[idx].count);
379 	if (!--zpci_iomap_start[idx].count) {
380 		zpci_iomap_start[idx].fh = 0;
381 		zpci_iomap_start[idx].bar = 0;
382 	}
383 	spin_unlock(&zpci_iomap_lock);
384 }
385 
pci_iounmap_mio(struct pci_dev * pdev,void __iomem * addr)386 static void pci_iounmap_mio(struct pci_dev *pdev, void __iomem *addr)
387 {
388 	iounmap(addr);
389 }
390 
pci_iounmap(struct pci_dev * pdev,void __iomem * addr)391 void pci_iounmap(struct pci_dev *pdev, void __iomem *addr)
392 {
393 	if (static_branch_likely(&have_mio))
394 		pci_iounmap_mio(pdev, addr);
395 	else
396 		pci_iounmap_fh(pdev, addr);
397 }
398 EXPORT_SYMBOL(pci_iounmap);
399 
pci_read(struct pci_bus * bus,unsigned int devfn,int where,int size,u32 * val)400 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where,
401 		    int size, u32 *val)
402 {
403 	struct zpci_dev *zdev = zdev_from_bus(bus, devfn);
404 
405 	return (zdev) ? zpci_cfg_load(zdev, where, val, size) : -ENODEV;
406 }
407 
pci_write(struct pci_bus * bus,unsigned int devfn,int where,int size,u32 val)408 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where,
409 		     int size, u32 val)
410 {
411 	struct zpci_dev *zdev = zdev_from_bus(bus, devfn);
412 
413 	return (zdev) ? zpci_cfg_store(zdev, where, val, size) : -ENODEV;
414 }
415 
416 static struct pci_ops pci_root_ops = {
417 	.read = pci_read,
418 	.write = pci_write,
419 };
420 
zpci_map_resources(struct pci_dev * pdev)421 static void zpci_map_resources(struct pci_dev *pdev)
422 {
423 	struct zpci_dev *zdev = to_zpci(pdev);
424 	resource_size_t len;
425 	int i;
426 
427 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
428 		len = pci_resource_len(pdev, i);
429 		if (!len)
430 			continue;
431 
432 		if (zpci_use_mio(zdev))
433 			pdev->resource[i].start =
434 				(resource_size_t __force) zdev->bars[i].mio_wt;
435 		else
436 			pdev->resource[i].start = (resource_size_t __force)
437 				pci_iomap_range_fh(pdev, i, 0, 0);
438 		pdev->resource[i].end = pdev->resource[i].start + len - 1;
439 	}
440 
441 	zpci_iov_map_resources(pdev);
442 }
443 
zpci_unmap_resources(struct pci_dev * pdev)444 static void zpci_unmap_resources(struct pci_dev *pdev)
445 {
446 	struct zpci_dev *zdev = to_zpci(pdev);
447 	resource_size_t len;
448 	int i;
449 
450 	if (zpci_use_mio(zdev))
451 		return;
452 
453 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
454 		len = pci_resource_len(pdev, i);
455 		if (!len)
456 			continue;
457 		pci_iounmap_fh(pdev, (void __iomem __force *)
458 			       pdev->resource[i].start);
459 	}
460 }
461 
zpci_alloc_iomap(struct zpci_dev * zdev)462 static int zpci_alloc_iomap(struct zpci_dev *zdev)
463 {
464 	unsigned long entry;
465 
466 	spin_lock(&zpci_iomap_lock);
467 	entry = find_first_zero_bit(zpci_iomap_bitmap, ZPCI_IOMAP_ENTRIES);
468 	if (entry == ZPCI_IOMAP_ENTRIES) {
469 		spin_unlock(&zpci_iomap_lock);
470 		return -ENOSPC;
471 	}
472 	set_bit(entry, zpci_iomap_bitmap);
473 	spin_unlock(&zpci_iomap_lock);
474 	return entry;
475 }
476 
zpci_free_iomap(struct zpci_dev * zdev,int entry)477 static void zpci_free_iomap(struct zpci_dev *zdev, int entry)
478 {
479 	spin_lock(&zpci_iomap_lock);
480 	memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry));
481 	clear_bit(entry, zpci_iomap_bitmap);
482 	spin_unlock(&zpci_iomap_lock);
483 }
484 
zpci_do_update_iomap_fh(struct zpci_dev * zdev,u32 fh)485 static void zpci_do_update_iomap_fh(struct zpci_dev *zdev, u32 fh)
486 {
487 	int bar, idx;
488 
489 	spin_lock(&zpci_iomap_lock);
490 	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
491 		if (!zdev->bars[bar].size)
492 			continue;
493 		idx = zdev->bars[bar].map_idx;
494 		if (!zpci_iomap_start[idx].count)
495 			continue;
496 		WRITE_ONCE(zpci_iomap_start[idx].fh, zdev->fh);
497 	}
498 	spin_unlock(&zpci_iomap_lock);
499 }
500 
zpci_update_fh(struct zpci_dev * zdev,u32 fh)501 void zpci_update_fh(struct zpci_dev *zdev, u32 fh)
502 {
503 	if (!fh || zdev->fh == fh)
504 		return;
505 
506 	zdev->fh = fh;
507 	if (zpci_use_mio(zdev))
508 		return;
509 	if (zdev->has_resources && zdev_enabled(zdev))
510 		zpci_do_update_iomap_fh(zdev, fh);
511 }
512 
__alloc_res(struct zpci_dev * zdev,unsigned long start,unsigned long size,unsigned long flags)513 static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start,
514 				    unsigned long size, unsigned long flags)
515 {
516 	struct resource *r;
517 
518 	r = kzalloc(sizeof(*r), GFP_KERNEL);
519 	if (!r)
520 		return NULL;
521 
522 	r->start = start;
523 	r->end = r->start + size - 1;
524 	r->flags = flags;
525 	r->name = zdev->res_name;
526 
527 	if (request_resource(&iomem_resource, r)) {
528 		kfree(r);
529 		return NULL;
530 	}
531 	return r;
532 }
533 
zpci_setup_bus_resources(struct zpci_dev * zdev,struct list_head * resources)534 int zpci_setup_bus_resources(struct zpci_dev *zdev,
535 			     struct list_head *resources)
536 {
537 	unsigned long addr, size, flags;
538 	struct resource *res;
539 	int i, entry;
540 
541 	snprintf(zdev->res_name, sizeof(zdev->res_name),
542 		 "PCI Bus %04x:%02x", zdev->uid, ZPCI_BUS_NR);
543 
544 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
545 		if (!zdev->bars[i].size)
546 			continue;
547 		entry = zpci_alloc_iomap(zdev);
548 		if (entry < 0)
549 			return entry;
550 		zdev->bars[i].map_idx = entry;
551 
552 		/* only MMIO is supported */
553 		flags = IORESOURCE_MEM;
554 		if (zdev->bars[i].val & 8)
555 			flags |= IORESOURCE_PREFETCH;
556 		if (zdev->bars[i].val & 4)
557 			flags |= IORESOURCE_MEM_64;
558 
559 		if (zpci_use_mio(zdev))
560 			addr = (unsigned long) zdev->bars[i].mio_wt;
561 		else
562 			addr = ZPCI_ADDR(entry);
563 		size = 1UL << zdev->bars[i].size;
564 
565 		res = __alloc_res(zdev, addr, size, flags);
566 		if (!res) {
567 			zpci_free_iomap(zdev, entry);
568 			return -ENOMEM;
569 		}
570 		zdev->bars[i].res = res;
571 		pci_add_resource(resources, res);
572 	}
573 	zdev->has_resources = 1;
574 
575 	return 0;
576 }
577 
zpci_cleanup_bus_resources(struct zpci_dev * zdev)578 static void zpci_cleanup_bus_resources(struct zpci_dev *zdev)
579 {
580 	int i;
581 
582 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
583 		if (!zdev->bars[i].size || !zdev->bars[i].res)
584 			continue;
585 
586 		zpci_free_iomap(zdev, zdev->bars[i].map_idx);
587 		release_resource(zdev->bars[i].res);
588 		kfree(zdev->bars[i].res);
589 	}
590 	zdev->has_resources = 0;
591 }
592 
pcibios_device_add(struct pci_dev * pdev)593 int pcibios_device_add(struct pci_dev *pdev)
594 {
595 	struct zpci_dev *zdev = to_zpci(pdev);
596 	struct resource *res;
597 	int i;
598 
599 	/* The pdev has a reference to the zdev via its bus */
600 	zpci_zdev_get(zdev);
601 	if (pdev->is_physfn)
602 		pdev->no_vf_scan = 1;
603 
604 	pdev->dev.groups = zpci_attr_groups;
605 	pdev->dev.dma_ops = &s390_pci_dma_ops;
606 	zpci_map_resources(pdev);
607 
608 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
609 		res = &pdev->resource[i];
610 		if (res->parent || !res->flags)
611 			continue;
612 		pci_claim_resource(pdev, i);
613 	}
614 
615 	return 0;
616 }
617 
pcibios_release_device(struct pci_dev * pdev)618 void pcibios_release_device(struct pci_dev *pdev)
619 {
620 	struct zpci_dev *zdev = to_zpci(pdev);
621 
622 	zpci_unmap_resources(pdev);
623 	zpci_zdev_put(zdev);
624 }
625 
pcibios_enable_device(struct pci_dev * pdev,int mask)626 int pcibios_enable_device(struct pci_dev *pdev, int mask)
627 {
628 	struct zpci_dev *zdev = to_zpci(pdev);
629 
630 	zpci_debug_init_device(zdev, dev_name(&pdev->dev));
631 	zpci_fmb_enable_device(zdev);
632 
633 	return pci_enable_resources(pdev, mask);
634 }
635 
pcibios_disable_device(struct pci_dev * pdev)636 void pcibios_disable_device(struct pci_dev *pdev)
637 {
638 	struct zpci_dev *zdev = to_zpci(pdev);
639 
640 	zpci_fmb_disable_device(zdev);
641 	zpci_debug_exit_device(zdev);
642 }
643 
__zpci_register_domain(int domain)644 static int __zpci_register_domain(int domain)
645 {
646 	spin_lock(&zpci_domain_lock);
647 	if (test_bit(domain, zpci_domain)) {
648 		spin_unlock(&zpci_domain_lock);
649 		pr_err("Domain %04x is already assigned\n", domain);
650 		return -EEXIST;
651 	}
652 	set_bit(domain, zpci_domain);
653 	spin_unlock(&zpci_domain_lock);
654 	return domain;
655 }
656 
__zpci_alloc_domain(void)657 static int __zpci_alloc_domain(void)
658 {
659 	int domain;
660 
661 	spin_lock(&zpci_domain_lock);
662 	/*
663 	 * We can always auto allocate domains below ZPCI_NR_DEVICES.
664 	 * There is either a free domain or we have reached the maximum in
665 	 * which case we would have bailed earlier.
666 	 */
667 	domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES);
668 	set_bit(domain, zpci_domain);
669 	spin_unlock(&zpci_domain_lock);
670 	return domain;
671 }
672 
zpci_alloc_domain(int domain)673 int zpci_alloc_domain(int domain)
674 {
675 	if (zpci_unique_uid) {
676 		if (domain)
677 			return __zpci_register_domain(domain);
678 		pr_warn("UID checking was active but no UID is provided: switching to automatic domain allocation\n");
679 		update_uid_checking(false);
680 	}
681 	return __zpci_alloc_domain();
682 }
683 
zpci_free_domain(int domain)684 void zpci_free_domain(int domain)
685 {
686 	spin_lock(&zpci_domain_lock);
687 	clear_bit(domain, zpci_domain);
688 	spin_unlock(&zpci_domain_lock);
689 }
690 
691 
zpci_enable_device(struct zpci_dev * zdev)692 int zpci_enable_device(struct zpci_dev *zdev)
693 {
694 	u32 fh = zdev->fh;
695 	int rc = 0;
696 
697 	if (clp_enable_fh(zdev, &fh, ZPCI_NR_DMA_SPACES))
698 		rc = -EIO;
699 	else
700 		zpci_update_fh(zdev, fh);
701 	return rc;
702 }
703 
zpci_disable_device(struct zpci_dev * zdev)704 int zpci_disable_device(struct zpci_dev *zdev)
705 {
706 	u32 fh = zdev->fh;
707 	int cc, rc = 0;
708 
709 	cc = clp_disable_fh(zdev, &fh);
710 	if (!cc) {
711 		zpci_update_fh(zdev, fh);
712 	} else if (cc == CLP_RC_SETPCIFN_ALRDY) {
713 		pr_info("Disabling PCI function %08x had no effect as it was already disabled\n",
714 			zdev->fid);
715 		/* Function is already disabled - update handle */
716 		rc = clp_refresh_fh(zdev->fid, &fh);
717 		if (!rc) {
718 			zpci_update_fh(zdev, fh);
719 			rc = -EINVAL;
720 		}
721 	} else {
722 		rc = -EIO;
723 	}
724 	return rc;
725 }
726 
727 /**
728  * zpci_hot_reset_device - perform a reset of the given zPCI function
729  * @zdev: the slot which should be reset
730  *
731  * Performs a low level reset of the zPCI function. The reset is low level in
732  * the sense that the zPCI function can be reset without detaching it from the
733  * common PCI subsystem. The reset may be performed while under control of
734  * either DMA or IOMMU APIs in which case the existing DMA/IOMMU translation
735  * table is reinstated at the end of the reset.
736  *
737  * After the reset the functions internal state is reset to an initial state
738  * equivalent to its state during boot when first probing a driver.
739  * Consequently after reset the PCI function requires re-initialization via the
740  * common PCI code including re-enabling IRQs via pci_alloc_irq_vectors()
741  * and enabling the function via e.g.pci_enablde_device_flags().The caller
742  * must guard against concurrent reset attempts.
743  *
744  * In most cases this function should not be called directly but through
745  * pci_reset_function() or pci_reset_bus() which handle the save/restore and
746  * locking.
747  *
748  * Return: 0 on success and an error value otherwise
749  */
zpci_hot_reset_device(struct zpci_dev * zdev)750 int zpci_hot_reset_device(struct zpci_dev *zdev)
751 {
752 	int rc;
753 
754 	zpci_dbg(3, "rst fid:%x, fh:%x\n", zdev->fid, zdev->fh);
755 	if (zdev_enabled(zdev)) {
756 		/* Disables device access, DMAs and IRQs (reset state) */
757 		rc = zpci_disable_device(zdev);
758 		/*
759 		 * Due to a z/VM vs LPAR inconsistency in the error state the
760 		 * FH may indicate an enabled device but disable says the
761 		 * device is already disabled don't treat it as an error here.
762 		 */
763 		if (rc == -EINVAL)
764 			rc = 0;
765 		if (rc)
766 			return rc;
767 	}
768 
769 	rc = zpci_enable_device(zdev);
770 	if (rc)
771 		return rc;
772 
773 	if (zdev->dma_table)
774 		rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
775 					virt_to_phys(zdev->dma_table));
776 	else
777 		rc = zpci_dma_init_device(zdev);
778 	if (rc) {
779 		zpci_disable_device(zdev);
780 		return rc;
781 	}
782 
783 	return 0;
784 }
785 
786 /**
787  * zpci_create_device() - Create a new zpci_dev and add it to the zbus
788  * @fid: Function ID of the device to be created
789  * @fh: Current Function Handle of the device to be created
790  * @state: Initial state after creation either Standby or Configured
791  *
792  * Creates a new zpci device and adds it to its, possibly newly created, zbus
793  * as well as zpci_list.
794  *
795  * Returns: the zdev on success or an error pointer otherwise
796  */
zpci_create_device(u32 fid,u32 fh,enum zpci_state state)797 struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state)
798 {
799 	struct zpci_dev *zdev;
800 	int rc;
801 
802 	zpci_dbg(1, "add fid:%x, fh:%x, c:%d\n", fid, fh, state);
803 	zdev = kzalloc(sizeof(*zdev), GFP_KERNEL);
804 	if (!zdev)
805 		return ERR_PTR(-ENOMEM);
806 
807 	/* FID and Function Handle are the static/dynamic identifiers */
808 	zdev->fid = fid;
809 	zdev->fh = fh;
810 
811 	/* Query function properties and update zdev */
812 	rc = clp_query_pci_fn(zdev);
813 	if (rc)
814 		goto error;
815 	zdev->state =  state;
816 
817 	kref_init(&zdev->kref);
818 	mutex_init(&zdev->lock);
819 
820 	rc = zpci_init_iommu(zdev);
821 	if (rc)
822 		goto error;
823 
824 	rc = zpci_bus_device_register(zdev, &pci_root_ops);
825 	if (rc)
826 		goto error_destroy_iommu;
827 
828 	spin_lock(&zpci_list_lock);
829 	list_add_tail(&zdev->entry, &zpci_list);
830 	spin_unlock(&zpci_list_lock);
831 
832 	return zdev;
833 
834 error_destroy_iommu:
835 	zpci_destroy_iommu(zdev);
836 error:
837 	zpci_dbg(0, "add fid:%x, rc:%d\n", fid, rc);
838 	kfree(zdev);
839 	return ERR_PTR(rc);
840 }
841 
zpci_is_device_configured(struct zpci_dev * zdev)842 bool zpci_is_device_configured(struct zpci_dev *zdev)
843 {
844 	enum zpci_state state = zdev->state;
845 
846 	return state != ZPCI_FN_STATE_RESERVED &&
847 		state != ZPCI_FN_STATE_STANDBY;
848 }
849 
850 /**
851  * zpci_scan_configured_device() - Scan a freshly configured zpci_dev
852  * @zdev: The zpci_dev to be configured
853  * @fh: The general function handle supplied by the platform
854  *
855  * Given a device in the configuration state Configured, enables, scans and
856  * adds it to the common code PCI subsystem if possible. If the PCI device is
857  * parked because we can not yet create a PCI bus because we have not seen
858  * function 0, it is ignored but will be scanned once function 0 appears.
859  * If any failure occurs, the zpci_dev is left disabled.
860  *
861  * Return: 0 on success, or an error code otherwise
862  */
zpci_scan_configured_device(struct zpci_dev * zdev,u32 fh)863 int zpci_scan_configured_device(struct zpci_dev *zdev, u32 fh)
864 {
865 	int rc;
866 
867 	zpci_update_fh(zdev, fh);
868 	/* the PCI function will be scanned once function 0 appears */
869 	if (!zdev->zbus->bus)
870 		return 0;
871 
872 	/* For function 0 on a multi-function bus scan whole bus as we might
873 	 * have to pick up existing functions waiting for it to allow creating
874 	 * the PCI bus
875 	 */
876 	if (zdev->devfn == 0 && zdev->zbus->multifunction)
877 		rc = zpci_bus_scan_bus(zdev->zbus);
878 	else
879 		rc = zpci_bus_scan_device(zdev);
880 
881 	return rc;
882 }
883 
884 /**
885  * zpci_deconfigure_device() - Deconfigure a zpci_dev
886  * @zdev: The zpci_dev to configure
887  *
888  * Deconfigure a zPCI function that is currently configured and possibly known
889  * to the common code PCI subsystem.
890  * If any failure occurs the device is left as is.
891  *
892  * Return: 0 on success, or an error code otherwise
893  */
zpci_deconfigure_device(struct zpci_dev * zdev)894 int zpci_deconfigure_device(struct zpci_dev *zdev)
895 {
896 	int rc;
897 
898 	if (zdev->zbus->bus)
899 		zpci_bus_remove_device(zdev, false);
900 
901 	if (zdev->dma_table) {
902 		rc = zpci_dma_exit_device(zdev);
903 		if (rc)
904 			return rc;
905 	}
906 	if (zdev_enabled(zdev)) {
907 		rc = zpci_disable_device(zdev);
908 		if (rc)
909 			return rc;
910 	}
911 
912 	rc = sclp_pci_deconfigure(zdev->fid);
913 	zpci_dbg(3, "deconf fid:%x, rc:%d\n", zdev->fid, rc);
914 	if (rc)
915 		return rc;
916 	zdev->state = ZPCI_FN_STATE_STANDBY;
917 
918 	return 0;
919 }
920 
921 /**
922  * zpci_device_reserved() - Mark device as resverved
923  * @zdev: the zpci_dev that was reserved
924  *
925  * Handle the case that a given zPCI function was reserved by another system.
926  * After a call to this function the zpci_dev can not be found via
927  * get_zdev_by_fid() anymore but may still be accessible via existing
928  * references though it will not be functional anymore.
929  */
zpci_device_reserved(struct zpci_dev * zdev)930 void zpci_device_reserved(struct zpci_dev *zdev)
931 {
932 	if (zdev->has_hp_slot)
933 		zpci_exit_slot(zdev);
934 	/*
935 	 * Remove device from zpci_list as it is going away. This also
936 	 * makes sure we ignore subsequent zPCI events for this device.
937 	 */
938 	spin_lock(&zpci_list_lock);
939 	list_del(&zdev->entry);
940 	spin_unlock(&zpci_list_lock);
941 	zdev->state = ZPCI_FN_STATE_RESERVED;
942 	zpci_dbg(3, "rsv fid:%x\n", zdev->fid);
943 	zpci_zdev_put(zdev);
944 }
945 
zpci_release_device(struct kref * kref)946 void zpci_release_device(struct kref *kref)
947 {
948 	struct zpci_dev *zdev = container_of(kref, struct zpci_dev, kref);
949 	int ret;
950 
951 	if (zdev->zbus->bus)
952 		zpci_bus_remove_device(zdev, false);
953 
954 	if (zdev->dma_table)
955 		zpci_dma_exit_device(zdev);
956 	if (zdev_enabled(zdev))
957 		zpci_disable_device(zdev);
958 
959 	switch (zdev->state) {
960 	case ZPCI_FN_STATE_CONFIGURED:
961 		ret = sclp_pci_deconfigure(zdev->fid);
962 		zpci_dbg(3, "deconf fid:%x, rc:%d\n", zdev->fid, ret);
963 		fallthrough;
964 	case ZPCI_FN_STATE_STANDBY:
965 		if (zdev->has_hp_slot)
966 			zpci_exit_slot(zdev);
967 		spin_lock(&zpci_list_lock);
968 		list_del(&zdev->entry);
969 		spin_unlock(&zpci_list_lock);
970 		zpci_dbg(3, "rsv fid:%x\n", zdev->fid);
971 		fallthrough;
972 	case ZPCI_FN_STATE_RESERVED:
973 		if (zdev->has_resources)
974 			zpci_cleanup_bus_resources(zdev);
975 		zpci_bus_device_unregister(zdev);
976 		zpci_destroy_iommu(zdev);
977 		fallthrough;
978 	default:
979 		break;
980 	}
981 	zpci_dbg(3, "rem fid:%x\n", zdev->fid);
982 	kfree(zdev);
983 }
984 
zpci_report_error(struct pci_dev * pdev,struct zpci_report_error_header * report)985 int zpci_report_error(struct pci_dev *pdev,
986 		      struct zpci_report_error_header *report)
987 {
988 	struct zpci_dev *zdev = to_zpci(pdev);
989 
990 	return sclp_pci_report(report, zdev->fh, zdev->fid);
991 }
992 EXPORT_SYMBOL(zpci_report_error);
993 
994 /**
995  * zpci_clear_error_state() - Clears the zPCI error state of the device
996  * @zdev: The zdev for which the zPCI error state should be reset
997  *
998  * Clear the zPCI error state of the device. If clearing the zPCI error state
999  * fails the device is left in the error state. In this case it may make sense
1000  * to call zpci_io_perm_failure() on the associated pdev if it exists.
1001  *
1002  * Returns: 0 on success, -EIO otherwise
1003  */
zpci_clear_error_state(struct zpci_dev * zdev)1004 int zpci_clear_error_state(struct zpci_dev *zdev)
1005 {
1006 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_ERROR);
1007 	struct zpci_fib fib = {0};
1008 	u8 status;
1009 	int cc;
1010 
1011 	cc = zpci_mod_fc(req, &fib, &status);
1012 	if (cc) {
1013 		zpci_dbg(3, "ces fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status);
1014 		return -EIO;
1015 	}
1016 
1017 	return 0;
1018 }
1019 
1020 /**
1021  * zpci_reset_load_store_blocked() - Re-enables L/S from error state
1022  * @zdev: The zdev for which to unblock load/store access
1023  *
1024  * Re-enables load/store access for a PCI function in the error state while
1025  * keeping DMA blocked. In this state drivers can poke MMIO space to determine
1026  * if error recovery is possible while catching any rogue DMA access from the
1027  * device.
1028  *
1029  * Returns: 0 on success, -EIO otherwise
1030  */
zpci_reset_load_store_blocked(struct zpci_dev * zdev)1031 int zpci_reset_load_store_blocked(struct zpci_dev *zdev)
1032 {
1033 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_BLOCK);
1034 	struct zpci_fib fib = {0};
1035 	u8 status;
1036 	int cc;
1037 
1038 	cc = zpci_mod_fc(req, &fib, &status);
1039 	if (cc) {
1040 		zpci_dbg(3, "rls fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status);
1041 		return -EIO;
1042 	}
1043 
1044 	return 0;
1045 }
1046 
zpci_mem_init(void)1047 static int zpci_mem_init(void)
1048 {
1049 	BUILD_BUG_ON(!is_power_of_2(__alignof__(struct zpci_fmb)) ||
1050 		     __alignof__(struct zpci_fmb) < sizeof(struct zpci_fmb));
1051 
1052 	zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb),
1053 					   __alignof__(struct zpci_fmb), 0, NULL);
1054 	if (!zdev_fmb_cache)
1055 		goto error_fmb;
1056 
1057 	zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES,
1058 				   sizeof(*zpci_iomap_start), GFP_KERNEL);
1059 	if (!zpci_iomap_start)
1060 		goto error_iomap;
1061 
1062 	zpci_iomap_bitmap = kcalloc(BITS_TO_LONGS(ZPCI_IOMAP_ENTRIES),
1063 				    sizeof(*zpci_iomap_bitmap), GFP_KERNEL);
1064 	if (!zpci_iomap_bitmap)
1065 		goto error_iomap_bitmap;
1066 
1067 	if (static_branch_likely(&have_mio))
1068 		clp_setup_writeback_mio();
1069 
1070 	return 0;
1071 error_iomap_bitmap:
1072 	kfree(zpci_iomap_start);
1073 error_iomap:
1074 	kmem_cache_destroy(zdev_fmb_cache);
1075 error_fmb:
1076 	return -ENOMEM;
1077 }
1078 
zpci_mem_exit(void)1079 static void zpci_mem_exit(void)
1080 {
1081 	kfree(zpci_iomap_bitmap);
1082 	kfree(zpci_iomap_start);
1083 	kmem_cache_destroy(zdev_fmb_cache);
1084 }
1085 
1086 static unsigned int s390_pci_probe __initdata = 1;
1087 unsigned int s390_pci_force_floating __initdata;
1088 static unsigned int s390_pci_initialized;
1089 
pcibios_setup(char * str)1090 char * __init pcibios_setup(char *str)
1091 {
1092 	if (!strcmp(str, "off")) {
1093 		s390_pci_probe = 0;
1094 		return NULL;
1095 	}
1096 	if (!strcmp(str, "nomio")) {
1097 		S390_lowcore.machine_flags &= ~MACHINE_FLAG_PCI_MIO;
1098 		return NULL;
1099 	}
1100 	if (!strcmp(str, "force_floating")) {
1101 		s390_pci_force_floating = 1;
1102 		return NULL;
1103 	}
1104 	if (!strcmp(str, "norid")) {
1105 		s390_pci_no_rid = 1;
1106 		return NULL;
1107 	}
1108 	return str;
1109 }
1110 
zpci_is_enabled(void)1111 bool zpci_is_enabled(void)
1112 {
1113 	return s390_pci_initialized;
1114 }
1115 
pci_base_init(void)1116 static int __init pci_base_init(void)
1117 {
1118 	int rc;
1119 
1120 	if (!s390_pci_probe)
1121 		return 0;
1122 
1123 	if (!test_facility(69) || !test_facility(71)) {
1124 		pr_info("PCI is not supported because CPU facilities 69 or 71 are not available\n");
1125 		return 0;
1126 	}
1127 
1128 	if (MACHINE_HAS_PCI_MIO) {
1129 		static_branch_enable(&have_mio);
1130 		ctl_set_bit(2, 5);
1131 	}
1132 
1133 	rc = zpci_debug_init();
1134 	if (rc)
1135 		goto out;
1136 
1137 	rc = zpci_mem_init();
1138 	if (rc)
1139 		goto out_mem;
1140 
1141 	rc = zpci_irq_init();
1142 	if (rc)
1143 		goto out_irq;
1144 
1145 	rc = zpci_dma_init();
1146 	if (rc)
1147 		goto out_dma;
1148 
1149 	rc = clp_scan_pci_devices();
1150 	if (rc)
1151 		goto out_find;
1152 	zpci_bus_scan_busses();
1153 
1154 	s390_pci_initialized = 1;
1155 	return 0;
1156 
1157 out_find:
1158 	zpci_dma_exit();
1159 out_dma:
1160 	zpci_irq_exit();
1161 out_irq:
1162 	zpci_mem_exit();
1163 out_mem:
1164 	zpci_debug_exit();
1165 out:
1166 	return rc;
1167 }
1168 subsys_initcall_sync(pci_base_init);
1169