1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2020 Intel Corporation
4 * Author: Johannes Berg <johannes@sipsolutions.net>
5 */
6 #include <linux/module.h>
7 #include <linux/pci.h>
8 #include <linux/virtio.h>
9 #include <linux/virtio_config.h>
10 #include <linux/logic_iomem.h>
11 #include <linux/irqdomain.h>
12 #include <linux/virtio_pcidev.h>
13 #include <linux/virtio-uml.h>
14 #include <linux/delay.h>
15 #include <linux/msi.h>
16 #include <asm/unaligned.h>
17 #include <irq_kern.h>
18
19 #define MAX_DEVICES 8
20 #define MAX_MSI_VECTORS 32
21 #define CFG_SPACE_SIZE 4096
22
23 /* for MSI-X we have a 32-bit payload */
24 #define MAX_IRQ_MSG_SIZE (sizeof(struct virtio_pcidev_msg) + sizeof(u32))
25 #define NUM_IRQ_MSGS 10
26
27 #define HANDLE_NO_FREE(ptr) ((void *)((unsigned long)(ptr) | 1))
28 #define HANDLE_IS_NO_FREE(ptr) ((unsigned long)(ptr) & 1)
29
30 struct um_pci_device {
31 struct virtio_device *vdev;
32
33 /* for now just standard BARs */
34 u8 resptr[PCI_STD_NUM_BARS];
35
36 struct virtqueue *cmd_vq, *irq_vq;
37
38 #define UM_PCI_STAT_WAITING 0
39 unsigned long status;
40
41 int irq;
42 };
43
44 struct um_pci_device_reg {
45 struct um_pci_device *dev;
46 void __iomem *iomem;
47 };
48
49 static struct pci_host_bridge *bridge;
50 static DEFINE_MUTEX(um_pci_mtx);
51 static struct um_pci_device_reg um_pci_devices[MAX_DEVICES];
52 static struct fwnode_handle *um_pci_fwnode;
53 static struct irq_domain *um_pci_inner_domain;
54 static struct irq_domain *um_pci_msi_domain;
55 static unsigned long um_pci_msi_used[BITS_TO_LONGS(MAX_MSI_VECTORS)];
56
57 #define UM_VIRT_PCI_MAXDELAY 40000
58
59 struct um_pci_message_buffer {
60 struct virtio_pcidev_msg hdr;
61 u8 data[8];
62 };
63
64 static struct um_pci_message_buffer __percpu *um_pci_msg_bufs;
65
um_pci_send_cmd(struct um_pci_device * dev,struct virtio_pcidev_msg * cmd,unsigned int cmd_size,const void * extra,unsigned int extra_size,void * out,unsigned int out_size)66 static int um_pci_send_cmd(struct um_pci_device *dev,
67 struct virtio_pcidev_msg *cmd,
68 unsigned int cmd_size,
69 const void *extra, unsigned int extra_size,
70 void *out, unsigned int out_size)
71 {
72 struct scatterlist out_sg, extra_sg, in_sg;
73 struct scatterlist *sgs_list[] = {
74 [0] = &out_sg,
75 [1] = extra ? &extra_sg : &in_sg,
76 [2] = extra ? &in_sg : NULL,
77 };
78 struct um_pci_message_buffer *buf;
79 int delay_count = 0;
80 int ret, len;
81 bool posted;
82
83 if (WARN_ON(cmd_size < sizeof(*cmd) || cmd_size > sizeof(*buf)))
84 return -EINVAL;
85
86 switch (cmd->op) {
87 case VIRTIO_PCIDEV_OP_CFG_WRITE:
88 case VIRTIO_PCIDEV_OP_MMIO_WRITE:
89 case VIRTIO_PCIDEV_OP_MMIO_MEMSET:
90 /* in PCI, writes are posted, so don't wait */
91 posted = !out;
92 WARN_ON(!posted);
93 break;
94 default:
95 posted = false;
96 break;
97 }
98
99 buf = get_cpu_var(um_pci_msg_bufs);
100 memcpy(buf, cmd, cmd_size);
101
102 if (posted) {
103 u8 *ncmd = kmalloc(cmd_size + extra_size, GFP_ATOMIC);
104
105 if (ncmd) {
106 memcpy(ncmd, cmd, cmd_size);
107 if (extra)
108 memcpy(ncmd + cmd_size, extra, extra_size);
109 cmd = (void *)ncmd;
110 cmd_size += extra_size;
111 extra = NULL;
112 extra_size = 0;
113 } else {
114 /* try without allocating memory */
115 posted = false;
116 cmd = (void *)buf;
117 }
118 } else {
119 cmd = (void *)buf;
120 }
121
122 sg_init_one(&out_sg, cmd, cmd_size);
123 if (extra)
124 sg_init_one(&extra_sg, extra, extra_size);
125 if (out)
126 sg_init_one(&in_sg, out, out_size);
127
128 /* add to internal virtio queue */
129 ret = virtqueue_add_sgs(dev->cmd_vq, sgs_list,
130 extra ? 2 : 1,
131 out ? 1 : 0,
132 posted ? cmd : HANDLE_NO_FREE(cmd),
133 GFP_ATOMIC);
134 if (ret)
135 goto out;
136
137 if (posted) {
138 virtqueue_kick(dev->cmd_vq);
139 ret = 0;
140 goto out;
141 }
142
143 /* kick and poll for getting a response on the queue */
144 set_bit(UM_PCI_STAT_WAITING, &dev->status);
145 virtqueue_kick(dev->cmd_vq);
146
147 while (1) {
148 void *completed = virtqueue_get_buf(dev->cmd_vq, &len);
149
150 if (completed == HANDLE_NO_FREE(cmd))
151 break;
152
153 if (completed && !HANDLE_IS_NO_FREE(completed))
154 kfree(completed);
155
156 if (WARN_ONCE(virtqueue_is_broken(dev->cmd_vq) ||
157 ++delay_count > UM_VIRT_PCI_MAXDELAY,
158 "um virt-pci delay: %d", delay_count)) {
159 ret = -EIO;
160 break;
161 }
162 udelay(1);
163 }
164 clear_bit(UM_PCI_STAT_WAITING, &dev->status);
165
166 out:
167 put_cpu_var(um_pci_msg_bufs);
168 return ret;
169 }
170
um_pci_cfgspace_read(void * priv,unsigned int offset,int size)171 static unsigned long um_pci_cfgspace_read(void *priv, unsigned int offset,
172 int size)
173 {
174 struct um_pci_device_reg *reg = priv;
175 struct um_pci_device *dev = reg->dev;
176 struct virtio_pcidev_msg hdr = {
177 .op = VIRTIO_PCIDEV_OP_CFG_READ,
178 .size = size,
179 .addr = offset,
180 };
181 /* buf->data is maximum size - we may only use parts of it */
182 struct um_pci_message_buffer *buf;
183 u8 *data;
184 unsigned long ret = ULONG_MAX;
185
186 if (!dev)
187 return ULONG_MAX;
188
189 buf = get_cpu_var(um_pci_msg_bufs);
190 data = buf->data;
191
192 memset(buf->data, 0xff, sizeof(buf->data));
193
194 switch (size) {
195 case 1:
196 case 2:
197 case 4:
198 #ifdef CONFIG_64BIT
199 case 8:
200 #endif
201 break;
202 default:
203 WARN(1, "invalid config space read size %d\n", size);
204 goto out;
205 }
206
207 if (um_pci_send_cmd(dev, &hdr, sizeof(hdr), NULL, 0, data, 8))
208 goto out;
209
210 switch (size) {
211 case 1:
212 ret = data[0];
213 break;
214 case 2:
215 ret = le16_to_cpup((void *)data);
216 break;
217 case 4:
218 ret = le32_to_cpup((void *)data);
219 break;
220 #ifdef CONFIG_64BIT
221 case 8:
222 ret = le64_to_cpup((void *)data);
223 break;
224 #endif
225 default:
226 break;
227 }
228
229 out:
230 put_cpu_var(um_pci_msg_bufs);
231 return ret;
232 }
233
um_pci_cfgspace_write(void * priv,unsigned int offset,int size,unsigned long val)234 static void um_pci_cfgspace_write(void *priv, unsigned int offset, int size,
235 unsigned long val)
236 {
237 struct um_pci_device_reg *reg = priv;
238 struct um_pci_device *dev = reg->dev;
239 struct {
240 struct virtio_pcidev_msg hdr;
241 /* maximum size - we may only use parts of it */
242 u8 data[8];
243 } msg = {
244 .hdr = {
245 .op = VIRTIO_PCIDEV_OP_CFG_WRITE,
246 .size = size,
247 .addr = offset,
248 },
249 };
250
251 if (!dev)
252 return;
253
254 switch (size) {
255 case 1:
256 msg.data[0] = (u8)val;
257 break;
258 case 2:
259 put_unaligned_le16(val, (void *)msg.data);
260 break;
261 case 4:
262 put_unaligned_le32(val, (void *)msg.data);
263 break;
264 #ifdef CONFIG_64BIT
265 case 8:
266 put_unaligned_le64(val, (void *)msg.data);
267 break;
268 #endif
269 default:
270 WARN(1, "invalid config space write size %d\n", size);
271 return;
272 }
273
274 WARN_ON(um_pci_send_cmd(dev, &msg.hdr, sizeof(msg), NULL, 0, NULL, 0));
275 }
276
277 static const struct logic_iomem_ops um_pci_device_cfgspace_ops = {
278 .read = um_pci_cfgspace_read,
279 .write = um_pci_cfgspace_write,
280 };
281
um_pci_bar_copy_from(void * priv,void * buffer,unsigned int offset,int size)282 static void um_pci_bar_copy_from(void *priv, void *buffer,
283 unsigned int offset, int size)
284 {
285 u8 *resptr = priv;
286 struct um_pci_device *dev = container_of(resptr - *resptr,
287 struct um_pci_device,
288 resptr[0]);
289 struct virtio_pcidev_msg hdr = {
290 .op = VIRTIO_PCIDEV_OP_MMIO_READ,
291 .bar = *resptr,
292 .size = size,
293 .addr = offset,
294 };
295
296 memset(buffer, 0xff, size);
297
298 um_pci_send_cmd(dev, &hdr, sizeof(hdr), NULL, 0, buffer, size);
299 }
300
um_pci_bar_read(void * priv,unsigned int offset,int size)301 static unsigned long um_pci_bar_read(void *priv, unsigned int offset,
302 int size)
303 {
304 /* buf->data is maximum size - we may only use parts of it */
305 struct um_pci_message_buffer *buf;
306 u8 *data;
307 unsigned long ret = ULONG_MAX;
308
309 buf = get_cpu_var(um_pci_msg_bufs);
310 data = buf->data;
311
312 switch (size) {
313 case 1:
314 case 2:
315 case 4:
316 #ifdef CONFIG_64BIT
317 case 8:
318 #endif
319 break;
320 default:
321 WARN(1, "invalid config space read size %d\n", size);
322 goto out;
323 }
324
325 um_pci_bar_copy_from(priv, data, offset, size);
326
327 switch (size) {
328 case 1:
329 ret = data[0];
330 break;
331 case 2:
332 ret = le16_to_cpup((void *)data);
333 break;
334 case 4:
335 ret = le32_to_cpup((void *)data);
336 break;
337 #ifdef CONFIG_64BIT
338 case 8:
339 ret = le64_to_cpup((void *)data);
340 break;
341 #endif
342 default:
343 break;
344 }
345
346 out:
347 put_cpu_var(um_pci_msg_bufs);
348 return ret;
349 }
350
um_pci_bar_copy_to(void * priv,unsigned int offset,const void * buffer,int size)351 static void um_pci_bar_copy_to(void *priv, unsigned int offset,
352 const void *buffer, int size)
353 {
354 u8 *resptr = priv;
355 struct um_pci_device *dev = container_of(resptr - *resptr,
356 struct um_pci_device,
357 resptr[0]);
358 struct virtio_pcidev_msg hdr = {
359 .op = VIRTIO_PCIDEV_OP_MMIO_WRITE,
360 .bar = *resptr,
361 .size = size,
362 .addr = offset,
363 };
364
365 um_pci_send_cmd(dev, &hdr, sizeof(hdr), buffer, size, NULL, 0);
366 }
367
um_pci_bar_write(void * priv,unsigned int offset,int size,unsigned long val)368 static void um_pci_bar_write(void *priv, unsigned int offset, int size,
369 unsigned long val)
370 {
371 /* maximum size - we may only use parts of it */
372 u8 data[8];
373
374 switch (size) {
375 case 1:
376 data[0] = (u8)val;
377 break;
378 case 2:
379 put_unaligned_le16(val, (void *)data);
380 break;
381 case 4:
382 put_unaligned_le32(val, (void *)data);
383 break;
384 #ifdef CONFIG_64BIT
385 case 8:
386 put_unaligned_le64(val, (void *)data);
387 break;
388 #endif
389 default:
390 WARN(1, "invalid config space write size %d\n", size);
391 return;
392 }
393
394 um_pci_bar_copy_to(priv, offset, data, size);
395 }
396
um_pci_bar_set(void * priv,unsigned int offset,u8 value,int size)397 static void um_pci_bar_set(void *priv, unsigned int offset, u8 value, int size)
398 {
399 u8 *resptr = priv;
400 struct um_pci_device *dev = container_of(resptr - *resptr,
401 struct um_pci_device,
402 resptr[0]);
403 struct {
404 struct virtio_pcidev_msg hdr;
405 u8 data;
406 } msg = {
407 .hdr = {
408 .op = VIRTIO_PCIDEV_OP_CFG_WRITE,
409 .bar = *resptr,
410 .size = size,
411 .addr = offset,
412 },
413 .data = value,
414 };
415
416 um_pci_send_cmd(dev, &msg.hdr, sizeof(msg), NULL, 0, NULL, 0);
417 }
418
419 static const struct logic_iomem_ops um_pci_device_bar_ops = {
420 .read = um_pci_bar_read,
421 .write = um_pci_bar_write,
422 .set = um_pci_bar_set,
423 .copy_from = um_pci_bar_copy_from,
424 .copy_to = um_pci_bar_copy_to,
425 };
426
um_pci_map_bus(struct pci_bus * bus,unsigned int devfn,int where)427 static void __iomem *um_pci_map_bus(struct pci_bus *bus, unsigned int devfn,
428 int where)
429 {
430 struct um_pci_device_reg *dev;
431 unsigned int busn = bus->number;
432
433 if (busn > 0)
434 return NULL;
435
436 /* not allowing functions for now ... */
437 if (devfn % 8)
438 return NULL;
439
440 if (devfn / 8 >= ARRAY_SIZE(um_pci_devices))
441 return NULL;
442
443 dev = &um_pci_devices[devfn / 8];
444 if (!dev)
445 return NULL;
446
447 return (void __iomem *)((unsigned long)dev->iomem + where);
448 }
449
450 static struct pci_ops um_pci_ops = {
451 .map_bus = um_pci_map_bus,
452 .read = pci_generic_config_read,
453 .write = pci_generic_config_write,
454 };
455
um_pci_rescan(void)456 static void um_pci_rescan(void)
457 {
458 pci_lock_rescan_remove();
459 pci_rescan_bus(bridge->bus);
460 pci_unlock_rescan_remove();
461 }
462
um_pci_irq_vq_addbuf(struct virtqueue * vq,void * buf,bool kick)463 static void um_pci_irq_vq_addbuf(struct virtqueue *vq, void *buf, bool kick)
464 {
465 struct scatterlist sg[1];
466
467 sg_init_one(sg, buf, MAX_IRQ_MSG_SIZE);
468 if (virtqueue_add_inbuf(vq, sg, 1, buf, GFP_ATOMIC))
469 kfree(buf);
470 else if (kick)
471 virtqueue_kick(vq);
472 }
473
um_pci_handle_irq_message(struct virtqueue * vq,struct virtio_pcidev_msg * msg)474 static void um_pci_handle_irq_message(struct virtqueue *vq,
475 struct virtio_pcidev_msg *msg)
476 {
477 struct virtio_device *vdev = vq->vdev;
478 struct um_pci_device *dev = vdev->priv;
479
480 /* we should properly chain interrupts, but on ARCH=um we don't care */
481
482 switch (msg->op) {
483 case VIRTIO_PCIDEV_OP_INT:
484 generic_handle_irq(dev->irq);
485 break;
486 case VIRTIO_PCIDEV_OP_MSI:
487 /* our MSI message is just the interrupt number */
488 if (msg->size == sizeof(u32))
489 generic_handle_irq(le32_to_cpup((void *)msg->data));
490 else
491 generic_handle_irq(le16_to_cpup((void *)msg->data));
492 break;
493 case VIRTIO_PCIDEV_OP_PME:
494 /* nothing to do - we already woke up due to the message */
495 break;
496 default:
497 dev_err(&vdev->dev, "unexpected virt-pci message %d\n", msg->op);
498 break;
499 }
500 }
501
um_pci_cmd_vq_cb(struct virtqueue * vq)502 static void um_pci_cmd_vq_cb(struct virtqueue *vq)
503 {
504 struct virtio_device *vdev = vq->vdev;
505 struct um_pci_device *dev = vdev->priv;
506 void *cmd;
507 int len;
508
509 if (test_bit(UM_PCI_STAT_WAITING, &dev->status))
510 return;
511
512 while ((cmd = virtqueue_get_buf(vq, &len))) {
513 if (WARN_ON(HANDLE_IS_NO_FREE(cmd)))
514 continue;
515 kfree(cmd);
516 }
517 }
518
um_pci_irq_vq_cb(struct virtqueue * vq)519 static void um_pci_irq_vq_cb(struct virtqueue *vq)
520 {
521 struct virtio_pcidev_msg *msg;
522 int len;
523
524 while ((msg = virtqueue_get_buf(vq, &len))) {
525 if (len >= sizeof(*msg))
526 um_pci_handle_irq_message(vq, msg);
527
528 /* recycle the message buffer */
529 um_pci_irq_vq_addbuf(vq, msg, true);
530 }
531 }
532
um_pci_init_vqs(struct um_pci_device * dev)533 static int um_pci_init_vqs(struct um_pci_device *dev)
534 {
535 struct virtqueue *vqs[2];
536 static const char *const names[2] = { "cmd", "irq" };
537 vq_callback_t *cbs[2] = { um_pci_cmd_vq_cb, um_pci_irq_vq_cb };
538 int err, i;
539
540 err = virtio_find_vqs(dev->vdev, 2, vqs, cbs, names, NULL);
541 if (err)
542 return err;
543
544 dev->cmd_vq = vqs[0];
545 dev->irq_vq = vqs[1];
546
547 virtio_device_ready(dev->vdev);
548
549 for (i = 0; i < NUM_IRQ_MSGS; i++) {
550 void *msg = kzalloc(MAX_IRQ_MSG_SIZE, GFP_KERNEL);
551
552 if (msg)
553 um_pci_irq_vq_addbuf(dev->irq_vq, msg, false);
554 }
555
556 virtqueue_kick(dev->irq_vq);
557
558 return 0;
559 }
560
um_pci_virtio_probe(struct virtio_device * vdev)561 static int um_pci_virtio_probe(struct virtio_device *vdev)
562 {
563 struct um_pci_device *dev;
564 int i, free = -1;
565 int err = -ENOSPC;
566
567 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
568 if (!dev)
569 return -ENOMEM;
570
571 dev->vdev = vdev;
572 vdev->priv = dev;
573
574 mutex_lock(&um_pci_mtx);
575 for (i = 0; i < MAX_DEVICES; i++) {
576 if (um_pci_devices[i].dev)
577 continue;
578 free = i;
579 break;
580 }
581
582 if (free < 0)
583 goto error;
584
585 err = um_pci_init_vqs(dev);
586 if (err)
587 goto error;
588
589 dev->irq = irq_alloc_desc(numa_node_id());
590 if (dev->irq < 0) {
591 err = dev->irq;
592 goto err_reset;
593 }
594 um_pci_devices[free].dev = dev;
595 vdev->priv = dev;
596
597 mutex_unlock(&um_pci_mtx);
598
599 device_set_wakeup_enable(&vdev->dev, true);
600
601 /*
602 * In order to do suspend-resume properly, don't allow VQs
603 * to be suspended.
604 */
605 virtio_uml_set_no_vq_suspend(vdev, true);
606
607 um_pci_rescan();
608 return 0;
609 err_reset:
610 virtio_reset_device(vdev);
611 vdev->config->del_vqs(vdev);
612 error:
613 mutex_unlock(&um_pci_mtx);
614 kfree(dev);
615 return err;
616 }
617
um_pci_virtio_remove(struct virtio_device * vdev)618 static void um_pci_virtio_remove(struct virtio_device *vdev)
619 {
620 struct um_pci_device *dev = vdev->priv;
621 int i;
622
623 /* Stop all virtqueues */
624 virtio_reset_device(vdev);
625 vdev->config->del_vqs(vdev);
626
627 device_set_wakeup_enable(&vdev->dev, false);
628
629 mutex_lock(&um_pci_mtx);
630 for (i = 0; i < MAX_DEVICES; i++) {
631 if (um_pci_devices[i].dev != dev)
632 continue;
633 um_pci_devices[i].dev = NULL;
634 irq_free_desc(dev->irq);
635 }
636 mutex_unlock(&um_pci_mtx);
637
638 um_pci_rescan();
639
640 kfree(dev);
641 }
642
643 static struct virtio_device_id id_table[] = {
644 { CONFIG_UML_PCI_OVER_VIRTIO_DEVICE_ID, VIRTIO_DEV_ANY_ID },
645 { 0 },
646 };
647 MODULE_DEVICE_TABLE(virtio, id_table);
648
649 static struct virtio_driver um_pci_virtio_driver = {
650 .driver.name = "virtio-pci",
651 .driver.owner = THIS_MODULE,
652 .id_table = id_table,
653 .probe = um_pci_virtio_probe,
654 .remove = um_pci_virtio_remove,
655 };
656
657 static struct resource virt_cfgspace_resource = {
658 .name = "PCI config space",
659 .start = 0xf0000000 - MAX_DEVICES * CFG_SPACE_SIZE,
660 .end = 0xf0000000 - 1,
661 .flags = IORESOURCE_MEM,
662 };
663
um_pci_map_cfgspace(unsigned long offset,size_t size,const struct logic_iomem_ops ** ops,void ** priv)664 static long um_pci_map_cfgspace(unsigned long offset, size_t size,
665 const struct logic_iomem_ops **ops,
666 void **priv)
667 {
668 if (WARN_ON(size > CFG_SPACE_SIZE || offset % CFG_SPACE_SIZE))
669 return -EINVAL;
670
671 if (offset / CFG_SPACE_SIZE < MAX_DEVICES) {
672 *ops = &um_pci_device_cfgspace_ops;
673 *priv = &um_pci_devices[offset / CFG_SPACE_SIZE];
674 return 0;
675 }
676
677 WARN(1, "cannot map offset 0x%lx/0x%zx\n", offset, size);
678 return -ENOENT;
679 }
680
681 static const struct logic_iomem_region_ops um_pci_cfgspace_ops = {
682 .map = um_pci_map_cfgspace,
683 };
684
685 static struct resource virt_iomem_resource = {
686 .name = "PCI iomem",
687 .start = 0xf0000000,
688 .end = 0xffffffff,
689 .flags = IORESOURCE_MEM,
690 };
691
692 struct um_pci_map_iomem_data {
693 unsigned long offset;
694 size_t size;
695 const struct logic_iomem_ops **ops;
696 void **priv;
697 long ret;
698 };
699
um_pci_map_iomem_walk(struct pci_dev * pdev,void * _data)700 static int um_pci_map_iomem_walk(struct pci_dev *pdev, void *_data)
701 {
702 struct um_pci_map_iomem_data *data = _data;
703 struct um_pci_device_reg *reg = &um_pci_devices[pdev->devfn / 8];
704 struct um_pci_device *dev;
705 int i;
706
707 if (!reg->dev)
708 return 0;
709
710 for (i = 0; i < ARRAY_SIZE(dev->resptr); i++) {
711 struct resource *r = &pdev->resource[i];
712
713 if ((r->flags & IORESOURCE_TYPE_BITS) != IORESOURCE_MEM)
714 continue;
715
716 /*
717 * must be the whole or part of the resource,
718 * not allowed to only overlap
719 */
720 if (data->offset < r->start || data->offset > r->end)
721 continue;
722 if (data->offset + data->size - 1 > r->end)
723 continue;
724
725 dev = reg->dev;
726 *data->ops = &um_pci_device_bar_ops;
727 dev->resptr[i] = i;
728 *data->priv = &dev->resptr[i];
729 data->ret = data->offset - r->start;
730
731 /* no need to continue */
732 return 1;
733 }
734
735 return 0;
736 }
737
um_pci_map_iomem(unsigned long offset,size_t size,const struct logic_iomem_ops ** ops,void ** priv)738 static long um_pci_map_iomem(unsigned long offset, size_t size,
739 const struct logic_iomem_ops **ops,
740 void **priv)
741 {
742 struct um_pci_map_iomem_data data = {
743 /* we want the full address here */
744 .offset = offset + virt_iomem_resource.start,
745 .size = size,
746 .ops = ops,
747 .priv = priv,
748 .ret = -ENOENT,
749 };
750
751 pci_walk_bus(bridge->bus, um_pci_map_iomem_walk, &data);
752 return data.ret;
753 }
754
755 static const struct logic_iomem_region_ops um_pci_iomem_ops = {
756 .map = um_pci_map_iomem,
757 };
758
um_pci_compose_msi_msg(struct irq_data * data,struct msi_msg * msg)759 static void um_pci_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
760 {
761 /*
762 * This is a very low address and not actually valid 'physical' memory
763 * in UML, so we can simply map MSI(-X) vectors to there, it cannot be
764 * legitimately written to by the device in any other way.
765 * We use the (virtual) IRQ number here as the message to simplify the
766 * code that receives the message, where for now we simply trust the
767 * device to send the correct message.
768 */
769 msg->address_hi = 0;
770 msg->address_lo = 0xa0000;
771 msg->data = data->irq;
772 }
773
774 static struct irq_chip um_pci_msi_bottom_irq_chip = {
775 .name = "UM virtio MSI",
776 .irq_compose_msi_msg = um_pci_compose_msi_msg,
777 };
778
um_pci_inner_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * args)779 static int um_pci_inner_domain_alloc(struct irq_domain *domain,
780 unsigned int virq, unsigned int nr_irqs,
781 void *args)
782 {
783 unsigned long bit;
784
785 WARN_ON(nr_irqs != 1);
786
787 mutex_lock(&um_pci_mtx);
788 bit = find_first_zero_bit(um_pci_msi_used, MAX_MSI_VECTORS);
789 if (bit >= MAX_MSI_VECTORS) {
790 mutex_unlock(&um_pci_mtx);
791 return -ENOSPC;
792 }
793
794 set_bit(bit, um_pci_msi_used);
795 mutex_unlock(&um_pci_mtx);
796
797 irq_domain_set_info(domain, virq, bit, &um_pci_msi_bottom_irq_chip,
798 domain->host_data, handle_simple_irq,
799 NULL, NULL);
800
801 return 0;
802 }
803
um_pci_inner_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)804 static void um_pci_inner_domain_free(struct irq_domain *domain,
805 unsigned int virq, unsigned int nr_irqs)
806 {
807 struct irq_data *d = irq_domain_get_irq_data(domain, virq);
808
809 mutex_lock(&um_pci_mtx);
810
811 if (!test_bit(d->hwirq, um_pci_msi_used))
812 pr_err("trying to free unused MSI#%lu\n", d->hwirq);
813 else
814 __clear_bit(d->hwirq, um_pci_msi_used);
815
816 mutex_unlock(&um_pci_mtx);
817 }
818
819 static const struct irq_domain_ops um_pci_inner_domain_ops = {
820 .alloc = um_pci_inner_domain_alloc,
821 .free = um_pci_inner_domain_free,
822 };
823
824 static struct irq_chip um_pci_msi_irq_chip = {
825 .name = "UM virtio PCIe MSI",
826 .irq_mask = pci_msi_mask_irq,
827 .irq_unmask = pci_msi_unmask_irq,
828 };
829
830 static struct msi_domain_info um_pci_msi_domain_info = {
831 .flags = MSI_FLAG_USE_DEF_DOM_OPS |
832 MSI_FLAG_USE_DEF_CHIP_OPS |
833 MSI_FLAG_PCI_MSIX,
834 .chip = &um_pci_msi_irq_chip,
835 };
836
837 static struct resource busn_resource = {
838 .name = "PCI busn",
839 .start = 0,
840 .end = 0,
841 .flags = IORESOURCE_BUS,
842 };
843
um_pci_map_irq(const struct pci_dev * pdev,u8 slot,u8 pin)844 static int um_pci_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
845 {
846 struct um_pci_device_reg *reg = &um_pci_devices[pdev->devfn / 8];
847
848 if (WARN_ON(!reg->dev))
849 return -EINVAL;
850
851 /* Yes, we map all pins to the same IRQ ... doesn't matter for now. */
852 return reg->dev->irq;
853 }
854
pci_root_bus_fwnode(struct pci_bus * bus)855 void *pci_root_bus_fwnode(struct pci_bus *bus)
856 {
857 return um_pci_fwnode;
858 }
859
um_pci_init(void)860 static int um_pci_init(void)
861 {
862 int err, i;
863
864 WARN_ON(logic_iomem_add_region(&virt_cfgspace_resource,
865 &um_pci_cfgspace_ops));
866 WARN_ON(logic_iomem_add_region(&virt_iomem_resource,
867 &um_pci_iomem_ops));
868
869 if (WARN(CONFIG_UML_PCI_OVER_VIRTIO_DEVICE_ID < 0,
870 "No virtio device ID configured for PCI - no PCI support\n"))
871 return 0;
872
873 um_pci_msg_bufs = alloc_percpu(struct um_pci_message_buffer);
874 if (!um_pci_msg_bufs)
875 return -ENOMEM;
876
877 bridge = pci_alloc_host_bridge(0);
878 if (!bridge) {
879 err = -ENOMEM;
880 goto free;
881 }
882
883 um_pci_fwnode = irq_domain_alloc_named_fwnode("um-pci");
884 if (!um_pci_fwnode) {
885 err = -ENOMEM;
886 goto free;
887 }
888
889 um_pci_inner_domain = __irq_domain_add(um_pci_fwnode, MAX_MSI_VECTORS,
890 MAX_MSI_VECTORS, 0,
891 &um_pci_inner_domain_ops, NULL);
892 if (!um_pci_inner_domain) {
893 err = -ENOMEM;
894 goto free;
895 }
896
897 um_pci_msi_domain = pci_msi_create_irq_domain(um_pci_fwnode,
898 &um_pci_msi_domain_info,
899 um_pci_inner_domain);
900 if (!um_pci_msi_domain) {
901 err = -ENOMEM;
902 goto free;
903 }
904
905 pci_add_resource(&bridge->windows, &virt_iomem_resource);
906 pci_add_resource(&bridge->windows, &busn_resource);
907 bridge->ops = &um_pci_ops;
908 bridge->map_irq = um_pci_map_irq;
909
910 for (i = 0; i < MAX_DEVICES; i++) {
911 resource_size_t start;
912
913 start = virt_cfgspace_resource.start + i * CFG_SPACE_SIZE;
914 um_pci_devices[i].iomem = ioremap(start, CFG_SPACE_SIZE);
915 if (WARN(!um_pci_devices[i].iomem, "failed to map %d\n", i)) {
916 err = -ENOMEM;
917 goto free;
918 }
919 }
920
921 err = pci_host_probe(bridge);
922 if (err)
923 goto free;
924
925 err = register_virtio_driver(&um_pci_virtio_driver);
926 if (err)
927 goto free;
928 return 0;
929 free:
930 if (um_pci_inner_domain)
931 irq_domain_remove(um_pci_inner_domain);
932 if (um_pci_fwnode)
933 irq_domain_free_fwnode(um_pci_fwnode);
934 if (bridge) {
935 pci_free_resource_list(&bridge->windows);
936 pci_free_host_bridge(bridge);
937 }
938 free_percpu(um_pci_msg_bufs);
939 return err;
940 }
941 module_init(um_pci_init);
942
um_pci_exit(void)943 static void um_pci_exit(void)
944 {
945 unregister_virtio_driver(&um_pci_virtio_driver);
946 irq_domain_remove(um_pci_msi_domain);
947 irq_domain_remove(um_pci_inner_domain);
948 pci_free_resource_list(&bridge->windows);
949 pci_free_host_bridge(bridge);
950 free_percpu(um_pci_msg_bufs);
951 }
952 module_exit(um_pci_exit);
953