1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
4  * Author: Joerg Roedel <jroedel@suse.de>
5  *         Leo Duran <leo.duran@amd.com>
6  */
7 
8 #define pr_fmt(fmt)     "AMD-Vi: " fmt
9 #define dev_fmt(fmt)    pr_fmt(fmt)
10 
11 #include <linux/pci.h>
12 #include <linux/acpi.h>
13 #include <linux/list.h>
14 #include <linux/bitmap.h>
15 #include <linux/slab.h>
16 #include <linux/syscore_ops.h>
17 #include <linux/interrupt.h>
18 #include <linux/msi.h>
19 #include <linux/irq.h>
20 #include <linux/amd-iommu.h>
21 #include <linux/export.h>
22 #include <linux/kmemleak.h>
23 #include <linux/cc_platform.h>
24 #include <linux/iopoll.h>
25 #include <asm/pci-direct.h>
26 #include <asm/iommu.h>
27 #include <asm/apic.h>
28 #include <asm/gart.h>
29 #include <asm/x86_init.h>
30 #include <asm/io_apic.h>
31 #include <asm/irq_remapping.h>
32 #include <asm/set_memory.h>
33 
34 #include <linux/crash_dump.h>
35 
36 #include "amd_iommu.h"
37 #include "../irq_remapping.h"
38 
39 /*
40  * definitions for the ACPI scanning code
41  */
42 #define IVRS_HEADER_LENGTH 48
43 
44 #define ACPI_IVHD_TYPE_MAX_SUPPORTED	0x40
45 #define ACPI_IVMD_TYPE_ALL              0x20
46 #define ACPI_IVMD_TYPE                  0x21
47 #define ACPI_IVMD_TYPE_RANGE            0x22
48 
49 #define IVHD_DEV_ALL                    0x01
50 #define IVHD_DEV_SELECT                 0x02
51 #define IVHD_DEV_SELECT_RANGE_START     0x03
52 #define IVHD_DEV_RANGE_END              0x04
53 #define IVHD_DEV_ALIAS                  0x42
54 #define IVHD_DEV_ALIAS_RANGE            0x43
55 #define IVHD_DEV_EXT_SELECT             0x46
56 #define IVHD_DEV_EXT_SELECT_RANGE       0x47
57 #define IVHD_DEV_SPECIAL		0x48
58 #define IVHD_DEV_ACPI_HID		0xf0
59 
60 #define UID_NOT_PRESENT                 0
61 #define UID_IS_INTEGER                  1
62 #define UID_IS_CHARACTER                2
63 
64 #define IVHD_SPECIAL_IOAPIC		1
65 #define IVHD_SPECIAL_HPET		2
66 
67 #define IVHD_FLAG_HT_TUN_EN_MASK        0x01
68 #define IVHD_FLAG_PASSPW_EN_MASK        0x02
69 #define IVHD_FLAG_RESPASSPW_EN_MASK     0x04
70 #define IVHD_FLAG_ISOC_EN_MASK          0x08
71 
72 #define IVMD_FLAG_EXCL_RANGE            0x08
73 #define IVMD_FLAG_IW                    0x04
74 #define IVMD_FLAG_IR                    0x02
75 #define IVMD_FLAG_UNITY_MAP             0x01
76 
77 #define ACPI_DEVFLAG_INITPASS           0x01
78 #define ACPI_DEVFLAG_EXTINT             0x02
79 #define ACPI_DEVFLAG_NMI                0x04
80 #define ACPI_DEVFLAG_SYSMGT1            0x10
81 #define ACPI_DEVFLAG_SYSMGT2            0x20
82 #define ACPI_DEVFLAG_LINT0              0x40
83 #define ACPI_DEVFLAG_LINT1              0x80
84 #define ACPI_DEVFLAG_ATSDIS             0x10000000
85 
86 #define LOOP_TIMEOUT	2000000
87 
88 #define IVRS_GET_SBDF_ID(seg, bus, dev, fn)	(((seg & 0xffff) << 16) | ((bus & 0xff) << 8) \
89 						 | ((dev & 0x1f) << 3) | (fn & 0x7))
90 
91 /*
92  * ACPI table definitions
93  *
94  * These data structures are laid over the table to parse the important values
95  * out of it.
96  */
97 
98 /*
99  * structure describing one IOMMU in the ACPI table. Typically followed by one
100  * or more ivhd_entrys.
101  */
102 struct ivhd_header {
103 	u8 type;
104 	u8 flags;
105 	u16 length;
106 	u16 devid;
107 	u16 cap_ptr;
108 	u64 mmio_phys;
109 	u16 pci_seg;
110 	u16 info;
111 	u32 efr_attr;
112 
113 	/* Following only valid on IVHD type 11h and 40h */
114 	u64 efr_reg; /* Exact copy of MMIO_EXT_FEATURES */
115 	u64 efr_reg2;
116 } __attribute__((packed));
117 
118 /*
119  * A device entry describing which devices a specific IOMMU translates and
120  * which requestor ids they use.
121  */
122 struct ivhd_entry {
123 	u8 type;
124 	u16 devid;
125 	u8 flags;
126 	struct_group(ext_hid,
127 		u32 ext;
128 		u32 hidh;
129 	);
130 	u64 cid;
131 	u8 uidf;
132 	u8 uidl;
133 	u8 uid;
134 } __attribute__((packed));
135 
136 /*
137  * An AMD IOMMU memory definition structure. It defines things like exclusion
138  * ranges for devices and regions that should be unity mapped.
139  */
140 struct ivmd_header {
141 	u8 type;
142 	u8 flags;
143 	u16 length;
144 	u16 devid;
145 	u16 aux;
146 	u16 pci_seg;
147 	u8  resv[6];
148 	u64 range_start;
149 	u64 range_length;
150 } __attribute__((packed));
151 
152 bool amd_iommu_dump;
153 bool amd_iommu_irq_remap __read_mostly;
154 
155 enum io_pgtable_fmt amd_iommu_pgtable = AMD_IOMMU_V1;
156 /* Guest page table level */
157 int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL;
158 
159 int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
160 static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE;
161 
162 static bool amd_iommu_detected;
163 static bool amd_iommu_disabled __initdata;
164 static bool amd_iommu_force_enable __initdata;
165 static bool amd_iommu_irtcachedis;
166 static int amd_iommu_target_ivhd_type;
167 
168 /* Global EFR and EFR2 registers */
169 u64 amd_iommu_efr;
170 u64 amd_iommu_efr2;
171 
172 /* SNP is enabled on the system? */
173 bool amd_iommu_snp_en;
174 EXPORT_SYMBOL(amd_iommu_snp_en);
175 
176 LIST_HEAD(amd_iommu_pci_seg_list);	/* list of all PCI segments */
177 LIST_HEAD(amd_iommu_list);		/* list of all AMD IOMMUs in the
178 					   system */
179 
180 /* Array to assign indices to IOMMUs*/
181 struct amd_iommu *amd_iommus[MAX_IOMMUS];
182 
183 /* Number of IOMMUs present in the system */
184 static int amd_iommus_present;
185 
186 /* IOMMUs have a non-present cache? */
187 bool amd_iommu_np_cache __read_mostly;
188 bool amd_iommu_iotlb_sup __read_mostly = true;
189 
190 u32 amd_iommu_max_pasid __read_mostly = ~0;
191 
192 bool amd_iommu_v2_present __read_mostly;
193 static bool amd_iommu_pc_present __read_mostly;
194 bool amdr_ivrs_remap_support __read_mostly;
195 
196 bool amd_iommu_force_isolation __read_mostly;
197 
198 /*
199  * AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap
200  * to know which ones are already in use.
201  */
202 unsigned long *amd_iommu_pd_alloc_bitmap;
203 
204 enum iommu_init_state {
205 	IOMMU_START_STATE,
206 	IOMMU_IVRS_DETECTED,
207 	IOMMU_ACPI_FINISHED,
208 	IOMMU_ENABLED,
209 	IOMMU_PCI_INIT,
210 	IOMMU_INTERRUPTS_EN,
211 	IOMMU_INITIALIZED,
212 	IOMMU_NOT_FOUND,
213 	IOMMU_INIT_ERROR,
214 	IOMMU_CMDLINE_DISABLED,
215 };
216 
217 /* Early ioapic and hpet maps from kernel command line */
218 #define EARLY_MAP_SIZE		4
219 static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE];
220 static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE];
221 static struct acpihid_map_entry __initdata early_acpihid_map[EARLY_MAP_SIZE];
222 
223 static int __initdata early_ioapic_map_size;
224 static int __initdata early_hpet_map_size;
225 static int __initdata early_acpihid_map_size;
226 
227 static bool __initdata cmdline_maps;
228 
229 static enum iommu_init_state init_state = IOMMU_START_STATE;
230 
231 static int amd_iommu_enable_interrupts(void);
232 static int __init iommu_go_to_state(enum iommu_init_state state);
233 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg);
234 
235 static bool amd_iommu_pre_enabled = true;
236 
237 static u32 amd_iommu_ivinfo __initdata;
238 
translation_pre_enabled(struct amd_iommu * iommu)239 bool translation_pre_enabled(struct amd_iommu *iommu)
240 {
241 	return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED);
242 }
243 
clear_translation_pre_enabled(struct amd_iommu * iommu)244 static void clear_translation_pre_enabled(struct amd_iommu *iommu)
245 {
246 	iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
247 }
248 
init_translation_status(struct amd_iommu * iommu)249 static void init_translation_status(struct amd_iommu *iommu)
250 {
251 	u64 ctrl;
252 
253 	ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
254 	if (ctrl & (1<<CONTROL_IOMMU_EN))
255 		iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED;
256 }
257 
tbl_size(int entry_size,int last_bdf)258 static inline unsigned long tbl_size(int entry_size, int last_bdf)
259 {
260 	unsigned shift = PAGE_SHIFT +
261 			 get_order((last_bdf + 1) * entry_size);
262 
263 	return 1UL << shift;
264 }
265 
amd_iommu_get_num_iommus(void)266 int amd_iommu_get_num_iommus(void)
267 {
268 	return amd_iommus_present;
269 }
270 
271 /*
272  * Iterate through all the IOMMUs to get common EFR
273  * masks among all IOMMUs and warn if found inconsistency.
274  */
get_global_efr(void)275 static void get_global_efr(void)
276 {
277 	struct amd_iommu *iommu;
278 
279 	for_each_iommu(iommu) {
280 		u64 tmp = iommu->features;
281 		u64 tmp2 = iommu->features2;
282 
283 		if (list_is_first(&iommu->list, &amd_iommu_list)) {
284 			amd_iommu_efr = tmp;
285 			amd_iommu_efr2 = tmp2;
286 			continue;
287 		}
288 
289 		if (amd_iommu_efr == tmp &&
290 		    amd_iommu_efr2 == tmp2)
291 			continue;
292 
293 		pr_err(FW_BUG
294 		       "Found inconsistent EFR/EFR2 %#llx,%#llx (global %#llx,%#llx) on iommu%d (%04x:%02x:%02x.%01x).\n",
295 		       tmp, tmp2, amd_iommu_efr, amd_iommu_efr2,
296 		       iommu->index, iommu->pci_seg->id,
297 		       PCI_BUS_NUM(iommu->devid), PCI_SLOT(iommu->devid),
298 		       PCI_FUNC(iommu->devid));
299 
300 		amd_iommu_efr &= tmp;
301 		amd_iommu_efr2 &= tmp2;
302 	}
303 
304 	pr_info("Using global IVHD EFR:%#llx, EFR2:%#llx\n", amd_iommu_efr, amd_iommu_efr2);
305 }
306 
check_feature_on_all_iommus(u64 mask)307 static bool check_feature_on_all_iommus(u64 mask)
308 {
309 	return !!(amd_iommu_efr & mask);
310 }
311 
check_feature_gpt_level(void)312 static inline int check_feature_gpt_level(void)
313 {
314 	return ((amd_iommu_efr >> FEATURE_GATS_SHIFT) & FEATURE_GATS_MASK);
315 }
316 
317 /*
318  * For IVHD type 0x11/0x40, EFR is also available via IVHD.
319  * Default to IVHD EFR since it is available sooner
320  * (i.e. before PCI init).
321  */
early_iommu_features_init(struct amd_iommu * iommu,struct ivhd_header * h)322 static void __init early_iommu_features_init(struct amd_iommu *iommu,
323 					     struct ivhd_header *h)
324 {
325 	if (amd_iommu_ivinfo & IOMMU_IVINFO_EFRSUP) {
326 		iommu->features = h->efr_reg;
327 		iommu->features2 = h->efr_reg2;
328 	}
329 	if (amd_iommu_ivinfo & IOMMU_IVINFO_DMA_REMAP)
330 		amdr_ivrs_remap_support = true;
331 }
332 
333 /* Access to l1 and l2 indexed register spaces */
334 
iommu_read_l1(struct amd_iommu * iommu,u16 l1,u8 address)335 static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)
336 {
337 	u32 val;
338 
339 	pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
340 	pci_read_config_dword(iommu->dev, 0xfc, &val);
341 	return val;
342 }
343 
iommu_write_l1(struct amd_iommu * iommu,u16 l1,u8 address,u32 val)344 static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)
345 {
346 	pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));
347 	pci_write_config_dword(iommu->dev, 0xfc, val);
348 	pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
349 }
350 
iommu_read_l2(struct amd_iommu * iommu,u8 address)351 static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)
352 {
353 	u32 val;
354 
355 	pci_write_config_dword(iommu->dev, 0xf0, address);
356 	pci_read_config_dword(iommu->dev, 0xf4, &val);
357 	return val;
358 }
359 
iommu_write_l2(struct amd_iommu * iommu,u8 address,u32 val)360 static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)
361 {
362 	pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));
363 	pci_write_config_dword(iommu->dev, 0xf4, val);
364 }
365 
366 /****************************************************************************
367  *
368  * AMD IOMMU MMIO register space handling functions
369  *
370  * These functions are used to program the IOMMU device registers in
371  * MMIO space required for that driver.
372  *
373  ****************************************************************************/
374 
375 /*
376  * This function set the exclusion range in the IOMMU. DMA accesses to the
377  * exclusion range are passed through untranslated
378  */
iommu_set_exclusion_range(struct amd_iommu * iommu)379 static void iommu_set_exclusion_range(struct amd_iommu *iommu)
380 {
381 	u64 start = iommu->exclusion_start & PAGE_MASK;
382 	u64 limit = (start + iommu->exclusion_length - 1) & PAGE_MASK;
383 	u64 entry;
384 
385 	if (!iommu->exclusion_start)
386 		return;
387 
388 	entry = start | MMIO_EXCL_ENABLE_MASK;
389 	memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
390 			&entry, sizeof(entry));
391 
392 	entry = limit;
393 	memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
394 			&entry, sizeof(entry));
395 }
396 
iommu_set_cwwb_range(struct amd_iommu * iommu)397 static void iommu_set_cwwb_range(struct amd_iommu *iommu)
398 {
399 	u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem);
400 	u64 entry = start & PM_ADDR_MASK;
401 
402 	if (!check_feature_on_all_iommus(FEATURE_SNP))
403 		return;
404 
405 	/* Note:
406 	 * Re-purpose Exclusion base/limit registers for Completion wait
407 	 * write-back base/limit.
408 	 */
409 	memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
410 		    &entry, sizeof(entry));
411 
412 	/* Note:
413 	 * Default to 4 Kbytes, which can be specified by setting base
414 	 * address equal to the limit address.
415 	 */
416 	memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
417 		    &entry, sizeof(entry));
418 }
419 
420 /* Programs the physical address of the device table into the IOMMU hardware */
iommu_set_device_table(struct amd_iommu * iommu)421 static void iommu_set_device_table(struct amd_iommu *iommu)
422 {
423 	u64 entry;
424 	u32 dev_table_size = iommu->pci_seg->dev_table_size;
425 	void *dev_table = (void *)get_dev_table(iommu);
426 
427 	BUG_ON(iommu->mmio_base == NULL);
428 
429 	entry = iommu_virt_to_phys(dev_table);
430 	entry |= (dev_table_size >> 12) - 1;
431 	memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,
432 			&entry, sizeof(entry));
433 }
434 
435 /* Generic functions to enable/disable certain features of the IOMMU. */
iommu_feature_enable(struct amd_iommu * iommu,u8 bit)436 static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
437 {
438 	u64 ctrl;
439 
440 	ctrl = readq(iommu->mmio_base +  MMIO_CONTROL_OFFSET);
441 	ctrl |= (1ULL << bit);
442 	writeq(ctrl, iommu->mmio_base +  MMIO_CONTROL_OFFSET);
443 }
444 
iommu_feature_disable(struct amd_iommu * iommu,u8 bit)445 static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
446 {
447 	u64 ctrl;
448 
449 	ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
450 	ctrl &= ~(1ULL << bit);
451 	writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
452 }
453 
iommu_set_inv_tlb_timeout(struct amd_iommu * iommu,int timeout)454 static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout)
455 {
456 	u64 ctrl;
457 
458 	ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET);
459 	ctrl &= ~CTRL_INV_TO_MASK;
460 	ctrl |= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK;
461 	writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
462 }
463 
464 /* Function to enable the hardware */
iommu_enable(struct amd_iommu * iommu)465 static void iommu_enable(struct amd_iommu *iommu)
466 {
467 	iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
468 }
469 
iommu_disable(struct amd_iommu * iommu)470 static void iommu_disable(struct amd_iommu *iommu)
471 {
472 	if (!iommu->mmio_base)
473 		return;
474 
475 	/* Disable command buffer */
476 	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
477 
478 	/* Disable event logging and event interrupts */
479 	iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);
480 	iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
481 
482 	/* Disable IOMMU GA_LOG */
483 	iommu_feature_disable(iommu, CONTROL_GALOG_EN);
484 	iommu_feature_disable(iommu, CONTROL_GAINT_EN);
485 
486 	/* Disable IOMMU PPR logging */
487 	iommu_feature_disable(iommu, CONTROL_PPRLOG_EN);
488 	iommu_feature_disable(iommu, CONTROL_PPRINT_EN);
489 
490 	/* Disable IOMMU hardware itself */
491 	iommu_feature_disable(iommu, CONTROL_IOMMU_EN);
492 
493 	/* Clear IRTE cache disabling bit */
494 	iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);
495 }
496 
497 /*
498  * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in
499  * the system has one.
500  */
iommu_map_mmio_space(u64 address,u64 end)501 static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end)
502 {
503 	if (!request_mem_region(address, end, "amd_iommu")) {
504 		pr_err("Can not reserve memory region %llx-%llx for mmio\n",
505 			address, end);
506 		pr_err("This is a BIOS bug. Please contact your hardware vendor\n");
507 		return NULL;
508 	}
509 
510 	return (u8 __iomem *)ioremap(address, end);
511 }
512 
iommu_unmap_mmio_space(struct amd_iommu * iommu)513 static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)
514 {
515 	if (iommu->mmio_base)
516 		iounmap(iommu->mmio_base);
517 	release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end);
518 }
519 
get_ivhd_header_size(struct ivhd_header * h)520 static inline u32 get_ivhd_header_size(struct ivhd_header *h)
521 {
522 	u32 size = 0;
523 
524 	switch (h->type) {
525 	case 0x10:
526 		size = 24;
527 		break;
528 	case 0x11:
529 	case 0x40:
530 		size = 40;
531 		break;
532 	}
533 	return size;
534 }
535 
536 /****************************************************************************
537  *
538  * The functions below belong to the first pass of AMD IOMMU ACPI table
539  * parsing. In this pass we try to find out the highest device id this
540  * code has to handle. Upon this information the size of the shared data
541  * structures is determined later.
542  *
543  ****************************************************************************/
544 
545 /*
546  * This function calculates the length of a given IVHD entry
547  */
ivhd_entry_length(u8 * ivhd)548 static inline int ivhd_entry_length(u8 *ivhd)
549 {
550 	u32 type = ((struct ivhd_entry *)ivhd)->type;
551 
552 	if (type < 0x80) {
553 		return 0x04 << (*ivhd >> 6);
554 	} else if (type == IVHD_DEV_ACPI_HID) {
555 		/* For ACPI_HID, offset 21 is uid len */
556 		return *((u8 *)ivhd + 21) + 22;
557 	}
558 	return 0;
559 }
560 
561 /*
562  * After reading the highest device id from the IOMMU PCI capability header
563  * this function looks if there is a higher device id defined in the ACPI table
564  */
find_last_devid_from_ivhd(struct ivhd_header * h)565 static int __init find_last_devid_from_ivhd(struct ivhd_header *h)
566 {
567 	u8 *p = (void *)h, *end = (void *)h;
568 	struct ivhd_entry *dev;
569 	int last_devid = -EINVAL;
570 
571 	u32 ivhd_size = get_ivhd_header_size(h);
572 
573 	if (!ivhd_size) {
574 		pr_err("Unsupported IVHD type %#x\n", h->type);
575 		return -EINVAL;
576 	}
577 
578 	p += ivhd_size;
579 	end += h->length;
580 
581 	while (p < end) {
582 		dev = (struct ivhd_entry *)p;
583 		switch (dev->type) {
584 		case IVHD_DEV_ALL:
585 			/* Use maximum BDF value for DEV_ALL */
586 			return 0xffff;
587 		case IVHD_DEV_SELECT:
588 		case IVHD_DEV_RANGE_END:
589 		case IVHD_DEV_ALIAS:
590 		case IVHD_DEV_EXT_SELECT:
591 			/* all the above subfield types refer to device ids */
592 			if (dev->devid > last_devid)
593 				last_devid = dev->devid;
594 			break;
595 		default:
596 			break;
597 		}
598 		p += ivhd_entry_length(p);
599 	}
600 
601 	WARN_ON(p != end);
602 
603 	return last_devid;
604 }
605 
check_ivrs_checksum(struct acpi_table_header * table)606 static int __init check_ivrs_checksum(struct acpi_table_header *table)
607 {
608 	int i;
609 	u8 checksum = 0, *p = (u8 *)table;
610 
611 	for (i = 0; i < table->length; ++i)
612 		checksum += p[i];
613 	if (checksum != 0) {
614 		/* ACPI table corrupt */
615 		pr_err(FW_BUG "IVRS invalid checksum\n");
616 		return -ENODEV;
617 	}
618 
619 	return 0;
620 }
621 
622 /*
623  * Iterate over all IVHD entries in the ACPI table and find the highest device
624  * id which we need to handle. This is the first of three functions which parse
625  * the ACPI table. So we check the checksum here.
626  */
find_last_devid_acpi(struct acpi_table_header * table,u16 pci_seg)627 static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_seg)
628 {
629 	u8 *p = (u8 *)table, *end = (u8 *)table;
630 	struct ivhd_header *h;
631 	int last_devid, last_bdf = 0;
632 
633 	p += IVRS_HEADER_LENGTH;
634 
635 	end += table->length;
636 	while (p < end) {
637 		h = (struct ivhd_header *)p;
638 		if (h->pci_seg == pci_seg &&
639 		    h->type == amd_iommu_target_ivhd_type) {
640 			last_devid = find_last_devid_from_ivhd(h);
641 
642 			if (last_devid < 0)
643 				return -EINVAL;
644 			if (last_devid > last_bdf)
645 				last_bdf = last_devid;
646 		}
647 		p += h->length;
648 	}
649 	WARN_ON(p != end);
650 
651 	return last_bdf;
652 }
653 
654 /****************************************************************************
655  *
656  * The following functions belong to the code path which parses the ACPI table
657  * the second time. In this ACPI parsing iteration we allocate IOMMU specific
658  * data structures, initialize the per PCI segment device/alias/rlookup table
659  * and also basically initialize the hardware.
660  *
661  ****************************************************************************/
662 
663 /* Allocate per PCI segment device table */
alloc_dev_table(struct amd_iommu_pci_seg * pci_seg)664 static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg)
665 {
666 	pci_seg->dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO | GFP_DMA32,
667 						      get_order(pci_seg->dev_table_size));
668 	if (!pci_seg->dev_table)
669 		return -ENOMEM;
670 
671 	return 0;
672 }
673 
free_dev_table(struct amd_iommu_pci_seg * pci_seg)674 static inline void free_dev_table(struct amd_iommu_pci_seg *pci_seg)
675 {
676 	free_pages((unsigned long)pci_seg->dev_table,
677 		    get_order(pci_seg->dev_table_size));
678 	pci_seg->dev_table = NULL;
679 }
680 
681 /* Allocate per PCI segment IOMMU rlookup table. */
alloc_rlookup_table(struct amd_iommu_pci_seg * pci_seg)682 static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg)
683 {
684 	pci_seg->rlookup_table = (void *)__get_free_pages(
685 						GFP_KERNEL | __GFP_ZERO,
686 						get_order(pci_seg->rlookup_table_size));
687 	if (pci_seg->rlookup_table == NULL)
688 		return -ENOMEM;
689 
690 	return 0;
691 }
692 
free_rlookup_table(struct amd_iommu_pci_seg * pci_seg)693 static inline void free_rlookup_table(struct amd_iommu_pci_seg *pci_seg)
694 {
695 	free_pages((unsigned long)pci_seg->rlookup_table,
696 		   get_order(pci_seg->rlookup_table_size));
697 	pci_seg->rlookup_table = NULL;
698 }
699 
alloc_irq_lookup_table(struct amd_iommu_pci_seg * pci_seg)700 static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg)
701 {
702 	pci_seg->irq_lookup_table = (void *)__get_free_pages(
703 					     GFP_KERNEL | __GFP_ZERO,
704 					     get_order(pci_seg->rlookup_table_size));
705 	kmemleak_alloc(pci_seg->irq_lookup_table,
706 		       pci_seg->rlookup_table_size, 1, GFP_KERNEL);
707 	if (pci_seg->irq_lookup_table == NULL)
708 		return -ENOMEM;
709 
710 	return 0;
711 }
712 
free_irq_lookup_table(struct amd_iommu_pci_seg * pci_seg)713 static inline void free_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg)
714 {
715 	kmemleak_free(pci_seg->irq_lookup_table);
716 	free_pages((unsigned long)pci_seg->irq_lookup_table,
717 		   get_order(pci_seg->rlookup_table_size));
718 	pci_seg->irq_lookup_table = NULL;
719 }
720 
alloc_alias_table(struct amd_iommu_pci_seg * pci_seg)721 static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg)
722 {
723 	int i;
724 
725 	pci_seg->alias_table = (void *)__get_free_pages(GFP_KERNEL,
726 					get_order(pci_seg->alias_table_size));
727 	if (!pci_seg->alias_table)
728 		return -ENOMEM;
729 
730 	/*
731 	 * let all alias entries point to itself
732 	 */
733 	for (i = 0; i <= pci_seg->last_bdf; ++i)
734 		pci_seg->alias_table[i] = i;
735 
736 	return 0;
737 }
738 
free_alias_table(struct amd_iommu_pci_seg * pci_seg)739 static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg)
740 {
741 	free_pages((unsigned long)pci_seg->alias_table,
742 		   get_order(pci_seg->alias_table_size));
743 	pci_seg->alias_table = NULL;
744 }
745 
746 /*
747  * Allocates the command buffer. This buffer is per AMD IOMMU. We can
748  * write commands to that buffer later and the IOMMU will execute them
749  * asynchronously
750  */
alloc_command_buffer(struct amd_iommu * iommu)751 static int __init alloc_command_buffer(struct amd_iommu *iommu)
752 {
753 	iommu->cmd_buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
754 						  get_order(CMD_BUFFER_SIZE));
755 
756 	return iommu->cmd_buf ? 0 : -ENOMEM;
757 }
758 
759 /*
760  * Interrupt handler has processed all pending events and adjusted head
761  * and tail pointer. Reset overflow mask and restart logging again.
762  */
amd_iommu_restart_log(struct amd_iommu * iommu,const char * evt_type,u8 cntrl_intr,u8 cntrl_log,u32 status_run_mask,u32 status_overflow_mask)763 static void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type,
764 				  u8 cntrl_intr, u8 cntrl_log,
765 				  u32 status_run_mask, u32 status_overflow_mask)
766 {
767 	u32 status;
768 
769 	status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
770 	if (status & status_run_mask)
771 		return;
772 
773 	pr_info_ratelimited("IOMMU %s log restarting\n", evt_type);
774 
775 	iommu_feature_disable(iommu, cntrl_log);
776 	iommu_feature_disable(iommu, cntrl_intr);
777 
778 	writel(status_overflow_mask, iommu->mmio_base + MMIO_STATUS_OFFSET);
779 
780 	iommu_feature_enable(iommu, cntrl_intr);
781 	iommu_feature_enable(iommu, cntrl_log);
782 }
783 
784 /*
785  * This function restarts event logging in case the IOMMU experienced
786  * an event log buffer overflow.
787  */
amd_iommu_restart_event_logging(struct amd_iommu * iommu)788 void amd_iommu_restart_event_logging(struct amd_iommu *iommu)
789 {
790 	amd_iommu_restart_log(iommu, "Event", CONTROL_EVT_INT_EN,
791 			      CONTROL_EVT_LOG_EN, MMIO_STATUS_EVT_RUN_MASK,
792 			      MMIO_STATUS_EVT_OVERFLOW_MASK);
793 }
794 
795 /*
796  * This function restarts event logging in case the IOMMU experienced
797  * GA log overflow.
798  */
amd_iommu_restart_ga_log(struct amd_iommu * iommu)799 void amd_iommu_restart_ga_log(struct amd_iommu *iommu)
800 {
801 	amd_iommu_restart_log(iommu, "GA", CONTROL_GAINT_EN,
802 			      CONTROL_GALOG_EN, MMIO_STATUS_GALOG_RUN_MASK,
803 			      MMIO_STATUS_GALOG_OVERFLOW_MASK);
804 }
805 
806 /*
807  * This function restarts ppr logging in case the IOMMU experienced
808  * PPR log overflow.
809  */
amd_iommu_restart_ppr_log(struct amd_iommu * iommu)810 void amd_iommu_restart_ppr_log(struct amd_iommu *iommu)
811 {
812 	amd_iommu_restart_log(iommu, "PPR", CONTROL_PPRINT_EN,
813 			      CONTROL_PPRLOG_EN, MMIO_STATUS_PPR_RUN_MASK,
814 			      MMIO_STATUS_PPR_OVERFLOW_MASK);
815 }
816 
817 /*
818  * This function resets the command buffer if the IOMMU stopped fetching
819  * commands from it.
820  */
amd_iommu_reset_cmd_buffer(struct amd_iommu * iommu)821 static void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
822 {
823 	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
824 
825 	writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
826 	writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
827 	iommu->cmd_buf_head = 0;
828 	iommu->cmd_buf_tail = 0;
829 
830 	iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
831 }
832 
833 /*
834  * This function writes the command buffer address to the hardware and
835  * enables it.
836  */
iommu_enable_command_buffer(struct amd_iommu * iommu)837 static void iommu_enable_command_buffer(struct amd_iommu *iommu)
838 {
839 	u64 entry;
840 
841 	BUG_ON(iommu->cmd_buf == NULL);
842 
843 	entry = iommu_virt_to_phys(iommu->cmd_buf);
844 	entry |= MMIO_CMD_SIZE_512;
845 
846 	memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
847 		    &entry, sizeof(entry));
848 
849 	amd_iommu_reset_cmd_buffer(iommu);
850 }
851 
852 /*
853  * This function disables the command buffer
854  */
iommu_disable_command_buffer(struct amd_iommu * iommu)855 static void iommu_disable_command_buffer(struct amd_iommu *iommu)
856 {
857 	iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
858 }
859 
free_command_buffer(struct amd_iommu * iommu)860 static void __init free_command_buffer(struct amd_iommu *iommu)
861 {
862 	free_pages((unsigned long)iommu->cmd_buf, get_order(CMD_BUFFER_SIZE));
863 }
864 
iommu_alloc_4k_pages(struct amd_iommu * iommu,gfp_t gfp,size_t size)865 static void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu,
866 					 gfp_t gfp, size_t size)
867 {
868 	int order = get_order(size);
869 	void *buf = (void *)__get_free_pages(gfp, order);
870 
871 	if (buf &&
872 	    check_feature_on_all_iommus(FEATURE_SNP) &&
873 	    set_memory_4k((unsigned long)buf, (1 << order))) {
874 		free_pages((unsigned long)buf, order);
875 		buf = NULL;
876 	}
877 
878 	return buf;
879 }
880 
881 /* allocates the memory where the IOMMU will log its events to */
alloc_event_buffer(struct amd_iommu * iommu)882 static int __init alloc_event_buffer(struct amd_iommu *iommu)
883 {
884 	iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO,
885 					      EVT_BUFFER_SIZE);
886 
887 	return iommu->evt_buf ? 0 : -ENOMEM;
888 }
889 
iommu_enable_event_buffer(struct amd_iommu * iommu)890 static void iommu_enable_event_buffer(struct amd_iommu *iommu)
891 {
892 	u64 entry;
893 
894 	BUG_ON(iommu->evt_buf == NULL);
895 
896 	entry = iommu_virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;
897 
898 	memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
899 		    &entry, sizeof(entry));
900 
901 	/* set head and tail to zero manually */
902 	writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
903 	writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
904 
905 	iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
906 }
907 
908 /*
909  * This function disables the event log buffer
910  */
iommu_disable_event_buffer(struct amd_iommu * iommu)911 static void iommu_disable_event_buffer(struct amd_iommu *iommu)
912 {
913 	iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
914 }
915 
free_event_buffer(struct amd_iommu * iommu)916 static void __init free_event_buffer(struct amd_iommu *iommu)
917 {
918 	free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
919 }
920 
921 /* allocates the memory where the IOMMU will log its events to */
alloc_ppr_log(struct amd_iommu * iommu)922 static int __init alloc_ppr_log(struct amd_iommu *iommu)
923 {
924 	iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO,
925 					      PPR_LOG_SIZE);
926 
927 	return iommu->ppr_log ? 0 : -ENOMEM;
928 }
929 
iommu_enable_ppr_log(struct amd_iommu * iommu)930 static void iommu_enable_ppr_log(struct amd_iommu *iommu)
931 {
932 	u64 entry;
933 
934 	if (iommu->ppr_log == NULL)
935 		return;
936 
937 	iommu_feature_enable(iommu, CONTROL_PPR_EN);
938 
939 	entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;
940 
941 	memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,
942 		    &entry, sizeof(entry));
943 
944 	/* set head and tail to zero manually */
945 	writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
946 	writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
947 
948 	iommu_feature_enable(iommu, CONTROL_PPRLOG_EN);
949 	iommu_feature_enable(iommu, CONTROL_PPRINT_EN);
950 }
951 
free_ppr_log(struct amd_iommu * iommu)952 static void __init free_ppr_log(struct amd_iommu *iommu)
953 {
954 	free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE));
955 }
956 
free_ga_log(struct amd_iommu * iommu)957 static void free_ga_log(struct amd_iommu *iommu)
958 {
959 #ifdef CONFIG_IRQ_REMAP
960 	free_pages((unsigned long)iommu->ga_log, get_order(GA_LOG_SIZE));
961 	free_pages((unsigned long)iommu->ga_log_tail, get_order(8));
962 #endif
963 }
964 
965 #ifdef CONFIG_IRQ_REMAP
iommu_ga_log_enable(struct amd_iommu * iommu)966 static int iommu_ga_log_enable(struct amd_iommu *iommu)
967 {
968 	u32 status, i;
969 	u64 entry;
970 
971 	if (!iommu->ga_log)
972 		return -EINVAL;
973 
974 	entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512;
975 	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET,
976 		    &entry, sizeof(entry));
977 	entry = (iommu_virt_to_phys(iommu->ga_log_tail) &
978 		 (BIT_ULL(52)-1)) & ~7ULL;
979 	memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET,
980 		    &entry, sizeof(entry));
981 	writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET);
982 	writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET);
983 
984 
985 	iommu_feature_enable(iommu, CONTROL_GAINT_EN);
986 	iommu_feature_enable(iommu, CONTROL_GALOG_EN);
987 
988 	for (i = 0; i < LOOP_TIMEOUT; ++i) {
989 		status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
990 		if (status & (MMIO_STATUS_GALOG_RUN_MASK))
991 			break;
992 		udelay(10);
993 	}
994 
995 	if (WARN_ON(i >= LOOP_TIMEOUT))
996 		return -EINVAL;
997 
998 	return 0;
999 }
1000 
iommu_init_ga_log(struct amd_iommu * iommu)1001 static int iommu_init_ga_log(struct amd_iommu *iommu)
1002 {
1003 	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
1004 		return 0;
1005 
1006 	iommu->ga_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1007 					get_order(GA_LOG_SIZE));
1008 	if (!iommu->ga_log)
1009 		goto err_out;
1010 
1011 	iommu->ga_log_tail = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1012 					get_order(8));
1013 	if (!iommu->ga_log_tail)
1014 		goto err_out;
1015 
1016 	return 0;
1017 err_out:
1018 	free_ga_log(iommu);
1019 	return -EINVAL;
1020 }
1021 #endif /* CONFIG_IRQ_REMAP */
1022 
alloc_cwwb_sem(struct amd_iommu * iommu)1023 static int __init alloc_cwwb_sem(struct amd_iommu *iommu)
1024 {
1025 	iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, 1);
1026 
1027 	return iommu->cmd_sem ? 0 : -ENOMEM;
1028 }
1029 
free_cwwb_sem(struct amd_iommu * iommu)1030 static void __init free_cwwb_sem(struct amd_iommu *iommu)
1031 {
1032 	if (iommu->cmd_sem)
1033 		free_page((unsigned long)iommu->cmd_sem);
1034 }
1035 
iommu_enable_xt(struct amd_iommu * iommu)1036 static void iommu_enable_xt(struct amd_iommu *iommu)
1037 {
1038 #ifdef CONFIG_IRQ_REMAP
1039 	/*
1040 	 * XT mode (32-bit APIC destination ID) requires
1041 	 * GA mode (128-bit IRTE support) as a prerequisite.
1042 	 */
1043 	if (AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir) &&
1044 	    amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
1045 		iommu_feature_enable(iommu, CONTROL_XT_EN);
1046 #endif /* CONFIG_IRQ_REMAP */
1047 }
1048 
iommu_enable_gt(struct amd_iommu * iommu)1049 static void iommu_enable_gt(struct amd_iommu *iommu)
1050 {
1051 	if (!iommu_feature(iommu, FEATURE_GT))
1052 		return;
1053 
1054 	iommu_feature_enable(iommu, CONTROL_GT_EN);
1055 }
1056 
1057 /* sets a specific bit in the device table entry. */
__set_dev_entry_bit(struct dev_table_entry * dev_table,u16 devid,u8 bit)1058 static void __set_dev_entry_bit(struct dev_table_entry *dev_table,
1059 				u16 devid, u8 bit)
1060 {
1061 	int i = (bit >> 6) & 0x03;
1062 	int _bit = bit & 0x3f;
1063 
1064 	dev_table[devid].data[i] |= (1UL << _bit);
1065 }
1066 
set_dev_entry_bit(struct amd_iommu * iommu,u16 devid,u8 bit)1067 static void set_dev_entry_bit(struct amd_iommu *iommu, u16 devid, u8 bit)
1068 {
1069 	struct dev_table_entry *dev_table = get_dev_table(iommu);
1070 
1071 	return __set_dev_entry_bit(dev_table, devid, bit);
1072 }
1073 
__get_dev_entry_bit(struct dev_table_entry * dev_table,u16 devid,u8 bit)1074 static int __get_dev_entry_bit(struct dev_table_entry *dev_table,
1075 			       u16 devid, u8 bit)
1076 {
1077 	int i = (bit >> 6) & 0x03;
1078 	int _bit = bit & 0x3f;
1079 
1080 	return (dev_table[devid].data[i] & (1UL << _bit)) >> _bit;
1081 }
1082 
get_dev_entry_bit(struct amd_iommu * iommu,u16 devid,u8 bit)1083 static int get_dev_entry_bit(struct amd_iommu *iommu, u16 devid, u8 bit)
1084 {
1085 	struct dev_table_entry *dev_table = get_dev_table(iommu);
1086 
1087 	return __get_dev_entry_bit(dev_table, devid, bit);
1088 }
1089 
__copy_device_table(struct amd_iommu * iommu)1090 static bool __copy_device_table(struct amd_iommu *iommu)
1091 {
1092 	u64 int_ctl, int_tab_len, entry = 0;
1093 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
1094 	struct dev_table_entry *old_devtb = NULL;
1095 	u32 lo, hi, devid, old_devtb_size;
1096 	phys_addr_t old_devtb_phys;
1097 	u16 dom_id, dte_v, irq_v;
1098 	gfp_t gfp_flag;
1099 	u64 tmp;
1100 
1101 	/* Each IOMMU use separate device table with the same size */
1102 	lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET);
1103 	hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4);
1104 	entry = (((u64) hi) << 32) + lo;
1105 
1106 	old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12;
1107 	if (old_devtb_size != pci_seg->dev_table_size) {
1108 		pr_err("The device table size of IOMMU:%d is not expected!\n",
1109 			iommu->index);
1110 		return false;
1111 	}
1112 
1113 	/*
1114 	 * When SME is enabled in the first kernel, the entry includes the
1115 	 * memory encryption mask(sme_me_mask), we must remove the memory
1116 	 * encryption mask to obtain the true physical address in kdump kernel.
1117 	 */
1118 	old_devtb_phys = __sme_clr(entry) & PAGE_MASK;
1119 
1120 	if (old_devtb_phys >= 0x100000000ULL) {
1121 		pr_err("The address of old device table is above 4G, not trustworthy!\n");
1122 		return false;
1123 	}
1124 	old_devtb = (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT) && is_kdump_kernel())
1125 		    ? (__force void *)ioremap_encrypted(old_devtb_phys,
1126 							pci_seg->dev_table_size)
1127 		    : memremap(old_devtb_phys, pci_seg->dev_table_size, MEMREMAP_WB);
1128 
1129 	if (!old_devtb)
1130 		return false;
1131 
1132 	gfp_flag = GFP_KERNEL | __GFP_ZERO | GFP_DMA32;
1133 	pci_seg->old_dev_tbl_cpy = (void *)__get_free_pages(gfp_flag,
1134 						    get_order(pci_seg->dev_table_size));
1135 	if (pci_seg->old_dev_tbl_cpy == NULL) {
1136 		pr_err("Failed to allocate memory for copying old device table!\n");
1137 		memunmap(old_devtb);
1138 		return false;
1139 	}
1140 
1141 	for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {
1142 		pci_seg->old_dev_tbl_cpy[devid] = old_devtb[devid];
1143 		dom_id = old_devtb[devid].data[1] & DEV_DOMID_MASK;
1144 		dte_v = old_devtb[devid].data[0] & DTE_FLAG_V;
1145 
1146 		if (dte_v && dom_id) {
1147 			pci_seg->old_dev_tbl_cpy[devid].data[0] = old_devtb[devid].data[0];
1148 			pci_seg->old_dev_tbl_cpy[devid].data[1] = old_devtb[devid].data[1];
1149 			__set_bit(dom_id, amd_iommu_pd_alloc_bitmap);
1150 			/* If gcr3 table existed, mask it out */
1151 			if (old_devtb[devid].data[0] & DTE_FLAG_GV) {
1152 				tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
1153 				tmp |= DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
1154 				pci_seg->old_dev_tbl_cpy[devid].data[1] &= ~tmp;
1155 				tmp = DTE_GCR3_VAL_A(~0ULL) << DTE_GCR3_SHIFT_A;
1156 				tmp |= DTE_FLAG_GV;
1157 				pci_seg->old_dev_tbl_cpy[devid].data[0] &= ~tmp;
1158 			}
1159 		}
1160 
1161 		irq_v = old_devtb[devid].data[2] & DTE_IRQ_REMAP_ENABLE;
1162 		int_ctl = old_devtb[devid].data[2] & DTE_IRQ_REMAP_INTCTL_MASK;
1163 		int_tab_len = old_devtb[devid].data[2] & DTE_INTTABLEN_MASK;
1164 		if (irq_v && (int_ctl || int_tab_len)) {
1165 			if ((int_ctl != DTE_IRQ_REMAP_INTCTL) ||
1166 			    (int_tab_len != DTE_INTTABLEN)) {
1167 				pr_err("Wrong old irq remapping flag: %#x\n", devid);
1168 				memunmap(old_devtb);
1169 				return false;
1170 			}
1171 
1172 			pci_seg->old_dev_tbl_cpy[devid].data[2] = old_devtb[devid].data[2];
1173 		}
1174 	}
1175 	memunmap(old_devtb);
1176 
1177 	return true;
1178 }
1179 
copy_device_table(void)1180 static bool copy_device_table(void)
1181 {
1182 	struct amd_iommu *iommu;
1183 	struct amd_iommu_pci_seg *pci_seg;
1184 
1185 	if (!amd_iommu_pre_enabled)
1186 		return false;
1187 
1188 	pr_warn("Translation is already enabled - trying to copy translation structures\n");
1189 
1190 	/*
1191 	 * All IOMMUs within PCI segment shares common device table.
1192 	 * Hence copy device table only once per PCI segment.
1193 	 */
1194 	for_each_pci_segment(pci_seg) {
1195 		for_each_iommu(iommu) {
1196 			if (pci_seg->id != iommu->pci_seg->id)
1197 				continue;
1198 			if (!__copy_device_table(iommu))
1199 				return false;
1200 			break;
1201 		}
1202 	}
1203 
1204 	return true;
1205 }
1206 
amd_iommu_apply_erratum_63(struct amd_iommu * iommu,u16 devid)1207 void amd_iommu_apply_erratum_63(struct amd_iommu *iommu, u16 devid)
1208 {
1209 	int sysmgt;
1210 
1211 	sysmgt = get_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT1) |
1212 		 (get_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT2) << 1);
1213 
1214 	if (sysmgt == 0x01)
1215 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_IW);
1216 }
1217 
1218 /*
1219  * This function takes the device specific flags read from the ACPI
1220  * table and sets up the device table entry with that information
1221  */
set_dev_entry_from_acpi(struct amd_iommu * iommu,u16 devid,u32 flags,u32 ext_flags)1222 static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,
1223 					   u16 devid, u32 flags, u32 ext_flags)
1224 {
1225 	if (flags & ACPI_DEVFLAG_INITPASS)
1226 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_INIT_PASS);
1227 	if (flags & ACPI_DEVFLAG_EXTINT)
1228 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_EINT_PASS);
1229 	if (flags & ACPI_DEVFLAG_NMI)
1230 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_NMI_PASS);
1231 	if (flags & ACPI_DEVFLAG_SYSMGT1)
1232 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT1);
1233 	if (flags & ACPI_DEVFLAG_SYSMGT2)
1234 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT2);
1235 	if (flags & ACPI_DEVFLAG_LINT0)
1236 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_LINT0_PASS);
1237 	if (flags & ACPI_DEVFLAG_LINT1)
1238 		set_dev_entry_bit(iommu, devid, DEV_ENTRY_LINT1_PASS);
1239 
1240 	amd_iommu_apply_erratum_63(iommu, devid);
1241 
1242 	amd_iommu_set_rlookup_table(iommu, devid);
1243 }
1244 
add_special_device(u8 type,u8 id,u32 * devid,bool cmd_line)1245 int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line)
1246 {
1247 	struct devid_map *entry;
1248 	struct list_head *list;
1249 
1250 	if (type == IVHD_SPECIAL_IOAPIC)
1251 		list = &ioapic_map;
1252 	else if (type == IVHD_SPECIAL_HPET)
1253 		list = &hpet_map;
1254 	else
1255 		return -EINVAL;
1256 
1257 	list_for_each_entry(entry, list, list) {
1258 		if (!(entry->id == id && entry->cmd_line))
1259 			continue;
1260 
1261 		pr_info("Command-line override present for %s id %d - ignoring\n",
1262 			type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id);
1263 
1264 		*devid = entry->devid;
1265 
1266 		return 0;
1267 	}
1268 
1269 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1270 	if (!entry)
1271 		return -ENOMEM;
1272 
1273 	entry->id	= id;
1274 	entry->devid	= *devid;
1275 	entry->cmd_line	= cmd_line;
1276 
1277 	list_add_tail(&entry->list, list);
1278 
1279 	return 0;
1280 }
1281 
add_acpi_hid_device(u8 * hid,u8 * uid,u32 * devid,bool cmd_line)1282 static int __init add_acpi_hid_device(u8 *hid, u8 *uid, u32 *devid,
1283 				      bool cmd_line)
1284 {
1285 	struct acpihid_map_entry *entry;
1286 	struct list_head *list = &acpihid_map;
1287 
1288 	list_for_each_entry(entry, list, list) {
1289 		if (strcmp(entry->hid, hid) ||
1290 		    (*uid && *entry->uid && strcmp(entry->uid, uid)) ||
1291 		    !entry->cmd_line)
1292 			continue;
1293 
1294 		pr_info("Command-line override for hid:%s uid:%s\n",
1295 			hid, uid);
1296 		*devid = entry->devid;
1297 		return 0;
1298 	}
1299 
1300 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1301 	if (!entry)
1302 		return -ENOMEM;
1303 
1304 	memcpy(entry->uid, uid, strlen(uid));
1305 	memcpy(entry->hid, hid, strlen(hid));
1306 	entry->devid = *devid;
1307 	entry->cmd_line	= cmd_line;
1308 	entry->root_devid = (entry->devid & (~0x7));
1309 
1310 	pr_info("%s, add hid:%s, uid:%s, rdevid:%d\n",
1311 		entry->cmd_line ? "cmd" : "ivrs",
1312 		entry->hid, entry->uid, entry->root_devid);
1313 
1314 	list_add_tail(&entry->list, list);
1315 	return 0;
1316 }
1317 
add_early_maps(void)1318 static int __init add_early_maps(void)
1319 {
1320 	int i, ret;
1321 
1322 	for (i = 0; i < early_ioapic_map_size; ++i) {
1323 		ret = add_special_device(IVHD_SPECIAL_IOAPIC,
1324 					 early_ioapic_map[i].id,
1325 					 &early_ioapic_map[i].devid,
1326 					 early_ioapic_map[i].cmd_line);
1327 		if (ret)
1328 			return ret;
1329 	}
1330 
1331 	for (i = 0; i < early_hpet_map_size; ++i) {
1332 		ret = add_special_device(IVHD_SPECIAL_HPET,
1333 					 early_hpet_map[i].id,
1334 					 &early_hpet_map[i].devid,
1335 					 early_hpet_map[i].cmd_line);
1336 		if (ret)
1337 			return ret;
1338 	}
1339 
1340 	for (i = 0; i < early_acpihid_map_size; ++i) {
1341 		ret = add_acpi_hid_device(early_acpihid_map[i].hid,
1342 					  early_acpihid_map[i].uid,
1343 					  &early_acpihid_map[i].devid,
1344 					  early_acpihid_map[i].cmd_line);
1345 		if (ret)
1346 			return ret;
1347 	}
1348 
1349 	return 0;
1350 }
1351 
1352 /*
1353  * Takes a pointer to an AMD IOMMU entry in the ACPI table and
1354  * initializes the hardware and our data structures with it.
1355  */
init_iommu_from_acpi(struct amd_iommu * iommu,struct ivhd_header * h)1356 static int __init init_iommu_from_acpi(struct amd_iommu *iommu,
1357 					struct ivhd_header *h)
1358 {
1359 	u8 *p = (u8 *)h;
1360 	u8 *end = p, flags = 0;
1361 	u16 devid = 0, devid_start = 0, devid_to = 0, seg_id;
1362 	u32 dev_i, ext_flags = 0;
1363 	bool alias = false;
1364 	struct ivhd_entry *e;
1365 	struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg;
1366 	u32 ivhd_size;
1367 	int ret;
1368 
1369 
1370 	ret = add_early_maps();
1371 	if (ret)
1372 		return ret;
1373 
1374 	amd_iommu_apply_ivrs_quirks();
1375 
1376 	/*
1377 	 * First save the recommended feature enable bits from ACPI
1378 	 */
1379 	iommu->acpi_flags = h->flags;
1380 
1381 	/*
1382 	 * Done. Now parse the device entries
1383 	 */
1384 	ivhd_size = get_ivhd_header_size(h);
1385 	if (!ivhd_size) {
1386 		pr_err("Unsupported IVHD type %#x\n", h->type);
1387 		return -EINVAL;
1388 	}
1389 
1390 	p += ivhd_size;
1391 
1392 	end += h->length;
1393 
1394 
1395 	while (p < end) {
1396 		e = (struct ivhd_entry *)p;
1397 		seg_id = pci_seg->id;
1398 
1399 		switch (e->type) {
1400 		case IVHD_DEV_ALL:
1401 
1402 			DUMP_printk("  DEV_ALL\t\t\tflags: %02x\n", e->flags);
1403 
1404 			for (dev_i = 0; dev_i <= pci_seg->last_bdf; ++dev_i)
1405 				set_dev_entry_from_acpi(iommu, dev_i, e->flags, 0);
1406 			break;
1407 		case IVHD_DEV_SELECT:
1408 
1409 			DUMP_printk("  DEV_SELECT\t\t\t devid: %04x:%02x:%02x.%x "
1410 				    "flags: %02x\n",
1411 				    seg_id, PCI_BUS_NUM(e->devid),
1412 				    PCI_SLOT(e->devid),
1413 				    PCI_FUNC(e->devid),
1414 				    e->flags);
1415 
1416 			devid = e->devid;
1417 			set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
1418 			break;
1419 		case IVHD_DEV_SELECT_RANGE_START:
1420 
1421 			DUMP_printk("  DEV_SELECT_RANGE_START\t "
1422 				    "devid: %04x:%02x:%02x.%x flags: %02x\n",
1423 				    seg_id, PCI_BUS_NUM(e->devid),
1424 				    PCI_SLOT(e->devid),
1425 				    PCI_FUNC(e->devid),
1426 				    e->flags);
1427 
1428 			devid_start = e->devid;
1429 			flags = e->flags;
1430 			ext_flags = 0;
1431 			alias = false;
1432 			break;
1433 		case IVHD_DEV_ALIAS:
1434 
1435 			DUMP_printk("  DEV_ALIAS\t\t\t devid: %04x:%02x:%02x.%x "
1436 				    "flags: %02x devid_to: %02x:%02x.%x\n",
1437 				    seg_id, PCI_BUS_NUM(e->devid),
1438 				    PCI_SLOT(e->devid),
1439 				    PCI_FUNC(e->devid),
1440 				    e->flags,
1441 				    PCI_BUS_NUM(e->ext >> 8),
1442 				    PCI_SLOT(e->ext >> 8),
1443 				    PCI_FUNC(e->ext >> 8));
1444 
1445 			devid = e->devid;
1446 			devid_to = e->ext >> 8;
1447 			set_dev_entry_from_acpi(iommu, devid   , e->flags, 0);
1448 			set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);
1449 			pci_seg->alias_table[devid] = devid_to;
1450 			break;
1451 		case IVHD_DEV_ALIAS_RANGE:
1452 
1453 			DUMP_printk("  DEV_ALIAS_RANGE\t\t "
1454 				    "devid: %04x:%02x:%02x.%x flags: %02x "
1455 				    "devid_to: %04x:%02x:%02x.%x\n",
1456 				    seg_id, PCI_BUS_NUM(e->devid),
1457 				    PCI_SLOT(e->devid),
1458 				    PCI_FUNC(e->devid),
1459 				    e->flags,
1460 				    seg_id, PCI_BUS_NUM(e->ext >> 8),
1461 				    PCI_SLOT(e->ext >> 8),
1462 				    PCI_FUNC(e->ext >> 8));
1463 
1464 			devid_start = e->devid;
1465 			flags = e->flags;
1466 			devid_to = e->ext >> 8;
1467 			ext_flags = 0;
1468 			alias = true;
1469 			break;
1470 		case IVHD_DEV_EXT_SELECT:
1471 
1472 			DUMP_printk("  DEV_EXT_SELECT\t\t devid: %04x:%02x:%02x.%x "
1473 				    "flags: %02x ext: %08x\n",
1474 				    seg_id, PCI_BUS_NUM(e->devid),
1475 				    PCI_SLOT(e->devid),
1476 				    PCI_FUNC(e->devid),
1477 				    e->flags, e->ext);
1478 
1479 			devid = e->devid;
1480 			set_dev_entry_from_acpi(iommu, devid, e->flags,
1481 						e->ext);
1482 			break;
1483 		case IVHD_DEV_EXT_SELECT_RANGE:
1484 
1485 			DUMP_printk("  DEV_EXT_SELECT_RANGE\t devid: "
1486 				    "%04x:%02x:%02x.%x flags: %02x ext: %08x\n",
1487 				    seg_id, PCI_BUS_NUM(e->devid),
1488 				    PCI_SLOT(e->devid),
1489 				    PCI_FUNC(e->devid),
1490 				    e->flags, e->ext);
1491 
1492 			devid_start = e->devid;
1493 			flags = e->flags;
1494 			ext_flags = e->ext;
1495 			alias = false;
1496 			break;
1497 		case IVHD_DEV_RANGE_END:
1498 
1499 			DUMP_printk("  DEV_RANGE_END\t\t devid: %04x:%02x:%02x.%x\n",
1500 				    seg_id, PCI_BUS_NUM(e->devid),
1501 				    PCI_SLOT(e->devid),
1502 				    PCI_FUNC(e->devid));
1503 
1504 			devid = e->devid;
1505 			for (dev_i = devid_start; dev_i <= devid; ++dev_i) {
1506 				if (alias) {
1507 					pci_seg->alias_table[dev_i] = devid_to;
1508 					set_dev_entry_from_acpi(iommu,
1509 						devid_to, flags, ext_flags);
1510 				}
1511 				set_dev_entry_from_acpi(iommu, dev_i,
1512 							flags, ext_flags);
1513 			}
1514 			break;
1515 		case IVHD_DEV_SPECIAL: {
1516 			u8 handle, type;
1517 			const char *var;
1518 			u32 devid;
1519 			int ret;
1520 
1521 			handle = e->ext & 0xff;
1522 			devid = PCI_SEG_DEVID_TO_SBDF(seg_id, (e->ext >> 8));
1523 			type   = (e->ext >> 24) & 0xff;
1524 
1525 			if (type == IVHD_SPECIAL_IOAPIC)
1526 				var = "IOAPIC";
1527 			else if (type == IVHD_SPECIAL_HPET)
1528 				var = "HPET";
1529 			else
1530 				var = "UNKNOWN";
1531 
1532 			DUMP_printk("  DEV_SPECIAL(%s[%d])\t\tdevid: %04x:%02x:%02x.%x\n",
1533 				    var, (int)handle,
1534 				    seg_id, PCI_BUS_NUM(devid),
1535 				    PCI_SLOT(devid),
1536 				    PCI_FUNC(devid));
1537 
1538 			ret = add_special_device(type, handle, &devid, false);
1539 			if (ret)
1540 				return ret;
1541 
1542 			/*
1543 			 * add_special_device might update the devid in case a
1544 			 * command-line override is present. So call
1545 			 * set_dev_entry_from_acpi after add_special_device.
1546 			 */
1547 			set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
1548 
1549 			break;
1550 		}
1551 		case IVHD_DEV_ACPI_HID: {
1552 			u32 devid;
1553 			u8 hid[ACPIHID_HID_LEN];
1554 			u8 uid[ACPIHID_UID_LEN];
1555 			int ret;
1556 
1557 			if (h->type != 0x40) {
1558 				pr_err(FW_BUG "Invalid IVHD device type %#x\n",
1559 				       e->type);
1560 				break;
1561 			}
1562 
1563 			BUILD_BUG_ON(sizeof(e->ext_hid) != ACPIHID_HID_LEN - 1);
1564 			memcpy(hid, &e->ext_hid, ACPIHID_HID_LEN - 1);
1565 			hid[ACPIHID_HID_LEN - 1] = '\0';
1566 
1567 			if (!(*hid)) {
1568 				pr_err(FW_BUG "Invalid HID.\n");
1569 				break;
1570 			}
1571 
1572 			uid[0] = '\0';
1573 			switch (e->uidf) {
1574 			case UID_NOT_PRESENT:
1575 
1576 				if (e->uidl != 0)
1577 					pr_warn(FW_BUG "Invalid UID length.\n");
1578 
1579 				break;
1580 			case UID_IS_INTEGER:
1581 
1582 				sprintf(uid, "%d", e->uid);
1583 
1584 				break;
1585 			case UID_IS_CHARACTER:
1586 
1587 				memcpy(uid, &e->uid, e->uidl);
1588 				uid[e->uidl] = '\0';
1589 
1590 				break;
1591 			default:
1592 				break;
1593 			}
1594 
1595 			devid = PCI_SEG_DEVID_TO_SBDF(seg_id, e->devid);
1596 			DUMP_printk("  DEV_ACPI_HID(%s[%s])\t\tdevid: %04x:%02x:%02x.%x\n",
1597 				    hid, uid, seg_id,
1598 				    PCI_BUS_NUM(devid),
1599 				    PCI_SLOT(devid),
1600 				    PCI_FUNC(devid));
1601 
1602 			flags = e->flags;
1603 
1604 			ret = add_acpi_hid_device(hid, uid, &devid, false);
1605 			if (ret)
1606 				return ret;
1607 
1608 			/*
1609 			 * add_special_device might update the devid in case a
1610 			 * command-line override is present. So call
1611 			 * set_dev_entry_from_acpi after add_special_device.
1612 			 */
1613 			set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
1614 
1615 			break;
1616 		}
1617 		default:
1618 			break;
1619 		}
1620 
1621 		p += ivhd_entry_length(p);
1622 	}
1623 
1624 	return 0;
1625 }
1626 
1627 /* Allocate PCI segment data structure */
alloc_pci_segment(u16 id,struct acpi_table_header * ivrs_base)1628 static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id,
1629 					  struct acpi_table_header *ivrs_base)
1630 {
1631 	struct amd_iommu_pci_seg *pci_seg;
1632 	int last_bdf;
1633 
1634 	/*
1635 	 * First parse ACPI tables to find the largest Bus/Dev/Func we need to
1636 	 * handle in this PCI segment. Upon this information the shared data
1637 	 * structures for the PCI segments in the system will be allocated.
1638 	 */
1639 	last_bdf = find_last_devid_acpi(ivrs_base, id);
1640 	if (last_bdf < 0)
1641 		return NULL;
1642 
1643 	pci_seg = kzalloc(sizeof(struct amd_iommu_pci_seg), GFP_KERNEL);
1644 	if (pci_seg == NULL)
1645 		return NULL;
1646 
1647 	pci_seg->last_bdf = last_bdf;
1648 	DUMP_printk("PCI segment : 0x%0x, last bdf : 0x%04x\n", id, last_bdf);
1649 	pci_seg->dev_table_size     = tbl_size(DEV_TABLE_ENTRY_SIZE, last_bdf);
1650 	pci_seg->alias_table_size   = tbl_size(ALIAS_TABLE_ENTRY_SIZE, last_bdf);
1651 	pci_seg->rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE, last_bdf);
1652 
1653 	pci_seg->id = id;
1654 	init_llist_head(&pci_seg->dev_data_list);
1655 	INIT_LIST_HEAD(&pci_seg->unity_map);
1656 	list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list);
1657 
1658 	if (alloc_dev_table(pci_seg))
1659 		return NULL;
1660 	if (alloc_alias_table(pci_seg))
1661 		return NULL;
1662 	if (alloc_rlookup_table(pci_seg))
1663 		return NULL;
1664 
1665 	return pci_seg;
1666 }
1667 
get_pci_segment(u16 id,struct acpi_table_header * ivrs_base)1668 static struct amd_iommu_pci_seg *__init get_pci_segment(u16 id,
1669 					struct acpi_table_header *ivrs_base)
1670 {
1671 	struct amd_iommu_pci_seg *pci_seg;
1672 
1673 	for_each_pci_segment(pci_seg) {
1674 		if (pci_seg->id == id)
1675 			return pci_seg;
1676 	}
1677 
1678 	return alloc_pci_segment(id, ivrs_base);
1679 }
1680 
free_pci_segments(void)1681 static void __init free_pci_segments(void)
1682 {
1683 	struct amd_iommu_pci_seg *pci_seg, *next;
1684 
1685 	for_each_pci_segment_safe(pci_seg, next) {
1686 		list_del(&pci_seg->list);
1687 		free_irq_lookup_table(pci_seg);
1688 		free_rlookup_table(pci_seg);
1689 		free_alias_table(pci_seg);
1690 		free_dev_table(pci_seg);
1691 		kfree(pci_seg);
1692 	}
1693 }
1694 
free_iommu_one(struct amd_iommu * iommu)1695 static void __init free_iommu_one(struct amd_iommu *iommu)
1696 {
1697 	free_cwwb_sem(iommu);
1698 	free_command_buffer(iommu);
1699 	free_event_buffer(iommu);
1700 	free_ppr_log(iommu);
1701 	free_ga_log(iommu);
1702 	iommu_unmap_mmio_space(iommu);
1703 }
1704 
free_iommu_all(void)1705 static void __init free_iommu_all(void)
1706 {
1707 	struct amd_iommu *iommu, *next;
1708 
1709 	for_each_iommu_safe(iommu, next) {
1710 		list_del(&iommu->list);
1711 		free_iommu_one(iommu);
1712 		kfree(iommu);
1713 	}
1714 }
1715 
1716 /*
1717  * Family15h Model 10h-1fh erratum 746 (IOMMU Logging May Stall Translations)
1718  * Workaround:
1719  *     BIOS should disable L2B micellaneous clock gating by setting
1720  *     L2_L2B_CK_GATE_CONTROL[CKGateL2BMiscDisable](D0F2xF4_x90[2]) = 1b
1721  */
amd_iommu_erratum_746_workaround(struct amd_iommu * iommu)1722 static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu)
1723 {
1724 	u32 value;
1725 
1726 	if ((boot_cpu_data.x86 != 0x15) ||
1727 	    (boot_cpu_data.x86_model < 0x10) ||
1728 	    (boot_cpu_data.x86_model > 0x1f))
1729 		return;
1730 
1731 	pci_write_config_dword(iommu->dev, 0xf0, 0x90);
1732 	pci_read_config_dword(iommu->dev, 0xf4, &value);
1733 
1734 	if (value & BIT(2))
1735 		return;
1736 
1737 	/* Select NB indirect register 0x90 and enable writing */
1738 	pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8));
1739 
1740 	pci_write_config_dword(iommu->dev, 0xf4, value | 0x4);
1741 	pci_info(iommu->dev, "Applying erratum 746 workaround\n");
1742 
1743 	/* Clear the enable writing bit */
1744 	pci_write_config_dword(iommu->dev, 0xf0, 0x90);
1745 }
1746 
1747 /*
1748  * Family15h Model 30h-3fh (IOMMU Mishandles ATS Write Permission)
1749  * Workaround:
1750  *     BIOS should enable ATS write permission check by setting
1751  *     L2_DEBUG_3[AtsIgnoreIWDis](D0F2xF4_x47[0]) = 1b
1752  */
amd_iommu_ats_write_check_workaround(struct amd_iommu * iommu)1753 static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu)
1754 {
1755 	u32 value;
1756 
1757 	if ((boot_cpu_data.x86 != 0x15) ||
1758 	    (boot_cpu_data.x86_model < 0x30) ||
1759 	    (boot_cpu_data.x86_model > 0x3f))
1760 		return;
1761 
1762 	/* Test L2_DEBUG_3[AtsIgnoreIWDis] == 1 */
1763 	value = iommu_read_l2(iommu, 0x47);
1764 
1765 	if (value & BIT(0))
1766 		return;
1767 
1768 	/* Set L2_DEBUG_3[AtsIgnoreIWDis] = 1 */
1769 	iommu_write_l2(iommu, 0x47, value | BIT(0));
1770 
1771 	pci_info(iommu->dev, "Applying ATS write check workaround\n");
1772 }
1773 
1774 /*
1775  * This function glues the initialization function for one IOMMU
1776  * together and also allocates the command buffer and programs the
1777  * hardware. It does NOT enable the IOMMU. This is done afterwards.
1778  */
init_iommu_one(struct amd_iommu * iommu,struct ivhd_header * h,struct acpi_table_header * ivrs_base)1779 static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h,
1780 				 struct acpi_table_header *ivrs_base)
1781 {
1782 	struct amd_iommu_pci_seg *pci_seg;
1783 
1784 	pci_seg = get_pci_segment(h->pci_seg, ivrs_base);
1785 	if (pci_seg == NULL)
1786 		return -ENOMEM;
1787 	iommu->pci_seg = pci_seg;
1788 
1789 	raw_spin_lock_init(&iommu->lock);
1790 	atomic64_set(&iommu->cmd_sem_val, 0);
1791 
1792 	/* Add IOMMU to internal data structures */
1793 	list_add_tail(&iommu->list, &amd_iommu_list);
1794 	iommu->index = amd_iommus_present++;
1795 
1796 	if (unlikely(iommu->index >= MAX_IOMMUS)) {
1797 		WARN(1, "System has more IOMMUs than supported by this driver\n");
1798 		return -ENOSYS;
1799 	}
1800 
1801 	/* Index is fine - add IOMMU to the array */
1802 	amd_iommus[iommu->index] = iommu;
1803 
1804 	/*
1805 	 * Copy data from ACPI table entry to the iommu struct
1806 	 */
1807 	iommu->devid   = h->devid;
1808 	iommu->cap_ptr = h->cap_ptr;
1809 	iommu->mmio_phys = h->mmio_phys;
1810 
1811 	switch (h->type) {
1812 	case 0x10:
1813 		/* Check if IVHD EFR contains proper max banks/counters */
1814 		if ((h->efr_attr != 0) &&
1815 		    ((h->efr_attr & (0xF << 13)) != 0) &&
1816 		    ((h->efr_attr & (0x3F << 17)) != 0))
1817 			iommu->mmio_phys_end = MMIO_REG_END_OFFSET;
1818 		else
1819 			iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
1820 
1821 		/*
1822 		 * Note: GA (128-bit IRTE) mode requires cmpxchg16b supports.
1823 		 * GAM also requires GA mode. Therefore, we need to
1824 		 * check cmpxchg16b support before enabling it.
1825 		 */
1826 		if (!boot_cpu_has(X86_FEATURE_CX16) ||
1827 		    ((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0))
1828 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
1829 		break;
1830 	case 0x11:
1831 	case 0x40:
1832 		if (h->efr_reg & (1 << 9))
1833 			iommu->mmio_phys_end = MMIO_REG_END_OFFSET;
1834 		else
1835 			iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET;
1836 
1837 		/*
1838 		 * Note: GA (128-bit IRTE) mode requires cmpxchg16b supports.
1839 		 * XT, GAM also requires GA mode. Therefore, we need to
1840 		 * check cmpxchg16b support before enabling them.
1841 		 */
1842 		if (!boot_cpu_has(X86_FEATURE_CX16) ||
1843 		    ((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0)) {
1844 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY;
1845 			break;
1846 		}
1847 
1848 		if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT))
1849 			amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE;
1850 
1851 		early_iommu_features_init(iommu, h);
1852 
1853 		break;
1854 	default:
1855 		return -EINVAL;
1856 	}
1857 
1858 	iommu->mmio_base = iommu_map_mmio_space(iommu->mmio_phys,
1859 						iommu->mmio_phys_end);
1860 	if (!iommu->mmio_base)
1861 		return -ENOMEM;
1862 
1863 	return init_iommu_from_acpi(iommu, h);
1864 }
1865 
init_iommu_one_late(struct amd_iommu * iommu)1866 static int __init init_iommu_one_late(struct amd_iommu *iommu)
1867 {
1868 	int ret;
1869 
1870 	if (alloc_cwwb_sem(iommu))
1871 		return -ENOMEM;
1872 
1873 	if (alloc_command_buffer(iommu))
1874 		return -ENOMEM;
1875 
1876 	if (alloc_event_buffer(iommu))
1877 		return -ENOMEM;
1878 
1879 	iommu->int_enabled = false;
1880 
1881 	init_translation_status(iommu);
1882 	if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
1883 		iommu_disable(iommu);
1884 		clear_translation_pre_enabled(iommu);
1885 		pr_warn("Translation was enabled for IOMMU:%d but we are not in kdump mode\n",
1886 			iommu->index);
1887 	}
1888 	if (amd_iommu_pre_enabled)
1889 		amd_iommu_pre_enabled = translation_pre_enabled(iommu);
1890 
1891 	if (amd_iommu_irq_remap) {
1892 		ret = amd_iommu_create_irq_domain(iommu);
1893 		if (ret)
1894 			return ret;
1895 	}
1896 
1897 	/*
1898 	 * Make sure IOMMU is not considered to translate itself. The IVRS
1899 	 * table tells us so, but this is a lie!
1900 	 */
1901 	iommu->pci_seg->rlookup_table[iommu->devid] = NULL;
1902 
1903 	return 0;
1904 }
1905 
1906 /**
1907  * get_highest_supported_ivhd_type - Look up the appropriate IVHD type
1908  * @ivrs: Pointer to the IVRS header
1909  *
1910  * This function search through all IVDB of the maximum supported IVHD
1911  */
get_highest_supported_ivhd_type(struct acpi_table_header * ivrs)1912 static u8 get_highest_supported_ivhd_type(struct acpi_table_header *ivrs)
1913 {
1914 	u8 *base = (u8 *)ivrs;
1915 	struct ivhd_header *ivhd = (struct ivhd_header *)
1916 					(base + IVRS_HEADER_LENGTH);
1917 	u8 last_type = ivhd->type;
1918 	u16 devid = ivhd->devid;
1919 
1920 	while (((u8 *)ivhd - base < ivrs->length) &&
1921 	       (ivhd->type <= ACPI_IVHD_TYPE_MAX_SUPPORTED)) {
1922 		u8 *p = (u8 *) ivhd;
1923 
1924 		if (ivhd->devid == devid)
1925 			last_type = ivhd->type;
1926 		ivhd = (struct ivhd_header *)(p + ivhd->length);
1927 	}
1928 
1929 	return last_type;
1930 }
1931 
1932 /*
1933  * Iterates over all IOMMU entries in the ACPI table, allocates the
1934  * IOMMU structure and initializes it with init_iommu_one()
1935  */
init_iommu_all(struct acpi_table_header * table)1936 static int __init init_iommu_all(struct acpi_table_header *table)
1937 {
1938 	u8 *p = (u8 *)table, *end = (u8 *)table;
1939 	struct ivhd_header *h;
1940 	struct amd_iommu *iommu;
1941 	int ret;
1942 
1943 	end += table->length;
1944 	p += IVRS_HEADER_LENGTH;
1945 
1946 	/* Phase 1: Process all IVHD blocks */
1947 	while (p < end) {
1948 		h = (struct ivhd_header *)p;
1949 		if (*p == amd_iommu_target_ivhd_type) {
1950 
1951 			DUMP_printk("device: %04x:%02x:%02x.%01x cap: %04x "
1952 				    "flags: %01x info %04x\n",
1953 				    h->pci_seg, PCI_BUS_NUM(h->devid),
1954 				    PCI_SLOT(h->devid), PCI_FUNC(h->devid),
1955 				    h->cap_ptr, h->flags, h->info);
1956 			DUMP_printk("       mmio-addr: %016llx\n",
1957 				    h->mmio_phys);
1958 
1959 			iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);
1960 			if (iommu == NULL)
1961 				return -ENOMEM;
1962 
1963 			ret = init_iommu_one(iommu, h, table);
1964 			if (ret)
1965 				return ret;
1966 		}
1967 		p += h->length;
1968 
1969 	}
1970 	WARN_ON(p != end);
1971 
1972 	/* Phase 2 : Early feature support check */
1973 	get_global_efr();
1974 
1975 	/* Phase 3 : Enabling IOMMU features */
1976 	for_each_iommu(iommu) {
1977 		ret = init_iommu_one_late(iommu);
1978 		if (ret)
1979 			return ret;
1980 	}
1981 
1982 	return 0;
1983 }
1984 
init_iommu_perf_ctr(struct amd_iommu * iommu)1985 static void init_iommu_perf_ctr(struct amd_iommu *iommu)
1986 {
1987 	u64 val;
1988 	struct pci_dev *pdev = iommu->dev;
1989 
1990 	if (!iommu_feature(iommu, FEATURE_PC))
1991 		return;
1992 
1993 	amd_iommu_pc_present = true;
1994 
1995 	pci_info(pdev, "IOMMU performance counters supported\n");
1996 
1997 	val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET);
1998 	iommu->max_banks = (u8) ((val >> 12) & 0x3f);
1999 	iommu->max_counters = (u8) ((val >> 7) & 0xf);
2000 
2001 	return;
2002 }
2003 
amd_iommu_show_cap(struct device * dev,struct device_attribute * attr,char * buf)2004 static ssize_t amd_iommu_show_cap(struct device *dev,
2005 				  struct device_attribute *attr,
2006 				  char *buf)
2007 {
2008 	struct amd_iommu *iommu = dev_to_amd_iommu(dev);
2009 	return sysfs_emit(buf, "%x\n", iommu->cap);
2010 }
2011 static DEVICE_ATTR(cap, S_IRUGO, amd_iommu_show_cap, NULL);
2012 
amd_iommu_show_features(struct device * dev,struct device_attribute * attr,char * buf)2013 static ssize_t amd_iommu_show_features(struct device *dev,
2014 				       struct device_attribute *attr,
2015 				       char *buf)
2016 {
2017 	struct amd_iommu *iommu = dev_to_amd_iommu(dev);
2018 	return sysfs_emit(buf, "%llx:%llx\n", iommu->features2, iommu->features);
2019 }
2020 static DEVICE_ATTR(features, S_IRUGO, amd_iommu_show_features, NULL);
2021 
2022 static struct attribute *amd_iommu_attrs[] = {
2023 	&dev_attr_cap.attr,
2024 	&dev_attr_features.attr,
2025 	NULL,
2026 };
2027 
2028 static struct attribute_group amd_iommu_group = {
2029 	.name = "amd-iommu",
2030 	.attrs = amd_iommu_attrs,
2031 };
2032 
2033 static const struct attribute_group *amd_iommu_groups[] = {
2034 	&amd_iommu_group,
2035 	NULL,
2036 };
2037 
2038 /*
2039  * Note: IVHD 0x11 and 0x40 also contains exact copy
2040  * of the IOMMU Extended Feature Register [MMIO Offset 0030h].
2041  * Default to EFR in IVHD since it is available sooner (i.e. before PCI init).
2042  */
late_iommu_features_init(struct amd_iommu * iommu)2043 static void __init late_iommu_features_init(struct amd_iommu *iommu)
2044 {
2045 	u64 features, features2;
2046 
2047 	if (!(iommu->cap & (1 << IOMMU_CAP_EFR)))
2048 		return;
2049 
2050 	/* read extended feature bits */
2051 	features = readq(iommu->mmio_base + MMIO_EXT_FEATURES);
2052 	features2 = readq(iommu->mmio_base + MMIO_EXT_FEATURES2);
2053 
2054 	if (!iommu->features) {
2055 		iommu->features = features;
2056 		iommu->features2 = features2;
2057 		return;
2058 	}
2059 
2060 	/*
2061 	 * Sanity check and warn if EFR values from
2062 	 * IVHD and MMIO conflict.
2063 	 */
2064 	if (features != iommu->features ||
2065 	    features2 != iommu->features2) {
2066 		pr_warn(FW_WARN
2067 			"EFR mismatch. Use IVHD EFR (%#llx : %#llx), EFR2 (%#llx : %#llx).\n",
2068 			features, iommu->features,
2069 			features2, iommu->features2);
2070 	}
2071 }
2072 
iommu_init_pci(struct amd_iommu * iommu)2073 static int __init iommu_init_pci(struct amd_iommu *iommu)
2074 {
2075 	int cap_ptr = iommu->cap_ptr;
2076 	int ret;
2077 
2078 	iommu->dev = pci_get_domain_bus_and_slot(iommu->pci_seg->id,
2079 						 PCI_BUS_NUM(iommu->devid),
2080 						 iommu->devid & 0xff);
2081 	if (!iommu->dev)
2082 		return -ENODEV;
2083 
2084 	/* Prevent binding other PCI device drivers to IOMMU devices */
2085 	iommu->dev->match_driver = false;
2086 
2087 	pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
2088 			      &iommu->cap);
2089 
2090 	if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
2091 		amd_iommu_iotlb_sup = false;
2092 
2093 	late_iommu_features_init(iommu);
2094 
2095 	if (iommu_feature(iommu, FEATURE_GT)) {
2096 		int glxval;
2097 		u32 max_pasid;
2098 		u64 pasmax;
2099 
2100 		pasmax = iommu->features & FEATURE_PASID_MASK;
2101 		pasmax >>= FEATURE_PASID_SHIFT;
2102 		max_pasid  = (1 << (pasmax + 1)) - 1;
2103 
2104 		amd_iommu_max_pasid = min(amd_iommu_max_pasid, max_pasid);
2105 
2106 		BUG_ON(amd_iommu_max_pasid & ~PASID_MASK);
2107 
2108 		glxval   = iommu->features & FEATURE_GLXVAL_MASK;
2109 		glxval >>= FEATURE_GLXVAL_SHIFT;
2110 
2111 		if (amd_iommu_max_glx_val == -1)
2112 			amd_iommu_max_glx_val = glxval;
2113 		else
2114 			amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);
2115 	}
2116 
2117 	if (iommu_feature(iommu, FEATURE_GT) &&
2118 	    iommu_feature(iommu, FEATURE_PPR)) {
2119 		iommu->is_iommu_v2   = true;
2120 		amd_iommu_v2_present = true;
2121 	}
2122 
2123 	if (iommu_feature(iommu, FEATURE_PPR) && alloc_ppr_log(iommu))
2124 		return -ENOMEM;
2125 
2126 	if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) {
2127 		pr_info("Using strict mode due to virtualization\n");
2128 		iommu_set_dma_strict();
2129 		amd_iommu_np_cache = true;
2130 	}
2131 
2132 	init_iommu_perf_ctr(iommu);
2133 
2134 	if (amd_iommu_pgtable == AMD_IOMMU_V2) {
2135 		if (!iommu_feature(iommu, FEATURE_GIOSUP) ||
2136 		    !iommu_feature(iommu, FEATURE_GT)) {
2137 			pr_warn("Cannot enable v2 page table for DMA-API. Fallback to v1.\n");
2138 			amd_iommu_pgtable = AMD_IOMMU_V1;
2139 		} else if (iommu_default_passthrough()) {
2140 			pr_warn("V2 page table doesn't support passthrough mode. Fallback to v1.\n");
2141 			amd_iommu_pgtable = AMD_IOMMU_V1;
2142 		}
2143 	}
2144 
2145 	if (is_rd890_iommu(iommu->dev)) {
2146 		int i, j;
2147 
2148 		iommu->root_pdev =
2149 			pci_get_domain_bus_and_slot(iommu->pci_seg->id,
2150 						    iommu->dev->bus->number,
2151 						    PCI_DEVFN(0, 0));
2152 
2153 		/*
2154 		 * Some rd890 systems may not be fully reconfigured by the
2155 		 * BIOS, so it's necessary for us to store this information so
2156 		 * it can be reprogrammed on resume
2157 		 */
2158 		pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,
2159 				&iommu->stored_addr_lo);
2160 		pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,
2161 				&iommu->stored_addr_hi);
2162 
2163 		/* Low bit locks writes to configuration space */
2164 		iommu->stored_addr_lo &= ~1;
2165 
2166 		for (i = 0; i < 6; i++)
2167 			for (j = 0; j < 0x12; j++)
2168 				iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);
2169 
2170 		for (i = 0; i < 0x83; i++)
2171 			iommu->stored_l2[i] = iommu_read_l2(iommu, i);
2172 	}
2173 
2174 	amd_iommu_erratum_746_workaround(iommu);
2175 	amd_iommu_ats_write_check_workaround(iommu);
2176 
2177 	ret = iommu_device_sysfs_add(&iommu->iommu, &iommu->dev->dev,
2178 			       amd_iommu_groups, "ivhd%d", iommu->index);
2179 	if (ret)
2180 		return ret;
2181 
2182 	iommu_device_register(&iommu->iommu, &amd_iommu_ops, NULL);
2183 
2184 	return pci_enable_device(iommu->dev);
2185 }
2186 
print_iommu_info(void)2187 static void print_iommu_info(void)
2188 {
2189 	static const char * const feat_str[] = {
2190 		"PreF", "PPR", "X2APIC", "NX", "GT", "[5]",
2191 		"IA", "GA", "HE", "PC"
2192 	};
2193 	struct amd_iommu *iommu;
2194 
2195 	for_each_iommu(iommu) {
2196 		struct pci_dev *pdev = iommu->dev;
2197 		int i;
2198 
2199 		pci_info(pdev, "Found IOMMU cap 0x%x\n", iommu->cap_ptr);
2200 
2201 		if (iommu->cap & (1 << IOMMU_CAP_EFR)) {
2202 			pr_info("Extended features (%#llx, %#llx):", iommu->features, iommu->features2);
2203 
2204 			for (i = 0; i < ARRAY_SIZE(feat_str); ++i) {
2205 				if (iommu_feature(iommu, (1ULL << i)))
2206 					pr_cont(" %s", feat_str[i]);
2207 			}
2208 
2209 			if (iommu->features & FEATURE_GAM_VAPIC)
2210 				pr_cont(" GA_vAPIC");
2211 
2212 			if (iommu->features & FEATURE_SNP)
2213 				pr_cont(" SNP");
2214 
2215 			pr_cont("\n");
2216 		}
2217 	}
2218 	if (irq_remapping_enabled) {
2219 		pr_info("Interrupt remapping enabled\n");
2220 		if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
2221 			pr_info("X2APIC enabled\n");
2222 	}
2223 	if (amd_iommu_pgtable == AMD_IOMMU_V2) {
2224 		pr_info("V2 page table enabled (Paging mode : %d level)\n",
2225 			amd_iommu_gpt_level);
2226 	}
2227 }
2228 
amd_iommu_init_pci(void)2229 static int __init amd_iommu_init_pci(void)
2230 {
2231 	struct amd_iommu *iommu;
2232 	struct amd_iommu_pci_seg *pci_seg;
2233 	int ret;
2234 
2235 	for_each_iommu(iommu) {
2236 		ret = iommu_init_pci(iommu);
2237 		if (ret) {
2238 			pr_err("IOMMU%d: Failed to initialize IOMMU Hardware (error=%d)!\n",
2239 			       iommu->index, ret);
2240 			goto out;
2241 		}
2242 		/* Need to setup range after PCI init */
2243 		iommu_set_cwwb_range(iommu);
2244 	}
2245 
2246 	/*
2247 	 * Order is important here to make sure any unity map requirements are
2248 	 * fulfilled. The unity mappings are created and written to the device
2249 	 * table during the iommu_init_pci() call.
2250 	 *
2251 	 * After that we call init_device_table_dma() to make sure any
2252 	 * uninitialized DTE will block DMA, and in the end we flush the caches
2253 	 * of all IOMMUs to make sure the changes to the device table are
2254 	 * active.
2255 	 */
2256 	for_each_pci_segment(pci_seg)
2257 		init_device_table_dma(pci_seg);
2258 
2259 	for_each_iommu(iommu)
2260 		iommu_flush_all_caches(iommu);
2261 
2262 	print_iommu_info();
2263 
2264 out:
2265 	return ret;
2266 }
2267 
2268 /****************************************************************************
2269  *
2270  * The following functions initialize the MSI interrupts for all IOMMUs
2271  * in the system. It's a bit challenging because there could be multiple
2272  * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per
2273  * pci_dev.
2274  *
2275  ****************************************************************************/
2276 
iommu_setup_msi(struct amd_iommu * iommu)2277 static int iommu_setup_msi(struct amd_iommu *iommu)
2278 {
2279 	int r;
2280 
2281 	r = pci_enable_msi(iommu->dev);
2282 	if (r)
2283 		return r;
2284 
2285 	r = request_threaded_irq(iommu->dev->irq,
2286 				 amd_iommu_int_handler,
2287 				 amd_iommu_int_thread,
2288 				 0, "AMD-Vi",
2289 				 iommu);
2290 
2291 	if (r) {
2292 		pci_disable_msi(iommu->dev);
2293 		return r;
2294 	}
2295 
2296 	return 0;
2297 }
2298 
2299 union intcapxt {
2300 	u64	capxt;
2301 	struct {
2302 		u64	reserved_0		:  2,
2303 			dest_mode_logical	:  1,
2304 			reserved_1		:  5,
2305 			destid_0_23		: 24,
2306 			vector			:  8,
2307 			reserved_2		: 16,
2308 			destid_24_31		:  8;
2309 	};
2310 } __attribute__ ((packed));
2311 
2312 
2313 static struct irq_chip intcapxt_controller;
2314 
intcapxt_irqdomain_activate(struct irq_domain * domain,struct irq_data * irqd,bool reserve)2315 static int intcapxt_irqdomain_activate(struct irq_domain *domain,
2316 				       struct irq_data *irqd, bool reserve)
2317 {
2318 	return 0;
2319 }
2320 
intcapxt_irqdomain_deactivate(struct irq_domain * domain,struct irq_data * irqd)2321 static void intcapxt_irqdomain_deactivate(struct irq_domain *domain,
2322 					  struct irq_data *irqd)
2323 {
2324 }
2325 
2326 
intcapxt_irqdomain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * arg)2327 static int intcapxt_irqdomain_alloc(struct irq_domain *domain, unsigned int virq,
2328 				    unsigned int nr_irqs, void *arg)
2329 {
2330 	struct irq_alloc_info *info = arg;
2331 	int i, ret;
2332 
2333 	if (!info || info->type != X86_IRQ_ALLOC_TYPE_AMDVI)
2334 		return -EINVAL;
2335 
2336 	ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg);
2337 	if (ret < 0)
2338 		return ret;
2339 
2340 	for (i = virq; i < virq + nr_irqs; i++) {
2341 		struct irq_data *irqd = irq_domain_get_irq_data(domain, i);
2342 
2343 		irqd->chip = &intcapxt_controller;
2344 		irqd->hwirq = info->hwirq;
2345 		irqd->chip_data = info->data;
2346 		__irq_set_handler(i, handle_edge_irq, 0, "edge");
2347 	}
2348 
2349 	return ret;
2350 }
2351 
intcapxt_irqdomain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)2352 static void intcapxt_irqdomain_free(struct irq_domain *domain, unsigned int virq,
2353 				    unsigned int nr_irqs)
2354 {
2355 	irq_domain_free_irqs_top(domain, virq, nr_irqs);
2356 }
2357 
2358 
intcapxt_unmask_irq(struct irq_data * irqd)2359 static void intcapxt_unmask_irq(struct irq_data *irqd)
2360 {
2361 	struct amd_iommu *iommu = irqd->chip_data;
2362 	struct irq_cfg *cfg = irqd_cfg(irqd);
2363 	union intcapxt xt;
2364 
2365 	xt.capxt = 0ULL;
2366 	xt.dest_mode_logical = apic->dest_mode_logical;
2367 	xt.vector = cfg->vector;
2368 	xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0);
2369 	xt.destid_24_31 = cfg->dest_apicid >> 24;
2370 
2371 	writeq(xt.capxt, iommu->mmio_base + irqd->hwirq);
2372 }
2373 
intcapxt_mask_irq(struct irq_data * irqd)2374 static void intcapxt_mask_irq(struct irq_data *irqd)
2375 {
2376 	struct amd_iommu *iommu = irqd->chip_data;
2377 
2378 	writeq(0, iommu->mmio_base + irqd->hwirq);
2379 }
2380 
2381 
intcapxt_set_affinity(struct irq_data * irqd,const struct cpumask * mask,bool force)2382 static int intcapxt_set_affinity(struct irq_data *irqd,
2383 				 const struct cpumask *mask, bool force)
2384 {
2385 	struct irq_data *parent = irqd->parent_data;
2386 	int ret;
2387 
2388 	ret = parent->chip->irq_set_affinity(parent, mask, force);
2389 	if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
2390 		return ret;
2391 	return 0;
2392 }
2393 
intcapxt_set_wake(struct irq_data * irqd,unsigned int on)2394 static int intcapxt_set_wake(struct irq_data *irqd, unsigned int on)
2395 {
2396 	return on ? -EOPNOTSUPP : 0;
2397 }
2398 
2399 static struct irq_chip intcapxt_controller = {
2400 	.name			= "IOMMU-MSI",
2401 	.irq_unmask		= intcapxt_unmask_irq,
2402 	.irq_mask		= intcapxt_mask_irq,
2403 	.irq_ack		= irq_chip_ack_parent,
2404 	.irq_retrigger		= irq_chip_retrigger_hierarchy,
2405 	.irq_set_affinity       = intcapxt_set_affinity,
2406 	.irq_set_wake		= intcapxt_set_wake,
2407 	.flags			= IRQCHIP_MASK_ON_SUSPEND,
2408 };
2409 
2410 static const struct irq_domain_ops intcapxt_domain_ops = {
2411 	.alloc			= intcapxt_irqdomain_alloc,
2412 	.free			= intcapxt_irqdomain_free,
2413 	.activate		= intcapxt_irqdomain_activate,
2414 	.deactivate		= intcapxt_irqdomain_deactivate,
2415 };
2416 
2417 
2418 static struct irq_domain *iommu_irqdomain;
2419 
iommu_get_irqdomain(void)2420 static struct irq_domain *iommu_get_irqdomain(void)
2421 {
2422 	struct fwnode_handle *fn;
2423 
2424 	/* No need for locking here (yet) as the init is single-threaded */
2425 	if (iommu_irqdomain)
2426 		return iommu_irqdomain;
2427 
2428 	fn = irq_domain_alloc_named_fwnode("AMD-Vi-MSI");
2429 	if (!fn)
2430 		return NULL;
2431 
2432 	iommu_irqdomain = irq_domain_create_hierarchy(x86_vector_domain, 0, 0,
2433 						      fn, &intcapxt_domain_ops,
2434 						      NULL);
2435 	if (!iommu_irqdomain)
2436 		irq_domain_free_fwnode(fn);
2437 
2438 	return iommu_irqdomain;
2439 }
2440 
__iommu_setup_intcapxt(struct amd_iommu * iommu,const char * devname,int hwirq,irq_handler_t thread_fn)2441 static int __iommu_setup_intcapxt(struct amd_iommu *iommu, const char *devname,
2442 				  int hwirq, irq_handler_t thread_fn)
2443 {
2444 	struct irq_domain *domain;
2445 	struct irq_alloc_info info;
2446 	int irq, ret;
2447 	int node = dev_to_node(&iommu->dev->dev);
2448 
2449 	domain = iommu_get_irqdomain();
2450 	if (!domain)
2451 		return -ENXIO;
2452 
2453 	init_irq_alloc_info(&info, NULL);
2454 	info.type = X86_IRQ_ALLOC_TYPE_AMDVI;
2455 	info.data = iommu;
2456 	info.hwirq = hwirq;
2457 
2458 	irq = irq_domain_alloc_irqs(domain, 1, node, &info);
2459 	if (irq < 0) {
2460 		irq_domain_remove(domain);
2461 		return irq;
2462 	}
2463 
2464 	ret = request_threaded_irq(irq, amd_iommu_int_handler,
2465 				   thread_fn, 0, devname, iommu);
2466 	if (ret) {
2467 		irq_domain_free_irqs(irq, 1);
2468 		irq_domain_remove(domain);
2469 		return ret;
2470 	}
2471 
2472 	return 0;
2473 }
2474 
iommu_setup_intcapxt(struct amd_iommu * iommu)2475 static int iommu_setup_intcapxt(struct amd_iommu *iommu)
2476 {
2477 	int ret;
2478 
2479 	snprintf(iommu->evt_irq_name, sizeof(iommu->evt_irq_name),
2480 		 "AMD-Vi%d-Evt", iommu->index);
2481 	ret = __iommu_setup_intcapxt(iommu, iommu->evt_irq_name,
2482 				     MMIO_INTCAPXT_EVT_OFFSET,
2483 				     amd_iommu_int_thread_evtlog);
2484 	if (ret)
2485 		return ret;
2486 
2487 	snprintf(iommu->ppr_irq_name, sizeof(iommu->ppr_irq_name),
2488 		 "AMD-Vi%d-PPR", iommu->index);
2489 	ret = __iommu_setup_intcapxt(iommu, iommu->ppr_irq_name,
2490 				     MMIO_INTCAPXT_PPR_OFFSET,
2491 				     amd_iommu_int_thread_pprlog);
2492 	if (ret)
2493 		return ret;
2494 
2495 #ifdef CONFIG_IRQ_REMAP
2496 	snprintf(iommu->ga_irq_name, sizeof(iommu->ga_irq_name),
2497 		 "AMD-Vi%d-GA", iommu->index);
2498 	ret = __iommu_setup_intcapxt(iommu, iommu->ga_irq_name,
2499 				     MMIO_INTCAPXT_GALOG_OFFSET,
2500 				     amd_iommu_int_thread_galog);
2501 #endif
2502 
2503 	return ret;
2504 }
2505 
iommu_init_irq(struct amd_iommu * iommu)2506 static int iommu_init_irq(struct amd_iommu *iommu)
2507 {
2508 	int ret;
2509 
2510 	if (iommu->int_enabled)
2511 		goto enable_faults;
2512 
2513 	if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
2514 		ret = iommu_setup_intcapxt(iommu);
2515 	else if (iommu->dev->msi_cap)
2516 		ret = iommu_setup_msi(iommu);
2517 	else
2518 		ret = -ENODEV;
2519 
2520 	if (ret)
2521 		return ret;
2522 
2523 	iommu->int_enabled = true;
2524 enable_faults:
2525 
2526 	if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE)
2527 		iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN);
2528 
2529 	iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
2530 
2531 	return 0;
2532 }
2533 
2534 /****************************************************************************
2535  *
2536  * The next functions belong to the third pass of parsing the ACPI
2537  * table. In this last pass the memory mapping requirements are
2538  * gathered (like exclusion and unity mapping ranges).
2539  *
2540  ****************************************************************************/
2541 
free_unity_maps(void)2542 static void __init free_unity_maps(void)
2543 {
2544 	struct unity_map_entry *entry, *next;
2545 	struct amd_iommu_pci_seg *p, *pci_seg;
2546 
2547 	for_each_pci_segment_safe(pci_seg, p) {
2548 		list_for_each_entry_safe(entry, next, &pci_seg->unity_map, list) {
2549 			list_del(&entry->list);
2550 			kfree(entry);
2551 		}
2552 	}
2553 }
2554 
2555 /* called for unity map ACPI definition */
init_unity_map_range(struct ivmd_header * m,struct acpi_table_header * ivrs_base)2556 static int __init init_unity_map_range(struct ivmd_header *m,
2557 				       struct acpi_table_header *ivrs_base)
2558 {
2559 	struct unity_map_entry *e = NULL;
2560 	struct amd_iommu_pci_seg *pci_seg;
2561 	char *s;
2562 
2563 	pci_seg = get_pci_segment(m->pci_seg, ivrs_base);
2564 	if (pci_seg == NULL)
2565 		return -ENOMEM;
2566 
2567 	e = kzalloc(sizeof(*e), GFP_KERNEL);
2568 	if (e == NULL)
2569 		return -ENOMEM;
2570 
2571 	switch (m->type) {
2572 	default:
2573 		kfree(e);
2574 		return 0;
2575 	case ACPI_IVMD_TYPE:
2576 		s = "IVMD_TYPEi\t\t\t";
2577 		e->devid_start = e->devid_end = m->devid;
2578 		break;
2579 	case ACPI_IVMD_TYPE_ALL:
2580 		s = "IVMD_TYPE_ALL\t\t";
2581 		e->devid_start = 0;
2582 		e->devid_end = pci_seg->last_bdf;
2583 		break;
2584 	case ACPI_IVMD_TYPE_RANGE:
2585 		s = "IVMD_TYPE_RANGE\t\t";
2586 		e->devid_start = m->devid;
2587 		e->devid_end = m->aux;
2588 		break;
2589 	}
2590 	e->address_start = PAGE_ALIGN(m->range_start);
2591 	e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
2592 	e->prot = m->flags >> 1;
2593 
2594 	/*
2595 	 * Treat per-device exclusion ranges as r/w unity-mapped regions
2596 	 * since some buggy BIOSes might lead to the overwritten exclusion
2597 	 * range (exclusion_start and exclusion_length members). This
2598 	 * happens when there are multiple exclusion ranges (IVMD entries)
2599 	 * defined in ACPI table.
2600 	 */
2601 	if (m->flags & IVMD_FLAG_EXCL_RANGE)
2602 		e->prot = (IVMD_FLAG_IW | IVMD_FLAG_IR) >> 1;
2603 
2604 	DUMP_printk("%s devid_start: %04x:%02x:%02x.%x devid_end: "
2605 		    "%04x:%02x:%02x.%x range_start: %016llx range_end: %016llx"
2606 		    " flags: %x\n", s, m->pci_seg,
2607 		    PCI_BUS_NUM(e->devid_start), PCI_SLOT(e->devid_start),
2608 		    PCI_FUNC(e->devid_start), m->pci_seg,
2609 		    PCI_BUS_NUM(e->devid_end),
2610 		    PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),
2611 		    e->address_start, e->address_end, m->flags);
2612 
2613 	list_add_tail(&e->list, &pci_seg->unity_map);
2614 
2615 	return 0;
2616 }
2617 
2618 /* iterates over all memory definitions we find in the ACPI table */
init_memory_definitions(struct acpi_table_header * table)2619 static int __init init_memory_definitions(struct acpi_table_header *table)
2620 {
2621 	u8 *p = (u8 *)table, *end = (u8 *)table;
2622 	struct ivmd_header *m;
2623 
2624 	end += table->length;
2625 	p += IVRS_HEADER_LENGTH;
2626 
2627 	while (p < end) {
2628 		m = (struct ivmd_header *)p;
2629 		if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE))
2630 			init_unity_map_range(m, table);
2631 
2632 		p += m->length;
2633 	}
2634 
2635 	return 0;
2636 }
2637 
2638 /*
2639  * Init the device table to not allow DMA access for devices
2640  */
init_device_table_dma(struct amd_iommu_pci_seg * pci_seg)2641 static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg)
2642 {
2643 	u32 devid;
2644 	struct dev_table_entry *dev_table = pci_seg->dev_table;
2645 
2646 	if (dev_table == NULL)
2647 		return;
2648 
2649 	for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {
2650 		__set_dev_entry_bit(dev_table, devid, DEV_ENTRY_VALID);
2651 		if (!amd_iommu_snp_en)
2652 			__set_dev_entry_bit(dev_table, devid, DEV_ENTRY_TRANSLATION);
2653 	}
2654 }
2655 
uninit_device_table_dma(struct amd_iommu_pci_seg * pci_seg)2656 static void __init uninit_device_table_dma(struct amd_iommu_pci_seg *pci_seg)
2657 {
2658 	u32 devid;
2659 	struct dev_table_entry *dev_table = pci_seg->dev_table;
2660 
2661 	if (dev_table == NULL)
2662 		return;
2663 
2664 	for (devid = 0; devid <= pci_seg->last_bdf; ++devid) {
2665 		dev_table[devid].data[0] = 0ULL;
2666 		dev_table[devid].data[1] = 0ULL;
2667 	}
2668 }
2669 
init_device_table(void)2670 static void init_device_table(void)
2671 {
2672 	struct amd_iommu_pci_seg *pci_seg;
2673 	u32 devid;
2674 
2675 	if (!amd_iommu_irq_remap)
2676 		return;
2677 
2678 	for_each_pci_segment(pci_seg) {
2679 		for (devid = 0; devid <= pci_seg->last_bdf; ++devid)
2680 			__set_dev_entry_bit(pci_seg->dev_table,
2681 					    devid, DEV_ENTRY_IRQ_TBL_EN);
2682 	}
2683 }
2684 
iommu_init_flags(struct amd_iommu * iommu)2685 static void iommu_init_flags(struct amd_iommu *iommu)
2686 {
2687 	iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?
2688 		iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :
2689 		iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);
2690 
2691 	iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?
2692 		iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :
2693 		iommu_feature_disable(iommu, CONTROL_PASSPW_EN);
2694 
2695 	iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?
2696 		iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :
2697 		iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);
2698 
2699 	iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?
2700 		iommu_feature_enable(iommu, CONTROL_ISOC_EN) :
2701 		iommu_feature_disable(iommu, CONTROL_ISOC_EN);
2702 
2703 	/*
2704 	 * make IOMMU memory accesses cache coherent
2705 	 */
2706 	iommu_feature_enable(iommu, CONTROL_COHERENT_EN);
2707 
2708 	/* Set IOTLB invalidation timeout to 1s */
2709 	iommu_set_inv_tlb_timeout(iommu, CTRL_INV_TO_1S);
2710 }
2711 
iommu_apply_resume_quirks(struct amd_iommu * iommu)2712 static void iommu_apply_resume_quirks(struct amd_iommu *iommu)
2713 {
2714 	int i, j;
2715 	u32 ioc_feature_control;
2716 	struct pci_dev *pdev = iommu->root_pdev;
2717 
2718 	/* RD890 BIOSes may not have completely reconfigured the iommu */
2719 	if (!is_rd890_iommu(iommu->dev) || !pdev)
2720 		return;
2721 
2722 	/*
2723 	 * First, we need to ensure that the iommu is enabled. This is
2724 	 * controlled by a register in the northbridge
2725 	 */
2726 
2727 	/* Select Northbridge indirect register 0x75 and enable writing */
2728 	pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7));
2729 	pci_read_config_dword(pdev, 0x64, &ioc_feature_control);
2730 
2731 	/* Enable the iommu */
2732 	if (!(ioc_feature_control & 0x1))
2733 		pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1);
2734 
2735 	/* Restore the iommu BAR */
2736 	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
2737 			       iommu->stored_addr_lo);
2738 	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,
2739 			       iommu->stored_addr_hi);
2740 
2741 	/* Restore the l1 indirect regs for each of the 6 l1s */
2742 	for (i = 0; i < 6; i++)
2743 		for (j = 0; j < 0x12; j++)
2744 			iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);
2745 
2746 	/* Restore the l2 indirect regs */
2747 	for (i = 0; i < 0x83; i++)
2748 		iommu_write_l2(iommu, i, iommu->stored_l2[i]);
2749 
2750 	/* Lock PCI setup registers */
2751 	pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
2752 			       iommu->stored_addr_lo | 1);
2753 }
2754 
iommu_enable_ga(struct amd_iommu * iommu)2755 static void iommu_enable_ga(struct amd_iommu *iommu)
2756 {
2757 #ifdef CONFIG_IRQ_REMAP
2758 	switch (amd_iommu_guest_ir) {
2759 	case AMD_IOMMU_GUEST_IR_VAPIC:
2760 	case AMD_IOMMU_GUEST_IR_LEGACY_GA:
2761 		iommu_feature_enable(iommu, CONTROL_GA_EN);
2762 		iommu->irte_ops = &irte_128_ops;
2763 		break;
2764 	default:
2765 		iommu->irte_ops = &irte_32_ops;
2766 		break;
2767 	}
2768 #endif
2769 }
2770 
iommu_disable_irtcachedis(struct amd_iommu * iommu)2771 static void iommu_disable_irtcachedis(struct amd_iommu *iommu)
2772 {
2773 	iommu_feature_disable(iommu, CONTROL_IRTCACHEDIS);
2774 }
2775 
iommu_enable_irtcachedis(struct amd_iommu * iommu)2776 static void iommu_enable_irtcachedis(struct amd_iommu *iommu)
2777 {
2778 	u64 ctrl;
2779 
2780 	if (!amd_iommu_irtcachedis)
2781 		return;
2782 
2783 	/*
2784 	 * Note:
2785 	 * The support for IRTCacheDis feature is dertermined by
2786 	 * checking if the bit is writable.
2787 	 */
2788 	iommu_feature_enable(iommu, CONTROL_IRTCACHEDIS);
2789 	ctrl = readq(iommu->mmio_base +  MMIO_CONTROL_OFFSET);
2790 	ctrl &= (1ULL << CONTROL_IRTCACHEDIS);
2791 	if (ctrl)
2792 		iommu->irtcachedis_enabled = true;
2793 	pr_info("iommu%d (%#06x) : IRT cache is %s\n",
2794 		iommu->index, iommu->devid,
2795 		iommu->irtcachedis_enabled ? "disabled" : "enabled");
2796 }
2797 
early_enable_iommu(struct amd_iommu * iommu)2798 static void early_enable_iommu(struct amd_iommu *iommu)
2799 {
2800 	iommu_disable(iommu);
2801 	iommu_init_flags(iommu);
2802 	iommu_set_device_table(iommu);
2803 	iommu_enable_command_buffer(iommu);
2804 	iommu_enable_event_buffer(iommu);
2805 	iommu_set_exclusion_range(iommu);
2806 	iommu_enable_ga(iommu);
2807 	iommu_enable_xt(iommu);
2808 	iommu_enable_irtcachedis(iommu);
2809 	iommu_enable(iommu);
2810 	iommu_flush_all_caches(iommu);
2811 }
2812 
2813 /*
2814  * This function finally enables all IOMMUs found in the system after
2815  * they have been initialized.
2816  *
2817  * Or if in kdump kernel and IOMMUs are all pre-enabled, try to copy
2818  * the old content of device table entries. Not this case or copy failed,
2819  * just continue as normal kernel does.
2820  */
early_enable_iommus(void)2821 static void early_enable_iommus(void)
2822 {
2823 	struct amd_iommu *iommu;
2824 	struct amd_iommu_pci_seg *pci_seg;
2825 
2826 	if (!copy_device_table()) {
2827 		/*
2828 		 * If come here because of failure in copying device table from old
2829 		 * kernel with all IOMMUs enabled, print error message and try to
2830 		 * free allocated old_dev_tbl_cpy.
2831 		 */
2832 		if (amd_iommu_pre_enabled)
2833 			pr_err("Failed to copy DEV table from previous kernel.\n");
2834 
2835 		for_each_pci_segment(pci_seg) {
2836 			if (pci_seg->old_dev_tbl_cpy != NULL) {
2837 				free_pages((unsigned long)pci_seg->old_dev_tbl_cpy,
2838 						get_order(pci_seg->dev_table_size));
2839 				pci_seg->old_dev_tbl_cpy = NULL;
2840 			}
2841 		}
2842 
2843 		for_each_iommu(iommu) {
2844 			clear_translation_pre_enabled(iommu);
2845 			early_enable_iommu(iommu);
2846 		}
2847 	} else {
2848 		pr_info("Copied DEV table from previous kernel.\n");
2849 
2850 		for_each_pci_segment(pci_seg) {
2851 			free_pages((unsigned long)pci_seg->dev_table,
2852 				   get_order(pci_seg->dev_table_size));
2853 			pci_seg->dev_table = pci_seg->old_dev_tbl_cpy;
2854 		}
2855 
2856 		for_each_iommu(iommu) {
2857 			iommu_disable_command_buffer(iommu);
2858 			iommu_disable_event_buffer(iommu);
2859 			iommu_disable_irtcachedis(iommu);
2860 			iommu_enable_command_buffer(iommu);
2861 			iommu_enable_event_buffer(iommu);
2862 			iommu_enable_ga(iommu);
2863 			iommu_enable_xt(iommu);
2864 			iommu_enable_irtcachedis(iommu);
2865 			iommu_set_device_table(iommu);
2866 			iommu_flush_all_caches(iommu);
2867 		}
2868 	}
2869 }
2870 
enable_iommus_v2(void)2871 static void enable_iommus_v2(void)
2872 {
2873 	struct amd_iommu *iommu;
2874 
2875 	for_each_iommu(iommu) {
2876 		iommu_enable_ppr_log(iommu);
2877 		iommu_enable_gt(iommu);
2878 	}
2879 }
2880 
enable_iommus_vapic(void)2881 static void enable_iommus_vapic(void)
2882 {
2883 #ifdef CONFIG_IRQ_REMAP
2884 	u32 status, i;
2885 	struct amd_iommu *iommu;
2886 
2887 	for_each_iommu(iommu) {
2888 		/*
2889 		 * Disable GALog if already running. It could have been enabled
2890 		 * in the previous boot before kdump.
2891 		 */
2892 		status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
2893 		if (!(status & MMIO_STATUS_GALOG_RUN_MASK))
2894 			continue;
2895 
2896 		iommu_feature_disable(iommu, CONTROL_GALOG_EN);
2897 		iommu_feature_disable(iommu, CONTROL_GAINT_EN);
2898 
2899 		/*
2900 		 * Need to set and poll check the GALOGRun bit to zero before
2901 		 * we can set/ modify GA Log registers safely.
2902 		 */
2903 		for (i = 0; i < LOOP_TIMEOUT; ++i) {
2904 			status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET);
2905 			if (!(status & MMIO_STATUS_GALOG_RUN_MASK))
2906 				break;
2907 			udelay(10);
2908 		}
2909 
2910 		if (WARN_ON(i >= LOOP_TIMEOUT))
2911 			return;
2912 	}
2913 
2914 	if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) &&
2915 	    !check_feature_on_all_iommus(FEATURE_GAM_VAPIC)) {
2916 		amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
2917 		return;
2918 	}
2919 
2920 	if (amd_iommu_snp_en &&
2921 	    !FEATURE_SNPAVICSUP_GAM(amd_iommu_efr2)) {
2922 		pr_warn("Force to disable Virtual APIC due to SNP\n");
2923 		amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
2924 		return;
2925 	}
2926 
2927 	/* Enabling GAM and SNPAVIC support */
2928 	for_each_iommu(iommu) {
2929 		if (iommu_init_ga_log(iommu) ||
2930 		    iommu_ga_log_enable(iommu))
2931 			return;
2932 
2933 		iommu_feature_enable(iommu, CONTROL_GAM_EN);
2934 		if (amd_iommu_snp_en)
2935 			iommu_feature_enable(iommu, CONTROL_SNPAVIC_EN);
2936 	}
2937 
2938 	amd_iommu_irq_ops.capability |= (1 << IRQ_POSTING_CAP);
2939 	pr_info("Virtual APIC enabled\n");
2940 #endif
2941 }
2942 
enable_iommus(void)2943 static void enable_iommus(void)
2944 {
2945 	early_enable_iommus();
2946 }
2947 
disable_iommus(void)2948 static void disable_iommus(void)
2949 {
2950 	struct amd_iommu *iommu;
2951 
2952 	for_each_iommu(iommu)
2953 		iommu_disable(iommu);
2954 
2955 #ifdef CONFIG_IRQ_REMAP
2956 	if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
2957 		amd_iommu_irq_ops.capability &= ~(1 << IRQ_POSTING_CAP);
2958 #endif
2959 }
2960 
2961 /*
2962  * Suspend/Resume support
2963  * disable suspend until real resume implemented
2964  */
2965 
amd_iommu_resume(void)2966 static void amd_iommu_resume(void)
2967 {
2968 	struct amd_iommu *iommu;
2969 
2970 	for_each_iommu(iommu)
2971 		iommu_apply_resume_quirks(iommu);
2972 
2973 	/* re-load the hardware */
2974 	enable_iommus();
2975 
2976 	amd_iommu_enable_interrupts();
2977 }
2978 
amd_iommu_suspend(void)2979 static int amd_iommu_suspend(void)
2980 {
2981 	/* disable IOMMUs to go out of the way for BIOS */
2982 	disable_iommus();
2983 
2984 	return 0;
2985 }
2986 
2987 static struct syscore_ops amd_iommu_syscore_ops = {
2988 	.suspend = amd_iommu_suspend,
2989 	.resume = amd_iommu_resume,
2990 };
2991 
free_iommu_resources(void)2992 static void __init free_iommu_resources(void)
2993 {
2994 	kmem_cache_destroy(amd_iommu_irq_cache);
2995 	amd_iommu_irq_cache = NULL;
2996 
2997 	free_iommu_all();
2998 	free_pci_segments();
2999 }
3000 
3001 /* SB IOAPIC is always on this device in AMD systems */
3002 #define IOAPIC_SB_DEVID		((0x00 << 8) | PCI_DEVFN(0x14, 0))
3003 
check_ioapic_information(void)3004 static bool __init check_ioapic_information(void)
3005 {
3006 	const char *fw_bug = FW_BUG;
3007 	bool ret, has_sb_ioapic;
3008 	int idx;
3009 
3010 	has_sb_ioapic = false;
3011 	ret           = false;
3012 
3013 	/*
3014 	 * If we have map overrides on the kernel command line the
3015 	 * messages in this function might not describe firmware bugs
3016 	 * anymore - so be careful
3017 	 */
3018 	if (cmdline_maps)
3019 		fw_bug = "";
3020 
3021 	for (idx = 0; idx < nr_ioapics; idx++) {
3022 		int devid, id = mpc_ioapic_id(idx);
3023 
3024 		devid = get_ioapic_devid(id);
3025 		if (devid < 0) {
3026 			pr_err("%s: IOAPIC[%d] not in IVRS table\n",
3027 				fw_bug, id);
3028 			ret = false;
3029 		} else if (devid == IOAPIC_SB_DEVID) {
3030 			has_sb_ioapic = true;
3031 			ret           = true;
3032 		}
3033 	}
3034 
3035 	if (!has_sb_ioapic) {
3036 		/*
3037 		 * We expect the SB IOAPIC to be listed in the IVRS
3038 		 * table. The system timer is connected to the SB IOAPIC
3039 		 * and if we don't have it in the list the system will
3040 		 * panic at boot time.  This situation usually happens
3041 		 * when the BIOS is buggy and provides us the wrong
3042 		 * device id for the IOAPIC in the system.
3043 		 */
3044 		pr_err("%s: No southbridge IOAPIC found\n", fw_bug);
3045 	}
3046 
3047 	if (!ret)
3048 		pr_err("Disabling interrupt remapping\n");
3049 
3050 	return ret;
3051 }
3052 
free_dma_resources(void)3053 static void __init free_dma_resources(void)
3054 {
3055 	free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,
3056 		   get_order(MAX_DOMAIN_ID/8));
3057 	amd_iommu_pd_alloc_bitmap = NULL;
3058 
3059 	free_unity_maps();
3060 }
3061 
ivinfo_init(void * ivrs)3062 static void __init ivinfo_init(void *ivrs)
3063 {
3064 	amd_iommu_ivinfo = *((u32 *)(ivrs + IOMMU_IVINFO_OFFSET));
3065 }
3066 
3067 /*
3068  * This is the hardware init function for AMD IOMMU in the system.
3069  * This function is called either from amd_iommu_init or from the interrupt
3070  * remapping setup code.
3071  *
3072  * This function basically parses the ACPI table for AMD IOMMU (IVRS)
3073  * four times:
3074  *
3075  *	1 pass) Discover the most comprehensive IVHD type to use.
3076  *
3077  *	2 pass) Find the highest PCI device id the driver has to handle.
3078  *		Upon this information the size of the data structures is
3079  *		determined that needs to be allocated.
3080  *
3081  *	3 pass) Initialize the data structures just allocated with the
3082  *		information in the ACPI table about available AMD IOMMUs
3083  *		in the system. It also maps the PCI devices in the
3084  *		system to specific IOMMUs
3085  *
3086  *	4 pass) After the basic data structures are allocated and
3087  *		initialized we update them with information about memory
3088  *		remapping requirements parsed out of the ACPI table in
3089  *		this last pass.
3090  *
3091  * After everything is set up the IOMMUs are enabled and the necessary
3092  * hotplug and suspend notifiers are registered.
3093  */
early_amd_iommu_init(void)3094 static int __init early_amd_iommu_init(void)
3095 {
3096 	struct acpi_table_header *ivrs_base;
3097 	int remap_cache_sz, ret;
3098 	acpi_status status;
3099 
3100 	if (!amd_iommu_detected)
3101 		return -ENODEV;
3102 
3103 	status = acpi_get_table("IVRS", 0, &ivrs_base);
3104 	if (status == AE_NOT_FOUND)
3105 		return -ENODEV;
3106 	else if (ACPI_FAILURE(status)) {
3107 		const char *err = acpi_format_exception(status);
3108 		pr_err("IVRS table error: %s\n", err);
3109 		return -EINVAL;
3110 	}
3111 
3112 	/*
3113 	 * Validate checksum here so we don't need to do it when
3114 	 * we actually parse the table
3115 	 */
3116 	ret = check_ivrs_checksum(ivrs_base);
3117 	if (ret)
3118 		goto out;
3119 
3120 	ivinfo_init(ivrs_base);
3121 
3122 	amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base);
3123 	DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type);
3124 
3125 	/* Device table - directly used by all IOMMUs */
3126 	ret = -ENOMEM;
3127 
3128 	amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages(
3129 					    GFP_KERNEL | __GFP_ZERO,
3130 					    get_order(MAX_DOMAIN_ID/8));
3131 	if (amd_iommu_pd_alloc_bitmap == NULL)
3132 		goto out;
3133 
3134 	/*
3135 	 * never allocate domain 0 because its used as the non-allocated and
3136 	 * error value placeholder
3137 	 */
3138 	__set_bit(0, amd_iommu_pd_alloc_bitmap);
3139 
3140 	/*
3141 	 * now the data structures are allocated and basically initialized
3142 	 * start the real acpi table scan
3143 	 */
3144 	ret = init_iommu_all(ivrs_base);
3145 	if (ret)
3146 		goto out;
3147 
3148 	/* 5 level guest page table */
3149 	if (cpu_feature_enabled(X86_FEATURE_LA57) &&
3150 	    check_feature_gpt_level() == GUEST_PGTABLE_5_LEVEL)
3151 		amd_iommu_gpt_level = PAGE_MODE_5_LEVEL;
3152 
3153 	/* Disable any previously enabled IOMMUs */
3154 	if (!is_kdump_kernel() || amd_iommu_disabled)
3155 		disable_iommus();
3156 
3157 	if (amd_iommu_irq_remap)
3158 		amd_iommu_irq_remap = check_ioapic_information();
3159 
3160 	if (amd_iommu_irq_remap) {
3161 		struct amd_iommu_pci_seg *pci_seg;
3162 		/*
3163 		 * Interrupt remapping enabled, create kmem_cache for the
3164 		 * remapping tables.
3165 		 */
3166 		ret = -ENOMEM;
3167 		if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir))
3168 			remap_cache_sz = MAX_IRQS_PER_TABLE * sizeof(u32);
3169 		else
3170 			remap_cache_sz = MAX_IRQS_PER_TABLE * (sizeof(u64) * 2);
3171 		amd_iommu_irq_cache = kmem_cache_create("irq_remap_cache",
3172 							remap_cache_sz,
3173 							DTE_INTTAB_ALIGNMENT,
3174 							0, NULL);
3175 		if (!amd_iommu_irq_cache)
3176 			goto out;
3177 
3178 		for_each_pci_segment(pci_seg) {
3179 			if (alloc_irq_lookup_table(pci_seg))
3180 				goto out;
3181 		}
3182 	}
3183 
3184 	ret = init_memory_definitions(ivrs_base);
3185 	if (ret)
3186 		goto out;
3187 
3188 	/* init the device table */
3189 	init_device_table();
3190 
3191 out:
3192 	/* Don't leak any ACPI memory */
3193 	acpi_put_table(ivrs_base);
3194 
3195 	return ret;
3196 }
3197 
amd_iommu_enable_interrupts(void)3198 static int amd_iommu_enable_interrupts(void)
3199 {
3200 	struct amd_iommu *iommu;
3201 	int ret = 0;
3202 
3203 	for_each_iommu(iommu) {
3204 		ret = iommu_init_irq(iommu);
3205 		if (ret)
3206 			goto out;
3207 	}
3208 
3209 	/*
3210 	 * Interrupt handler is ready to process interrupts. Enable
3211 	 * PPR and GA log interrupt for all IOMMUs.
3212 	 */
3213 	enable_iommus_vapic();
3214 	enable_iommus_v2();
3215 
3216 out:
3217 	return ret;
3218 }
3219 
detect_ivrs(void)3220 static bool __init detect_ivrs(void)
3221 {
3222 	struct acpi_table_header *ivrs_base;
3223 	acpi_status status;
3224 	int i;
3225 
3226 	status = acpi_get_table("IVRS", 0, &ivrs_base);
3227 	if (status == AE_NOT_FOUND)
3228 		return false;
3229 	else if (ACPI_FAILURE(status)) {
3230 		const char *err = acpi_format_exception(status);
3231 		pr_err("IVRS table error: %s\n", err);
3232 		return false;
3233 	}
3234 
3235 	acpi_put_table(ivrs_base);
3236 
3237 	if (amd_iommu_force_enable)
3238 		goto out;
3239 
3240 	/* Don't use IOMMU if there is Stoney Ridge graphics */
3241 	for (i = 0; i < 32; i++) {
3242 		u32 pci_id;
3243 
3244 		pci_id = read_pci_config(0, i, 0, 0);
3245 		if ((pci_id & 0xffff) == 0x1002 && (pci_id >> 16) == 0x98e4) {
3246 			pr_info("Disable IOMMU on Stoney Ridge\n");
3247 			return false;
3248 		}
3249 	}
3250 
3251 out:
3252 	/* Make sure ACS will be enabled during PCI probe */
3253 	pci_request_acs();
3254 
3255 	return true;
3256 }
3257 
3258 /****************************************************************************
3259  *
3260  * AMD IOMMU Initialization State Machine
3261  *
3262  ****************************************************************************/
3263 
state_next(void)3264 static int __init state_next(void)
3265 {
3266 	int ret = 0;
3267 
3268 	switch (init_state) {
3269 	case IOMMU_START_STATE:
3270 		if (!detect_ivrs()) {
3271 			init_state	= IOMMU_NOT_FOUND;
3272 			ret		= -ENODEV;
3273 		} else {
3274 			init_state	= IOMMU_IVRS_DETECTED;
3275 		}
3276 		break;
3277 	case IOMMU_IVRS_DETECTED:
3278 		if (amd_iommu_disabled) {
3279 			init_state = IOMMU_CMDLINE_DISABLED;
3280 			ret = -EINVAL;
3281 		} else {
3282 			ret = early_amd_iommu_init();
3283 			init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;
3284 		}
3285 		break;
3286 	case IOMMU_ACPI_FINISHED:
3287 		early_enable_iommus();
3288 		x86_platform.iommu_shutdown = disable_iommus;
3289 		init_state = IOMMU_ENABLED;
3290 		break;
3291 	case IOMMU_ENABLED:
3292 		register_syscore_ops(&amd_iommu_syscore_ops);
3293 		ret = amd_iommu_init_pci();
3294 		init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT;
3295 		break;
3296 	case IOMMU_PCI_INIT:
3297 		ret = amd_iommu_enable_interrupts();
3298 		init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN;
3299 		break;
3300 	case IOMMU_INTERRUPTS_EN:
3301 		init_state = IOMMU_INITIALIZED;
3302 		break;
3303 	case IOMMU_INITIALIZED:
3304 		/* Nothing to do */
3305 		break;
3306 	case IOMMU_NOT_FOUND:
3307 	case IOMMU_INIT_ERROR:
3308 	case IOMMU_CMDLINE_DISABLED:
3309 		/* Error states => do nothing */
3310 		ret = -EINVAL;
3311 		break;
3312 	default:
3313 		/* Unknown state */
3314 		BUG();
3315 	}
3316 
3317 	if (ret) {
3318 		free_dma_resources();
3319 		if (!irq_remapping_enabled) {
3320 			disable_iommus();
3321 			free_iommu_resources();
3322 		} else {
3323 			struct amd_iommu *iommu;
3324 			struct amd_iommu_pci_seg *pci_seg;
3325 
3326 			for_each_pci_segment(pci_seg)
3327 				uninit_device_table_dma(pci_seg);
3328 
3329 			for_each_iommu(iommu)
3330 				iommu_flush_all_caches(iommu);
3331 		}
3332 	}
3333 	return ret;
3334 }
3335 
iommu_go_to_state(enum iommu_init_state state)3336 static int __init iommu_go_to_state(enum iommu_init_state state)
3337 {
3338 	int ret = -EINVAL;
3339 
3340 	while (init_state != state) {
3341 		if (init_state == IOMMU_NOT_FOUND         ||
3342 		    init_state == IOMMU_INIT_ERROR        ||
3343 		    init_state == IOMMU_CMDLINE_DISABLED)
3344 			break;
3345 		ret = state_next();
3346 	}
3347 
3348 	return ret;
3349 }
3350 
3351 #ifdef CONFIG_IRQ_REMAP
amd_iommu_prepare(void)3352 int __init amd_iommu_prepare(void)
3353 {
3354 	int ret;
3355 
3356 	amd_iommu_irq_remap = true;
3357 
3358 	ret = iommu_go_to_state(IOMMU_ACPI_FINISHED);
3359 	if (ret) {
3360 		amd_iommu_irq_remap = false;
3361 		return ret;
3362 	}
3363 
3364 	return amd_iommu_irq_remap ? 0 : -ENODEV;
3365 }
3366 
amd_iommu_enable(void)3367 int __init amd_iommu_enable(void)
3368 {
3369 	int ret;
3370 
3371 	ret = iommu_go_to_state(IOMMU_ENABLED);
3372 	if (ret)
3373 		return ret;
3374 
3375 	irq_remapping_enabled = 1;
3376 	return amd_iommu_xt_mode;
3377 }
3378 
amd_iommu_disable(void)3379 void amd_iommu_disable(void)
3380 {
3381 	amd_iommu_suspend();
3382 }
3383 
amd_iommu_reenable(int mode)3384 int amd_iommu_reenable(int mode)
3385 {
3386 	amd_iommu_resume();
3387 
3388 	return 0;
3389 }
3390 
amd_iommu_enable_faulting(void)3391 int __init amd_iommu_enable_faulting(void)
3392 {
3393 	/* We enable MSI later when PCI is initialized */
3394 	return 0;
3395 }
3396 #endif
3397 
3398 /*
3399  * This is the core init function for AMD IOMMU hardware in the system.
3400  * This function is called from the generic x86 DMA layer initialization
3401  * code.
3402  */
amd_iommu_init(void)3403 static int __init amd_iommu_init(void)
3404 {
3405 	struct amd_iommu *iommu;
3406 	int ret;
3407 
3408 	ret = iommu_go_to_state(IOMMU_INITIALIZED);
3409 #ifdef CONFIG_GART_IOMMU
3410 	if (ret && list_empty(&amd_iommu_list)) {
3411 		/*
3412 		 * We failed to initialize the AMD IOMMU - try fallback
3413 		 * to GART if possible.
3414 		 */
3415 		gart_iommu_init();
3416 	}
3417 #endif
3418 
3419 	for_each_iommu(iommu)
3420 		amd_iommu_debugfs_setup(iommu);
3421 
3422 	return ret;
3423 }
3424 
amd_iommu_sme_check(void)3425 static bool amd_iommu_sme_check(void)
3426 {
3427 	if (!cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT) ||
3428 	    (boot_cpu_data.x86 != 0x17))
3429 		return true;
3430 
3431 	/* For Fam17h, a specific level of support is required */
3432 	if (boot_cpu_data.microcode >= 0x08001205)
3433 		return true;
3434 
3435 	if ((boot_cpu_data.microcode >= 0x08001126) &&
3436 	    (boot_cpu_data.microcode <= 0x080011ff))
3437 		return true;
3438 
3439 	pr_notice("IOMMU not currently supported when SME is active\n");
3440 
3441 	return false;
3442 }
3443 
3444 /****************************************************************************
3445  *
3446  * Early detect code. This code runs at IOMMU detection time in the DMA
3447  * layer. It just looks if there is an IVRS ACPI table to detect AMD
3448  * IOMMUs
3449  *
3450  ****************************************************************************/
amd_iommu_detect(void)3451 int __init amd_iommu_detect(void)
3452 {
3453 	int ret;
3454 
3455 	if (no_iommu || (iommu_detected && !gart_iommu_aperture))
3456 		return -ENODEV;
3457 
3458 	if (!amd_iommu_sme_check())
3459 		return -ENODEV;
3460 
3461 	ret = iommu_go_to_state(IOMMU_IVRS_DETECTED);
3462 	if (ret)
3463 		return ret;
3464 
3465 	amd_iommu_detected = true;
3466 	iommu_detected = 1;
3467 	x86_init.iommu.iommu_init = amd_iommu_init;
3468 
3469 	return 1;
3470 }
3471 
3472 /****************************************************************************
3473  *
3474  * Parsing functions for the AMD IOMMU specific kernel command line
3475  * options.
3476  *
3477  ****************************************************************************/
3478 
parse_amd_iommu_dump(char * str)3479 static int __init parse_amd_iommu_dump(char *str)
3480 {
3481 	amd_iommu_dump = true;
3482 
3483 	return 1;
3484 }
3485 
parse_amd_iommu_intr(char * str)3486 static int __init parse_amd_iommu_intr(char *str)
3487 {
3488 	for (; *str; ++str) {
3489 		if (strncmp(str, "legacy", 6) == 0) {
3490 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
3491 			break;
3492 		}
3493 		if (strncmp(str, "vapic", 5) == 0) {
3494 			amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC;
3495 			break;
3496 		}
3497 	}
3498 	return 1;
3499 }
3500 
parse_amd_iommu_options(char * str)3501 static int __init parse_amd_iommu_options(char *str)
3502 {
3503 	if (!str)
3504 		return -EINVAL;
3505 
3506 	while (*str) {
3507 		if (strncmp(str, "fullflush", 9) == 0) {
3508 			pr_warn("amd_iommu=fullflush deprecated; use iommu.strict=1 instead\n");
3509 			iommu_set_dma_strict();
3510 		} else if (strncmp(str, "force_enable", 12) == 0) {
3511 			amd_iommu_force_enable = true;
3512 		} else if (strncmp(str, "off", 3) == 0) {
3513 			amd_iommu_disabled = true;
3514 		} else if (strncmp(str, "force_isolation", 15) == 0) {
3515 			amd_iommu_force_isolation = true;
3516 		} else if (strncmp(str, "pgtbl_v1", 8) == 0) {
3517 			amd_iommu_pgtable = AMD_IOMMU_V1;
3518 		} else if (strncmp(str, "pgtbl_v2", 8) == 0) {
3519 			amd_iommu_pgtable = AMD_IOMMU_V2;
3520 		} else if (strncmp(str, "irtcachedis", 11) == 0) {
3521 			amd_iommu_irtcachedis = true;
3522 		} else {
3523 			pr_notice("Unknown option - '%s'\n", str);
3524 		}
3525 
3526 		str += strcspn(str, ",");
3527 		while (*str == ',')
3528 			str++;
3529 	}
3530 
3531 	return 1;
3532 }
3533 
parse_ivrs_ioapic(char * str)3534 static int __init parse_ivrs_ioapic(char *str)
3535 {
3536 	u32 seg = 0, bus, dev, fn;
3537 	int id, i;
3538 	u32 devid;
3539 
3540 	if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3541 	    sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5)
3542 		goto found;
3543 
3544 	if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3545 	    sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) {
3546 		pr_warn("ivrs_ioapic%s option format deprecated; use ivrs_ioapic=%d@%04x:%02x:%02x.%d instead\n",
3547 			str, id, seg, bus, dev, fn);
3548 		goto found;
3549 	}
3550 
3551 	pr_err("Invalid command line: ivrs_ioapic%s\n", str);
3552 	return 1;
3553 
3554 found:
3555 	if (early_ioapic_map_size == EARLY_MAP_SIZE) {
3556 		pr_err("Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n",
3557 			str);
3558 		return 1;
3559 	}
3560 
3561 	devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn);
3562 
3563 	cmdline_maps			= true;
3564 	i				= early_ioapic_map_size++;
3565 	early_ioapic_map[i].id		= id;
3566 	early_ioapic_map[i].devid	= devid;
3567 	early_ioapic_map[i].cmd_line	= true;
3568 
3569 	return 1;
3570 }
3571 
parse_ivrs_hpet(char * str)3572 static int __init parse_ivrs_hpet(char *str)
3573 {
3574 	u32 seg = 0, bus, dev, fn;
3575 	int id, i;
3576 	u32 devid;
3577 
3578 	if (sscanf(str, "=%d@%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3579 	    sscanf(str, "=%d@%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5)
3580 		goto found;
3581 
3582 	if (sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn) == 4 ||
3583 	    sscanf(str, "[%d]=%x:%x:%x.%x", &id, &seg, &bus, &dev, &fn) == 5) {
3584 		pr_warn("ivrs_hpet%s option format deprecated; use ivrs_hpet=%d@%04x:%02x:%02x.%d instead\n",
3585 			str, id, seg, bus, dev, fn);
3586 		goto found;
3587 	}
3588 
3589 	pr_err("Invalid command line: ivrs_hpet%s\n", str);
3590 	return 1;
3591 
3592 found:
3593 	if (early_hpet_map_size == EARLY_MAP_SIZE) {
3594 		pr_err("Early HPET map overflow - ignoring ivrs_hpet%s\n",
3595 			str);
3596 		return 1;
3597 	}
3598 
3599 	devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn);
3600 
3601 	cmdline_maps			= true;
3602 	i				= early_hpet_map_size++;
3603 	early_hpet_map[i].id		= id;
3604 	early_hpet_map[i].devid		= devid;
3605 	early_hpet_map[i].cmd_line	= true;
3606 
3607 	return 1;
3608 }
3609 
3610 #define ACPIID_LEN (ACPIHID_UID_LEN + ACPIHID_HID_LEN)
3611 
parse_ivrs_acpihid(char * str)3612 static int __init parse_ivrs_acpihid(char *str)
3613 {
3614 	u32 seg = 0, bus, dev, fn;
3615 	char *hid, *uid, *p, *addr;
3616 	char acpiid[ACPIID_LEN] = {0};
3617 	int i;
3618 
3619 	addr = strchr(str, '@');
3620 	if (!addr) {
3621 		addr = strchr(str, '=');
3622 		if (!addr)
3623 			goto not_found;
3624 
3625 		++addr;
3626 
3627 		if (strlen(addr) > ACPIID_LEN)
3628 			goto not_found;
3629 
3630 		if (sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid) == 4 ||
3631 		    sscanf(str, "[%x:%x:%x.%x]=%s", &seg, &bus, &dev, &fn, acpiid) == 5) {
3632 			pr_warn("ivrs_acpihid%s option format deprecated; use ivrs_acpihid=%s@%04x:%02x:%02x.%d instead\n",
3633 				str, acpiid, seg, bus, dev, fn);
3634 			goto found;
3635 		}
3636 		goto not_found;
3637 	}
3638 
3639 	/* We have the '@', make it the terminator to get just the acpiid */
3640 	*addr++ = 0;
3641 
3642 	if (strlen(str) > ACPIID_LEN + 1)
3643 		goto not_found;
3644 
3645 	if (sscanf(str, "=%s", acpiid) != 1)
3646 		goto not_found;
3647 
3648 	if (sscanf(addr, "%x:%x.%x", &bus, &dev, &fn) == 3 ||
3649 	    sscanf(addr, "%x:%x:%x.%x", &seg, &bus, &dev, &fn) == 4)
3650 		goto found;
3651 
3652 not_found:
3653 	pr_err("Invalid command line: ivrs_acpihid%s\n", str);
3654 	return 1;
3655 
3656 found:
3657 	p = acpiid;
3658 	hid = strsep(&p, ":");
3659 	uid = p;
3660 
3661 	if (!hid || !(*hid) || !uid) {
3662 		pr_err("Invalid command line: hid or uid\n");
3663 		return 1;
3664 	}
3665 
3666 	/*
3667 	 * Ignore leading zeroes after ':', so e.g., AMDI0095:00
3668 	 * will match AMDI0095:0 in the second strcmp in acpi_dev_hid_uid_match
3669 	 */
3670 	while (*uid == '0' && *(uid + 1))
3671 		uid++;
3672 
3673 	i = early_acpihid_map_size++;
3674 	memcpy(early_acpihid_map[i].hid, hid, strlen(hid));
3675 	memcpy(early_acpihid_map[i].uid, uid, strlen(uid));
3676 	early_acpihid_map[i].devid = IVRS_GET_SBDF_ID(seg, bus, dev, fn);
3677 	early_acpihid_map[i].cmd_line	= true;
3678 
3679 	return 1;
3680 }
3681 
3682 __setup("amd_iommu_dump",	parse_amd_iommu_dump);
3683 __setup("amd_iommu=",		parse_amd_iommu_options);
3684 __setup("amd_iommu_intr=",	parse_amd_iommu_intr);
3685 __setup("ivrs_ioapic",		parse_ivrs_ioapic);
3686 __setup("ivrs_hpet",		parse_ivrs_hpet);
3687 __setup("ivrs_acpihid",		parse_ivrs_acpihid);
3688 
amd_iommu_v2_supported(void)3689 bool amd_iommu_v2_supported(void)
3690 {
3691 	/* CPU page table size should match IOMMU guest page table size */
3692 	if (cpu_feature_enabled(X86_FEATURE_LA57) &&
3693 	    amd_iommu_gpt_level != PAGE_MODE_5_LEVEL)
3694 		return false;
3695 
3696 	/*
3697 	 * Since DTE[Mode]=0 is prohibited on SNP-enabled system
3698 	 * (i.e. EFR[SNPSup]=1), IOMMUv2 page table cannot be used without
3699 	 * setting up IOMMUv1 page table.
3700 	 */
3701 	return amd_iommu_v2_present && !amd_iommu_snp_en;
3702 }
3703 EXPORT_SYMBOL(amd_iommu_v2_supported);
3704 
get_amd_iommu(unsigned int idx)3705 struct amd_iommu *get_amd_iommu(unsigned int idx)
3706 {
3707 	unsigned int i = 0;
3708 	struct amd_iommu *iommu;
3709 
3710 	for_each_iommu(iommu)
3711 		if (i++ == idx)
3712 			return iommu;
3713 	return NULL;
3714 }
3715 
3716 /****************************************************************************
3717  *
3718  * IOMMU EFR Performance Counter support functionality. This code allows
3719  * access to the IOMMU PC functionality.
3720  *
3721  ****************************************************************************/
3722 
amd_iommu_pc_get_max_banks(unsigned int idx)3723 u8 amd_iommu_pc_get_max_banks(unsigned int idx)
3724 {
3725 	struct amd_iommu *iommu = get_amd_iommu(idx);
3726 
3727 	if (iommu)
3728 		return iommu->max_banks;
3729 
3730 	return 0;
3731 }
3732 EXPORT_SYMBOL(amd_iommu_pc_get_max_banks);
3733 
amd_iommu_pc_supported(void)3734 bool amd_iommu_pc_supported(void)
3735 {
3736 	return amd_iommu_pc_present;
3737 }
3738 EXPORT_SYMBOL(amd_iommu_pc_supported);
3739 
amd_iommu_pc_get_max_counters(unsigned int idx)3740 u8 amd_iommu_pc_get_max_counters(unsigned int idx)
3741 {
3742 	struct amd_iommu *iommu = get_amd_iommu(idx);
3743 
3744 	if (iommu)
3745 		return iommu->max_counters;
3746 
3747 	return 0;
3748 }
3749 EXPORT_SYMBOL(amd_iommu_pc_get_max_counters);
3750 
iommu_pc_get_set_reg(struct amd_iommu * iommu,u8 bank,u8 cntr,u8 fxn,u64 * value,bool is_write)3751 static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr,
3752 				u8 fxn, u64 *value, bool is_write)
3753 {
3754 	u32 offset;
3755 	u32 max_offset_lim;
3756 
3757 	/* Make sure the IOMMU PC resource is available */
3758 	if (!amd_iommu_pc_present)
3759 		return -ENODEV;
3760 
3761 	/* Check for valid iommu and pc register indexing */
3762 	if (WARN_ON(!iommu || (fxn > 0x28) || (fxn & 7)))
3763 		return -ENODEV;
3764 
3765 	offset = (u32)(((0x40 | bank) << 12) | (cntr << 8) | fxn);
3766 
3767 	/* Limit the offset to the hw defined mmio region aperture */
3768 	max_offset_lim = (u32)(((0x40 | iommu->max_banks) << 12) |
3769 				(iommu->max_counters << 8) | 0x28);
3770 	if ((offset < MMIO_CNTR_REG_OFFSET) ||
3771 	    (offset > max_offset_lim))
3772 		return -EINVAL;
3773 
3774 	if (is_write) {
3775 		u64 val = *value & GENMASK_ULL(47, 0);
3776 
3777 		writel((u32)val, iommu->mmio_base + offset);
3778 		writel((val >> 32), iommu->mmio_base + offset + 4);
3779 	} else {
3780 		*value = readl(iommu->mmio_base + offset + 4);
3781 		*value <<= 32;
3782 		*value |= readl(iommu->mmio_base + offset);
3783 		*value &= GENMASK_ULL(47, 0);
3784 	}
3785 
3786 	return 0;
3787 }
3788 
amd_iommu_pc_get_reg(struct amd_iommu * iommu,u8 bank,u8 cntr,u8 fxn,u64 * value)3789 int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)
3790 {
3791 	if (!iommu)
3792 		return -EINVAL;
3793 
3794 	return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, false);
3795 }
3796 
amd_iommu_pc_set_reg(struct amd_iommu * iommu,u8 bank,u8 cntr,u8 fxn,u64 * value)3797 int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value)
3798 {
3799 	if (!iommu)
3800 		return -EINVAL;
3801 
3802 	return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, true);
3803 }
3804 
3805 #ifdef CONFIG_AMD_MEM_ENCRYPT
amd_iommu_snp_enable(void)3806 int amd_iommu_snp_enable(void)
3807 {
3808 	/*
3809 	 * The SNP support requires that IOMMU must be enabled, and is
3810 	 * not configured in the passthrough mode.
3811 	 */
3812 	if (no_iommu || iommu_default_passthrough()) {
3813 		pr_err("SNP: IOMMU is disabled or configured in passthrough mode, SNP cannot be supported");
3814 		return -EINVAL;
3815 	}
3816 
3817 	/*
3818 	 * Prevent enabling SNP after IOMMU_ENABLED state because this process
3819 	 * affect how IOMMU driver sets up data structures and configures
3820 	 * IOMMU hardware.
3821 	 */
3822 	if (init_state > IOMMU_ENABLED) {
3823 		pr_err("SNP: Too late to enable SNP for IOMMU.\n");
3824 		return -EINVAL;
3825 	}
3826 
3827 	amd_iommu_snp_en = check_feature_on_all_iommus(FEATURE_SNP);
3828 	if (!amd_iommu_snp_en)
3829 		return -EINVAL;
3830 
3831 	pr_info("SNP enabled\n");
3832 
3833 	/* Enforce IOMMU v1 pagetable when SNP is enabled. */
3834 	if (amd_iommu_pgtable != AMD_IOMMU_V1) {
3835 		pr_warn("Force to using AMD IOMMU v1 page table due to SNP\n");
3836 		amd_iommu_pgtable = AMD_IOMMU_V1;
3837 	}
3838 
3839 	return 0;
3840 }
3841 #endif
3842