1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2015-2016 MediaTek Inc.
4 * Author: Yong Wu <yong.wu@mediatek.com>
5 */
6 #include <linux/bitfield.h>
7 #include <linux/bug.h>
8 #include <linux/clk.h>
9 #include <linux/component.h>
10 #include <linux/device.h>
11 #include <linux/dma-direct.h>
12 #include <linux/err.h>
13 #include <linux/interrupt.h>
14 #include <linux/io.h>
15 #include <linux/iommu.h>
16 #include <linux/iopoll.h>
17 #include <linux/io-pgtable.h>
18 #include <linux/list.h>
19 #include <linux/mfd/syscon.h>
20 #include <linux/module.h>
21 #include <linux/of_address.h>
22 #include <linux/of_irq.h>
23 #include <linux/of_platform.h>
24 #include <linux/pci.h>
25 #include <linux/platform_device.h>
26 #include <linux/pm_runtime.h>
27 #include <linux/regmap.h>
28 #include <linux/slab.h>
29 #include <linux/spinlock.h>
30 #include <linux/soc/mediatek/infracfg.h>
31 #include <asm/barrier.h>
32 #include <soc/mediatek/smi.h>
33
34 #include <dt-bindings/memory/mtk-memory-port.h>
35
36 #define REG_MMU_PT_BASE_ADDR 0x000
37
38 #define REG_MMU_INVALIDATE 0x020
39 #define F_ALL_INVLD 0x2
40 #define F_MMU_INV_RANGE 0x1
41
42 #define REG_MMU_INVLD_START_A 0x024
43 #define REG_MMU_INVLD_END_A 0x028
44
45 #define REG_MMU_INV_SEL_GEN2 0x02c
46 #define REG_MMU_INV_SEL_GEN1 0x038
47 #define F_INVLD_EN0 BIT(0)
48 #define F_INVLD_EN1 BIT(1)
49
50 #define REG_MMU_MISC_CTRL 0x048
51 #define F_MMU_IN_ORDER_WR_EN_MASK (BIT(1) | BIT(17))
52 #define F_MMU_STANDARD_AXI_MODE_MASK (BIT(3) | BIT(19))
53
54 #define REG_MMU_DCM_DIS 0x050
55 #define F_MMU_DCM BIT(8)
56
57 #define REG_MMU_WR_LEN_CTRL 0x054
58 #define F_MMU_WR_THROT_DIS_MASK (BIT(5) | BIT(21))
59
60 #define REG_MMU_CTRL_REG 0x110
61 #define F_MMU_TF_PROT_TO_PROGRAM_ADDR (2 << 4)
62 #define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4)
63 #define F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173 (2 << 5)
64
65 #define REG_MMU_IVRP_PADDR 0x114
66
67 #define REG_MMU_VLD_PA_RNG 0x118
68 #define F_MMU_VLD_PA_RNG(EA, SA) (((EA) << 8) | (SA))
69
70 #define REG_MMU_INT_CONTROL0 0x120
71 #define F_L2_MULIT_HIT_EN BIT(0)
72 #define F_TABLE_WALK_FAULT_INT_EN BIT(1)
73 #define F_PREETCH_FIFO_OVERFLOW_INT_EN BIT(2)
74 #define F_MISS_FIFO_OVERFLOW_INT_EN BIT(3)
75 #define F_PREFETCH_FIFO_ERR_INT_EN BIT(5)
76 #define F_MISS_FIFO_ERR_INT_EN BIT(6)
77 #define F_INT_CLR_BIT BIT(12)
78
79 #define REG_MMU_INT_MAIN_CONTROL 0x124
80 /* mmu0 | mmu1 */
81 #define F_INT_TRANSLATION_FAULT (BIT(0) | BIT(7))
82 #define F_INT_MAIN_MULTI_HIT_FAULT (BIT(1) | BIT(8))
83 #define F_INT_INVALID_PA_FAULT (BIT(2) | BIT(9))
84 #define F_INT_ENTRY_REPLACEMENT_FAULT (BIT(3) | BIT(10))
85 #define F_INT_TLB_MISS_FAULT (BIT(4) | BIT(11))
86 #define F_INT_MISS_TRANSACTION_FIFO_FAULT (BIT(5) | BIT(12))
87 #define F_INT_PRETETCH_TRANSATION_FIFO_FAULT (BIT(6) | BIT(13))
88
89 #define REG_MMU_CPE_DONE 0x12C
90
91 #define REG_MMU_FAULT_ST1 0x134
92 #define F_REG_MMU0_FAULT_MASK GENMASK(6, 0)
93 #define F_REG_MMU1_FAULT_MASK GENMASK(13, 7)
94
95 #define REG_MMU0_FAULT_VA 0x13c
96 #define F_MMU_INVAL_VA_31_12_MASK GENMASK(31, 12)
97 #define F_MMU_INVAL_VA_34_32_MASK GENMASK(11, 9)
98 #define F_MMU_INVAL_PA_34_32_MASK GENMASK(8, 6)
99 #define F_MMU_FAULT_VA_WRITE_BIT BIT(1)
100 #define F_MMU_FAULT_VA_LAYER_BIT BIT(0)
101
102 #define REG_MMU0_INVLD_PA 0x140
103 #define REG_MMU1_FAULT_VA 0x144
104 #define REG_MMU1_INVLD_PA 0x148
105 #define REG_MMU0_INT_ID 0x150
106 #define REG_MMU1_INT_ID 0x154
107 #define F_MMU_INT_ID_COMM_ID(a) (((a) >> 9) & 0x7)
108 #define F_MMU_INT_ID_SUB_COMM_ID(a) (((a) >> 7) & 0x3)
109 #define F_MMU_INT_ID_COMM_ID_EXT(a) (((a) >> 10) & 0x7)
110 #define F_MMU_INT_ID_SUB_COMM_ID_EXT(a) (((a) >> 7) & 0x7)
111 #define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7)
112 #define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f)
113
114 #define MTK_PROTECT_PA_ALIGN 256
115 #define MTK_IOMMU_BANK_SZ 0x1000
116
117 #define PERICFG_IOMMU_1 0x714
118
119 #define HAS_4GB_MODE BIT(0)
120 /* HW will use the EMI clock if there isn't the "bclk". */
121 #define HAS_BCLK BIT(1)
122 #define HAS_VLD_PA_RNG BIT(2)
123 #define RESET_AXI BIT(3)
124 #define OUT_ORDER_WR_EN BIT(4)
125 #define HAS_SUB_COMM_2BITS BIT(5)
126 #define HAS_SUB_COMM_3BITS BIT(6)
127 #define WR_THROT_EN BIT(7)
128 #define HAS_LEGACY_IVRP_PADDR BIT(8)
129 #define IOVA_34_EN BIT(9)
130 #define SHARE_PGTABLE BIT(10) /* 2 HW share pgtable */
131 #define DCM_DISABLE BIT(11)
132 #define STD_AXI_MODE BIT(12) /* For non MM iommu */
133 /* 2 bits: iommu type */
134 #define MTK_IOMMU_TYPE_MM (0x0 << 13)
135 #define MTK_IOMMU_TYPE_INFRA (0x1 << 13)
136 #define MTK_IOMMU_TYPE_MASK (0x3 << 13)
137 /* PM and clock always on. e.g. infra iommu */
138 #define PM_CLK_AO BIT(15)
139 #define IFA_IOMMU_PCIE_SUPPORT BIT(16)
140 #define PGTABLE_PA_35_EN BIT(17)
141 #define TF_PORT_TO_ADDR_MT8173 BIT(18)
142
143 #define MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, mask) \
144 ((((pdata)->flags) & (mask)) == (_x))
145
146 #define MTK_IOMMU_HAS_FLAG(pdata, _x) MTK_IOMMU_HAS_FLAG_MASK(pdata, _x, _x)
147 #define MTK_IOMMU_IS_TYPE(pdata, _x) MTK_IOMMU_HAS_FLAG_MASK(pdata, _x,\
148 MTK_IOMMU_TYPE_MASK)
149
150 #define MTK_INVALID_LARBID MTK_LARB_NR_MAX
151
152 #define MTK_LARB_COM_MAX 8
153 #define MTK_LARB_SUBCOM_MAX 8
154
155 #define MTK_IOMMU_GROUP_MAX 8
156 #define MTK_IOMMU_BANK_MAX 5
157
158 enum mtk_iommu_plat {
159 M4U_MT2712,
160 M4U_MT6779,
161 M4U_MT6795,
162 M4U_MT8167,
163 M4U_MT8173,
164 M4U_MT8183,
165 M4U_MT8186,
166 M4U_MT8192,
167 M4U_MT8195,
168 };
169
170 struct mtk_iommu_iova_region {
171 dma_addr_t iova_base;
172 unsigned long long size;
173 };
174
175 struct mtk_iommu_suspend_reg {
176 u32 misc_ctrl;
177 u32 dcm_dis;
178 u32 ctrl_reg;
179 u32 vld_pa_rng;
180 u32 wr_len_ctrl;
181
182 u32 int_control[MTK_IOMMU_BANK_MAX];
183 u32 int_main_control[MTK_IOMMU_BANK_MAX];
184 u32 ivrp_paddr[MTK_IOMMU_BANK_MAX];
185 };
186
187 struct mtk_iommu_plat_data {
188 enum mtk_iommu_plat m4u_plat;
189 u32 flags;
190 u32 inv_sel_reg;
191
192 char *pericfg_comp_str;
193 struct list_head *hw_list;
194 unsigned int iova_region_nr;
195 const struct mtk_iommu_iova_region *iova_region;
196
197 u8 banks_num;
198 bool banks_enable[MTK_IOMMU_BANK_MAX];
199 unsigned int banks_portmsk[MTK_IOMMU_BANK_MAX];
200 unsigned char larbid_remap[MTK_LARB_COM_MAX][MTK_LARB_SUBCOM_MAX];
201 };
202
203 struct mtk_iommu_bank_data {
204 void __iomem *base;
205 int irq;
206 u8 id;
207 struct device *parent_dev;
208 struct mtk_iommu_data *parent_data;
209 spinlock_t tlb_lock; /* lock for tlb range flush */
210 struct mtk_iommu_domain *m4u_dom; /* Each bank has a domain */
211 };
212
213 struct mtk_iommu_data {
214 struct device *dev;
215 struct clk *bclk;
216 phys_addr_t protect_base; /* protect memory base */
217 struct mtk_iommu_suspend_reg reg;
218 struct iommu_group *m4u_group[MTK_IOMMU_GROUP_MAX];
219 bool enable_4GB;
220
221 struct iommu_device iommu;
222 const struct mtk_iommu_plat_data *plat_data;
223 struct device *smicomm_dev;
224
225 struct mtk_iommu_bank_data *bank;
226
227 struct dma_iommu_mapping *mapping; /* For mtk_iommu_v1.c */
228 struct regmap *pericfg;
229
230 struct mutex mutex; /* Protect m4u_group/m4u_dom above */
231
232 /*
233 * In the sharing pgtable case, list data->list to the global list like m4ulist.
234 * In the non-sharing pgtable case, list data->list to the itself hw_list_head.
235 */
236 struct list_head *hw_list;
237 struct list_head hw_list_head;
238 struct list_head list;
239 struct mtk_smi_larb_iommu larb_imu[MTK_LARB_NR_MAX];
240 };
241
242 struct mtk_iommu_domain {
243 struct io_pgtable_cfg cfg;
244 struct io_pgtable_ops *iop;
245
246 struct mtk_iommu_bank_data *bank;
247 struct iommu_domain domain;
248
249 struct mutex mutex; /* Protect "data" in this structure */
250 };
251
mtk_iommu_bind(struct device * dev)252 static int mtk_iommu_bind(struct device *dev)
253 {
254 struct mtk_iommu_data *data = dev_get_drvdata(dev);
255
256 return component_bind_all(dev, &data->larb_imu);
257 }
258
mtk_iommu_unbind(struct device * dev)259 static void mtk_iommu_unbind(struct device *dev)
260 {
261 struct mtk_iommu_data *data = dev_get_drvdata(dev);
262
263 component_unbind_all(dev, &data->larb_imu);
264 }
265
266 static const struct iommu_ops mtk_iommu_ops;
267
268 static int mtk_iommu_hw_init(const struct mtk_iommu_data *data, unsigned int bankid);
269
270 #define MTK_IOMMU_TLB_ADDR(iova) ({ \
271 dma_addr_t _addr = iova; \
272 ((lower_32_bits(_addr) & GENMASK(31, 12)) | upper_32_bits(_addr));\
273 })
274
275 /*
276 * In M4U 4GB mode, the physical address is remapped as below:
277 *
278 * CPU Physical address:
279 * ====================
280 *
281 * 0 1G 2G 3G 4G 5G
282 * |---A---|---B---|---C---|---D---|---E---|
283 * +--I/O--+------------Memory-------------+
284 *
285 * IOMMU output physical address:
286 * =============================
287 *
288 * 4G 5G 6G 7G 8G
289 * |---E---|---B---|---C---|---D---|
290 * +------------Memory-------------+
291 *
292 * The Region 'A'(I/O) can NOT be mapped by M4U; For Region 'B'/'C'/'D', the
293 * bit32 of the CPU physical address always is needed to set, and for Region
294 * 'E', the CPU physical address keep as is.
295 * Additionally, The iommu consumers always use the CPU phyiscal address.
296 */
297 #define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x140000000UL
298
299 static LIST_HEAD(m4ulist); /* List all the M4U HWs */
300
301 #define for_each_m4u(data, head) list_for_each_entry(data, head, list)
302
303 static const struct mtk_iommu_iova_region single_domain[] = {
304 {.iova_base = 0, .size = SZ_4G},
305 };
306
307 static const struct mtk_iommu_iova_region mt8192_multi_dom[] = {
308 { .iova_base = 0x0, .size = SZ_4G}, /* 0 ~ 4G */
309 #if IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)
310 { .iova_base = SZ_4G, .size = SZ_4G}, /* 4G ~ 8G */
311 { .iova_base = SZ_4G * 2, .size = SZ_4G}, /* 8G ~ 12G */
312 { .iova_base = SZ_4G * 3, .size = SZ_4G}, /* 12G ~ 16G */
313
314 { .iova_base = 0x240000000ULL, .size = 0x4000000}, /* CCU0 */
315 { .iova_base = 0x244000000ULL, .size = 0x4000000}, /* CCU1 */
316 #endif
317 };
318
319 /* If 2 M4U share a domain(use the same hwlist), Put the corresponding info in first data.*/
mtk_iommu_get_frst_data(struct list_head * hwlist)320 static struct mtk_iommu_data *mtk_iommu_get_frst_data(struct list_head *hwlist)
321 {
322 return list_first_entry(hwlist, struct mtk_iommu_data, list);
323 }
324
to_mtk_domain(struct iommu_domain * dom)325 static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom)
326 {
327 return container_of(dom, struct mtk_iommu_domain, domain);
328 }
329
mtk_iommu_tlb_flush_all(struct mtk_iommu_data * data)330 static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data)
331 {
332 /* Tlb flush all always is in bank0. */
333 struct mtk_iommu_bank_data *bank = &data->bank[0];
334 void __iomem *base = bank->base;
335 unsigned long flags;
336
337 spin_lock_irqsave(&bank->tlb_lock, flags);
338 writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, base + data->plat_data->inv_sel_reg);
339 writel_relaxed(F_ALL_INVLD, base + REG_MMU_INVALIDATE);
340 wmb(); /* Make sure the tlb flush all done */
341 spin_unlock_irqrestore(&bank->tlb_lock, flags);
342 }
343
mtk_iommu_tlb_flush_range_sync(unsigned long iova,size_t size,struct mtk_iommu_bank_data * bank)344 static void mtk_iommu_tlb_flush_range_sync(unsigned long iova, size_t size,
345 struct mtk_iommu_bank_data *bank)
346 {
347 struct list_head *head = bank->parent_data->hw_list;
348 struct mtk_iommu_bank_data *curbank;
349 struct mtk_iommu_data *data;
350 bool check_pm_status;
351 unsigned long flags;
352 void __iomem *base;
353 int ret;
354 u32 tmp;
355
356 for_each_m4u(data, head) {
357 /*
358 * To avoid resume the iommu device frequently when the iommu device
359 * is not active, it doesn't always call pm_runtime_get here, then tlb
360 * flush depends on the tlb flush all in the runtime resume.
361 *
362 * There are 2 special cases:
363 *
364 * Case1: The iommu dev doesn't have power domain but has bclk. This case
365 * should also avoid the tlb flush while the dev is not active to mute
366 * the tlb timeout log. like mt8173.
367 *
368 * Case2: The power/clock of infra iommu is always on, and it doesn't
369 * have the device link with the master devices. This case should avoid
370 * the PM status check.
371 */
372 check_pm_status = !MTK_IOMMU_HAS_FLAG(data->plat_data, PM_CLK_AO);
373
374 if (check_pm_status) {
375 if (pm_runtime_get_if_in_use(data->dev) <= 0)
376 continue;
377 }
378
379 curbank = &data->bank[bank->id];
380 base = curbank->base;
381
382 spin_lock_irqsave(&curbank->tlb_lock, flags);
383 writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0,
384 base + data->plat_data->inv_sel_reg);
385
386 writel_relaxed(MTK_IOMMU_TLB_ADDR(iova), base + REG_MMU_INVLD_START_A);
387 writel_relaxed(MTK_IOMMU_TLB_ADDR(iova + size - 1),
388 base + REG_MMU_INVLD_END_A);
389 writel_relaxed(F_MMU_INV_RANGE, base + REG_MMU_INVALIDATE);
390
391 /* tlb sync */
392 ret = readl_poll_timeout_atomic(base + REG_MMU_CPE_DONE,
393 tmp, tmp != 0, 10, 1000);
394
395 /* Clear the CPE status */
396 writel_relaxed(0, base + REG_MMU_CPE_DONE);
397 spin_unlock_irqrestore(&curbank->tlb_lock, flags);
398
399 if (ret) {
400 dev_warn(data->dev,
401 "Partial TLB flush timed out, falling back to full flush\n");
402 mtk_iommu_tlb_flush_all(data);
403 }
404
405 if (check_pm_status)
406 pm_runtime_put(data->dev);
407 }
408 }
409
mtk_iommu_isr(int irq,void * dev_id)410 static irqreturn_t mtk_iommu_isr(int irq, void *dev_id)
411 {
412 struct mtk_iommu_bank_data *bank = dev_id;
413 struct mtk_iommu_data *data = bank->parent_data;
414 struct mtk_iommu_domain *dom = bank->m4u_dom;
415 unsigned int fault_larb = MTK_INVALID_LARBID, fault_port = 0, sub_comm = 0;
416 u32 int_state, regval, va34_32, pa34_32;
417 const struct mtk_iommu_plat_data *plat_data = data->plat_data;
418 void __iomem *base = bank->base;
419 u64 fault_iova, fault_pa;
420 bool layer, write;
421
422 /* Read error info from registers */
423 int_state = readl_relaxed(base + REG_MMU_FAULT_ST1);
424 if (int_state & F_REG_MMU0_FAULT_MASK) {
425 regval = readl_relaxed(base + REG_MMU0_INT_ID);
426 fault_iova = readl_relaxed(base + REG_MMU0_FAULT_VA);
427 fault_pa = readl_relaxed(base + REG_MMU0_INVLD_PA);
428 } else {
429 regval = readl_relaxed(base + REG_MMU1_INT_ID);
430 fault_iova = readl_relaxed(base + REG_MMU1_FAULT_VA);
431 fault_pa = readl_relaxed(base + REG_MMU1_INVLD_PA);
432 }
433 layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT;
434 write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT;
435 if (MTK_IOMMU_HAS_FLAG(plat_data, IOVA_34_EN)) {
436 va34_32 = FIELD_GET(F_MMU_INVAL_VA_34_32_MASK, fault_iova);
437 fault_iova = fault_iova & F_MMU_INVAL_VA_31_12_MASK;
438 fault_iova |= (u64)va34_32 << 32;
439 }
440 pa34_32 = FIELD_GET(F_MMU_INVAL_PA_34_32_MASK, fault_iova);
441 fault_pa |= (u64)pa34_32 << 32;
442
443 if (MTK_IOMMU_IS_TYPE(plat_data, MTK_IOMMU_TYPE_MM)) {
444 fault_port = F_MMU_INT_ID_PORT_ID(regval);
445 if (MTK_IOMMU_HAS_FLAG(plat_data, HAS_SUB_COMM_2BITS)) {
446 fault_larb = F_MMU_INT_ID_COMM_ID(regval);
447 sub_comm = F_MMU_INT_ID_SUB_COMM_ID(regval);
448 } else if (MTK_IOMMU_HAS_FLAG(plat_data, HAS_SUB_COMM_3BITS)) {
449 fault_larb = F_MMU_INT_ID_COMM_ID_EXT(regval);
450 sub_comm = F_MMU_INT_ID_SUB_COMM_ID_EXT(regval);
451 } else {
452 fault_larb = F_MMU_INT_ID_LARB_ID(regval);
453 }
454 fault_larb = data->plat_data->larbid_remap[fault_larb][sub_comm];
455 }
456
457 if (!dom || report_iommu_fault(&dom->domain, bank->parent_dev, fault_iova,
458 write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) {
459 dev_err_ratelimited(
460 bank->parent_dev,
461 "fault type=0x%x iova=0x%llx pa=0x%llx master=0x%x(larb=%d port=%d) layer=%d %s\n",
462 int_state, fault_iova, fault_pa, regval, fault_larb, fault_port,
463 layer, write ? "write" : "read");
464 }
465
466 /* Interrupt clear */
467 regval = readl_relaxed(base + REG_MMU_INT_CONTROL0);
468 regval |= F_INT_CLR_BIT;
469 writel_relaxed(regval, base + REG_MMU_INT_CONTROL0);
470
471 mtk_iommu_tlb_flush_all(data);
472
473 return IRQ_HANDLED;
474 }
475
mtk_iommu_get_bank_id(struct device * dev,const struct mtk_iommu_plat_data * plat_data)476 static unsigned int mtk_iommu_get_bank_id(struct device *dev,
477 const struct mtk_iommu_plat_data *plat_data)
478 {
479 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
480 unsigned int i, portmsk = 0, bankid = 0;
481
482 if (plat_data->banks_num == 1)
483 return bankid;
484
485 for (i = 0; i < fwspec->num_ids; i++)
486 portmsk |= BIT(MTK_M4U_TO_PORT(fwspec->ids[i]));
487
488 for (i = 0; i < plat_data->banks_num && i < MTK_IOMMU_BANK_MAX; i++) {
489 if (!plat_data->banks_enable[i])
490 continue;
491
492 if (portmsk & plat_data->banks_portmsk[i]) {
493 bankid = i;
494 break;
495 }
496 }
497 return bankid; /* default is 0 */
498 }
499
mtk_iommu_get_iova_region_id(struct device * dev,const struct mtk_iommu_plat_data * plat_data)500 static int mtk_iommu_get_iova_region_id(struct device *dev,
501 const struct mtk_iommu_plat_data *plat_data)
502 {
503 const struct mtk_iommu_iova_region *rgn = plat_data->iova_region;
504 const struct bus_dma_region *dma_rgn = dev->dma_range_map;
505 int i, candidate = -1;
506 dma_addr_t dma_end;
507
508 if (!dma_rgn || plat_data->iova_region_nr == 1)
509 return 0;
510
511 dma_end = dma_rgn->dma_start + dma_rgn->size - 1;
512 for (i = 0; i < plat_data->iova_region_nr; i++, rgn++) {
513 /* Best fit. */
514 if (dma_rgn->dma_start == rgn->iova_base &&
515 dma_end == rgn->iova_base + rgn->size - 1)
516 return i;
517 /* ok if it is inside this region. */
518 if (dma_rgn->dma_start >= rgn->iova_base &&
519 dma_end < rgn->iova_base + rgn->size)
520 candidate = i;
521 }
522
523 if (candidate >= 0)
524 return candidate;
525 dev_err(dev, "Can NOT find the iommu domain id(%pad 0x%llx).\n",
526 &dma_rgn->dma_start, dma_rgn->size);
527 return -EINVAL;
528 }
529
mtk_iommu_config(struct mtk_iommu_data * data,struct device * dev,bool enable,unsigned int regionid)530 static int mtk_iommu_config(struct mtk_iommu_data *data, struct device *dev,
531 bool enable, unsigned int regionid)
532 {
533 struct mtk_smi_larb_iommu *larb_mmu;
534 unsigned int larbid, portid;
535 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
536 const struct mtk_iommu_iova_region *region;
537 u32 peri_mmuen, peri_mmuen_msk;
538 int i, ret = 0;
539
540 for (i = 0; i < fwspec->num_ids; ++i) {
541 larbid = MTK_M4U_TO_LARB(fwspec->ids[i]);
542 portid = MTK_M4U_TO_PORT(fwspec->ids[i]);
543
544 if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) {
545 larb_mmu = &data->larb_imu[larbid];
546
547 region = data->plat_data->iova_region + regionid;
548 larb_mmu->bank[portid] = upper_32_bits(region->iova_base);
549
550 dev_dbg(dev, "%s iommu for larb(%s) port %d region %d rgn-bank %d.\n",
551 enable ? "enable" : "disable", dev_name(larb_mmu->dev),
552 portid, regionid, larb_mmu->bank[portid]);
553
554 if (enable)
555 larb_mmu->mmu |= MTK_SMI_MMU_EN(portid);
556 else
557 larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid);
558 } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA)) {
559 peri_mmuen_msk = BIT(portid);
560 /* PCI dev has only one output id, enable the next writing bit for PCIe */
561 if (dev_is_pci(dev))
562 peri_mmuen_msk |= BIT(portid + 1);
563
564 peri_mmuen = enable ? peri_mmuen_msk : 0;
565 ret = regmap_update_bits(data->pericfg, PERICFG_IOMMU_1,
566 peri_mmuen_msk, peri_mmuen);
567 if (ret)
568 dev_err(dev, "%s iommu(%s) inframaster 0x%x fail(%d).\n",
569 enable ? "enable" : "disable",
570 dev_name(data->dev), peri_mmuen_msk, ret);
571 }
572 }
573 return ret;
574 }
575
mtk_iommu_domain_finalise(struct mtk_iommu_domain * dom,struct mtk_iommu_data * data,unsigned int region_id)576 static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom,
577 struct mtk_iommu_data *data,
578 unsigned int region_id)
579 {
580 const struct mtk_iommu_iova_region *region;
581 struct mtk_iommu_domain *m4u_dom;
582
583 /* Always use bank0 in sharing pgtable case */
584 m4u_dom = data->bank[0].m4u_dom;
585 if (m4u_dom) {
586 dom->iop = m4u_dom->iop;
587 dom->cfg = m4u_dom->cfg;
588 dom->domain.pgsize_bitmap = m4u_dom->cfg.pgsize_bitmap;
589 goto update_iova_region;
590 }
591
592 dom->cfg = (struct io_pgtable_cfg) {
593 .quirks = IO_PGTABLE_QUIRK_ARM_NS |
594 IO_PGTABLE_QUIRK_NO_PERMS |
595 IO_PGTABLE_QUIRK_ARM_MTK_EXT,
596 .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap,
597 .ias = MTK_IOMMU_HAS_FLAG(data->plat_data, IOVA_34_EN) ? 34 : 32,
598 .iommu_dev = data->dev,
599 };
600
601 if (MTK_IOMMU_HAS_FLAG(data->plat_data, PGTABLE_PA_35_EN))
602 dom->cfg.quirks |= IO_PGTABLE_QUIRK_ARM_MTK_TTBR_EXT;
603
604 if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE))
605 dom->cfg.oas = data->enable_4GB ? 33 : 32;
606 else
607 dom->cfg.oas = 35;
608
609 dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data);
610 if (!dom->iop) {
611 dev_err(data->dev, "Failed to alloc io pgtable\n");
612 return -EINVAL;
613 }
614
615 /* Update our support page sizes bitmap */
616 dom->domain.pgsize_bitmap = dom->cfg.pgsize_bitmap;
617
618 update_iova_region:
619 /* Update the iova region for this domain */
620 region = data->plat_data->iova_region + region_id;
621 dom->domain.geometry.aperture_start = region->iova_base;
622 dom->domain.geometry.aperture_end = region->iova_base + region->size - 1;
623 dom->domain.geometry.force_aperture = true;
624 return 0;
625 }
626
mtk_iommu_domain_alloc(unsigned type)627 static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type)
628 {
629 struct mtk_iommu_domain *dom;
630
631 if (type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_UNMANAGED)
632 return NULL;
633
634 dom = kzalloc(sizeof(*dom), GFP_KERNEL);
635 if (!dom)
636 return NULL;
637 mutex_init(&dom->mutex);
638
639 return &dom->domain;
640 }
641
mtk_iommu_domain_free(struct iommu_domain * domain)642 static void mtk_iommu_domain_free(struct iommu_domain *domain)
643 {
644 kfree(to_mtk_domain(domain));
645 }
646
mtk_iommu_attach_device(struct iommu_domain * domain,struct device * dev)647 static int mtk_iommu_attach_device(struct iommu_domain *domain,
648 struct device *dev)
649 {
650 struct mtk_iommu_data *data = dev_iommu_priv_get(dev), *frstdata;
651 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
652 struct list_head *hw_list = data->hw_list;
653 struct device *m4udev = data->dev;
654 struct mtk_iommu_bank_data *bank;
655 unsigned int bankid;
656 int ret, region_id;
657
658 region_id = mtk_iommu_get_iova_region_id(dev, data->plat_data);
659 if (region_id < 0)
660 return region_id;
661
662 bankid = mtk_iommu_get_bank_id(dev, data->plat_data);
663 mutex_lock(&dom->mutex);
664 if (!dom->bank) {
665 /* Data is in the frstdata in sharing pgtable case. */
666 frstdata = mtk_iommu_get_frst_data(hw_list);
667
668 ret = mtk_iommu_domain_finalise(dom, frstdata, region_id);
669 if (ret) {
670 mutex_unlock(&dom->mutex);
671 return -ENODEV;
672 }
673 dom->bank = &data->bank[bankid];
674 }
675 mutex_unlock(&dom->mutex);
676
677 mutex_lock(&data->mutex);
678 bank = &data->bank[bankid];
679 if (!bank->m4u_dom) { /* Initialize the M4U HW for each a BANK */
680 ret = pm_runtime_resume_and_get(m4udev);
681 if (ret < 0) {
682 dev_err(m4udev, "pm get fail(%d) in attach.\n", ret);
683 goto err_unlock;
684 }
685
686 ret = mtk_iommu_hw_init(data, bankid);
687 if (ret) {
688 pm_runtime_put(m4udev);
689 goto err_unlock;
690 }
691 bank->m4u_dom = dom;
692 writel(dom->cfg.arm_v7s_cfg.ttbr, bank->base + REG_MMU_PT_BASE_ADDR);
693
694 pm_runtime_put(m4udev);
695 }
696 mutex_unlock(&data->mutex);
697
698 return mtk_iommu_config(data, dev, true, region_id);
699
700 err_unlock:
701 mutex_unlock(&data->mutex);
702 return ret;
703 }
704
mtk_iommu_detach_device(struct iommu_domain * domain,struct device * dev)705 static void mtk_iommu_detach_device(struct iommu_domain *domain,
706 struct device *dev)
707 {
708 struct mtk_iommu_data *data = dev_iommu_priv_get(dev);
709
710 mtk_iommu_config(data, dev, false, 0);
711 }
712
mtk_iommu_map(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t size,int prot,gfp_t gfp)713 static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
714 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
715 {
716 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
717
718 /* The "4GB mode" M4U physically can not use the lower remap of Dram. */
719 if (dom->bank->parent_data->enable_4GB)
720 paddr |= BIT_ULL(32);
721
722 /* Synchronize with the tlb_lock */
723 return dom->iop->map(dom->iop, iova, paddr, size, prot, gfp);
724 }
725
mtk_iommu_unmap(struct iommu_domain * domain,unsigned long iova,size_t size,struct iommu_iotlb_gather * gather)726 static size_t mtk_iommu_unmap(struct iommu_domain *domain,
727 unsigned long iova, size_t size,
728 struct iommu_iotlb_gather *gather)
729 {
730 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
731
732 iommu_iotlb_gather_add_range(gather, iova, size);
733 return dom->iop->unmap(dom->iop, iova, size, gather);
734 }
735
mtk_iommu_flush_iotlb_all(struct iommu_domain * domain)736 static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain)
737 {
738 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
739
740 mtk_iommu_tlb_flush_all(dom->bank->parent_data);
741 }
742
mtk_iommu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)743 static void mtk_iommu_iotlb_sync(struct iommu_domain *domain,
744 struct iommu_iotlb_gather *gather)
745 {
746 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
747 size_t length = gather->end - gather->start + 1;
748
749 mtk_iommu_tlb_flush_range_sync(gather->start, length, dom->bank);
750 }
751
mtk_iommu_sync_map(struct iommu_domain * domain,unsigned long iova,size_t size)752 static void mtk_iommu_sync_map(struct iommu_domain *domain, unsigned long iova,
753 size_t size)
754 {
755 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
756
757 mtk_iommu_tlb_flush_range_sync(iova, size, dom->bank);
758 }
759
mtk_iommu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)760 static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
761 dma_addr_t iova)
762 {
763 struct mtk_iommu_domain *dom = to_mtk_domain(domain);
764 phys_addr_t pa;
765
766 pa = dom->iop->iova_to_phys(dom->iop, iova);
767 if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT) &&
768 dom->bank->parent_data->enable_4GB &&
769 pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE)
770 pa &= ~BIT_ULL(32);
771
772 return pa;
773 }
774
mtk_iommu_probe_device(struct device * dev)775 static struct iommu_device *mtk_iommu_probe_device(struct device *dev)
776 {
777 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
778 struct mtk_iommu_data *data;
779 struct device_link *link;
780 struct device *larbdev;
781 unsigned int larbid, larbidx, i;
782
783 if (!fwspec || fwspec->ops != &mtk_iommu_ops)
784 return ERR_PTR(-ENODEV); /* Not a iommu client device */
785
786 data = dev_iommu_priv_get(dev);
787
788 if (!MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM))
789 return &data->iommu;
790
791 /*
792 * Link the consumer device with the smi-larb device(supplier).
793 * The device that connects with each a larb is a independent HW.
794 * All the ports in each a device should be in the same larbs.
795 */
796 larbid = MTK_M4U_TO_LARB(fwspec->ids[0]);
797 if (larbid >= MTK_LARB_NR_MAX)
798 return ERR_PTR(-EINVAL);
799
800 for (i = 1; i < fwspec->num_ids; i++) {
801 larbidx = MTK_M4U_TO_LARB(fwspec->ids[i]);
802 if (larbid != larbidx) {
803 dev_err(dev, "Can only use one larb. Fail@larb%d-%d.\n",
804 larbid, larbidx);
805 return ERR_PTR(-EINVAL);
806 }
807 }
808 larbdev = data->larb_imu[larbid].dev;
809 if (!larbdev)
810 return ERR_PTR(-EINVAL);
811
812 link = device_link_add(dev, larbdev,
813 DL_FLAG_PM_RUNTIME | DL_FLAG_STATELESS);
814 if (!link)
815 dev_err(dev, "Unable to link %s\n", dev_name(larbdev));
816 return &data->iommu;
817 }
818
mtk_iommu_release_device(struct device * dev)819 static void mtk_iommu_release_device(struct device *dev)
820 {
821 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
822 struct mtk_iommu_data *data;
823 struct device *larbdev;
824 unsigned int larbid;
825
826 data = dev_iommu_priv_get(dev);
827 if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) {
828 larbid = MTK_M4U_TO_LARB(fwspec->ids[0]);
829 larbdev = data->larb_imu[larbid].dev;
830 device_link_remove(dev, larbdev);
831 }
832 }
833
mtk_iommu_get_group_id(struct device * dev,const struct mtk_iommu_plat_data * plat_data)834 static int mtk_iommu_get_group_id(struct device *dev, const struct mtk_iommu_plat_data *plat_data)
835 {
836 unsigned int bankid;
837
838 /*
839 * If the bank function is enabled, each bank is a iommu group/domain.
840 * Otherwise, each iova region is a iommu group/domain.
841 */
842 bankid = mtk_iommu_get_bank_id(dev, plat_data);
843 if (bankid)
844 return bankid;
845
846 return mtk_iommu_get_iova_region_id(dev, plat_data);
847 }
848
mtk_iommu_device_group(struct device * dev)849 static struct iommu_group *mtk_iommu_device_group(struct device *dev)
850 {
851 struct mtk_iommu_data *c_data = dev_iommu_priv_get(dev), *data;
852 struct list_head *hw_list = c_data->hw_list;
853 struct iommu_group *group;
854 int groupid;
855
856 data = mtk_iommu_get_frst_data(hw_list);
857 if (!data)
858 return ERR_PTR(-ENODEV);
859
860 groupid = mtk_iommu_get_group_id(dev, data->plat_data);
861 if (groupid < 0)
862 return ERR_PTR(groupid);
863
864 mutex_lock(&data->mutex);
865 group = data->m4u_group[groupid];
866 if (!group) {
867 group = iommu_group_alloc();
868 if (!IS_ERR(group))
869 data->m4u_group[groupid] = group;
870 } else {
871 iommu_group_ref_get(group);
872 }
873 mutex_unlock(&data->mutex);
874 return group;
875 }
876
mtk_iommu_of_xlate(struct device * dev,struct of_phandle_args * args)877 static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args)
878 {
879 struct platform_device *m4updev;
880
881 if (args->args_count != 1) {
882 dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n",
883 args->args_count);
884 return -EINVAL;
885 }
886
887 if (!dev_iommu_priv_get(dev)) {
888 /* Get the m4u device */
889 m4updev = of_find_device_by_node(args->np);
890 if (WARN_ON(!m4updev))
891 return -EINVAL;
892
893 dev_iommu_priv_set(dev, platform_get_drvdata(m4updev));
894 }
895
896 return iommu_fwspec_add_ids(dev, args->args, 1);
897 }
898
mtk_iommu_get_resv_regions(struct device * dev,struct list_head * head)899 static void mtk_iommu_get_resv_regions(struct device *dev,
900 struct list_head *head)
901 {
902 struct mtk_iommu_data *data = dev_iommu_priv_get(dev);
903 unsigned int regionid = mtk_iommu_get_iova_region_id(dev, data->plat_data), i;
904 const struct mtk_iommu_iova_region *resv, *curdom;
905 struct iommu_resv_region *region;
906 int prot = IOMMU_WRITE | IOMMU_READ;
907
908 if ((int)regionid < 0)
909 return;
910 curdom = data->plat_data->iova_region + regionid;
911 for (i = 0; i < data->plat_data->iova_region_nr; i++) {
912 resv = data->plat_data->iova_region + i;
913
914 /* Only reserve when the region is inside the current domain */
915 if (resv->iova_base <= curdom->iova_base ||
916 resv->iova_base + resv->size >= curdom->iova_base + curdom->size)
917 continue;
918
919 region = iommu_alloc_resv_region(resv->iova_base, resv->size,
920 prot, IOMMU_RESV_RESERVED,
921 GFP_KERNEL);
922 if (!region)
923 return;
924
925 list_add_tail(®ion->list, head);
926 }
927 }
928
929 static const struct iommu_ops mtk_iommu_ops = {
930 .domain_alloc = mtk_iommu_domain_alloc,
931 .probe_device = mtk_iommu_probe_device,
932 .release_device = mtk_iommu_release_device,
933 .device_group = mtk_iommu_device_group,
934 .of_xlate = mtk_iommu_of_xlate,
935 .get_resv_regions = mtk_iommu_get_resv_regions,
936 .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M,
937 .owner = THIS_MODULE,
938 .default_domain_ops = &(const struct iommu_domain_ops) {
939 .attach_dev = mtk_iommu_attach_device,
940 .detach_dev = mtk_iommu_detach_device,
941 .map = mtk_iommu_map,
942 .unmap = mtk_iommu_unmap,
943 .flush_iotlb_all = mtk_iommu_flush_iotlb_all,
944 .iotlb_sync = mtk_iommu_iotlb_sync,
945 .iotlb_sync_map = mtk_iommu_sync_map,
946 .iova_to_phys = mtk_iommu_iova_to_phys,
947 .free = mtk_iommu_domain_free,
948 }
949 };
950
mtk_iommu_hw_init(const struct mtk_iommu_data * data,unsigned int bankid)951 static int mtk_iommu_hw_init(const struct mtk_iommu_data *data, unsigned int bankid)
952 {
953 const struct mtk_iommu_bank_data *bankx = &data->bank[bankid];
954 const struct mtk_iommu_bank_data *bank0 = &data->bank[0];
955 u32 regval;
956
957 /*
958 * Global control settings are in bank0. May re-init these global registers
959 * since no sure if there is bank0 consumers.
960 */
961 if (MTK_IOMMU_HAS_FLAG(data->plat_data, TF_PORT_TO_ADDR_MT8173)) {
962 regval = F_MMU_PREFETCH_RT_REPLACE_MOD |
963 F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173;
964 } else {
965 regval = readl_relaxed(bank0->base + REG_MMU_CTRL_REG);
966 regval |= F_MMU_TF_PROT_TO_PROGRAM_ADDR;
967 }
968 writel_relaxed(regval, bank0->base + REG_MMU_CTRL_REG);
969
970 if (data->enable_4GB &&
971 MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_VLD_PA_RNG)) {
972 /*
973 * If 4GB mode is enabled, the validate PA range is from
974 * 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30].
975 */
976 regval = F_MMU_VLD_PA_RNG(7, 4);
977 writel_relaxed(regval, bank0->base + REG_MMU_VLD_PA_RNG);
978 }
979 if (MTK_IOMMU_HAS_FLAG(data->plat_data, DCM_DISABLE))
980 writel_relaxed(F_MMU_DCM, bank0->base + REG_MMU_DCM_DIS);
981 else
982 writel_relaxed(0, bank0->base + REG_MMU_DCM_DIS);
983
984 if (MTK_IOMMU_HAS_FLAG(data->plat_data, WR_THROT_EN)) {
985 /* write command throttling mode */
986 regval = readl_relaxed(bank0->base + REG_MMU_WR_LEN_CTRL);
987 regval &= ~F_MMU_WR_THROT_DIS_MASK;
988 writel_relaxed(regval, bank0->base + REG_MMU_WR_LEN_CTRL);
989 }
990
991 if (MTK_IOMMU_HAS_FLAG(data->plat_data, RESET_AXI)) {
992 /* The register is called STANDARD_AXI_MODE in this case */
993 regval = 0;
994 } else {
995 regval = readl_relaxed(bank0->base + REG_MMU_MISC_CTRL);
996 if (!MTK_IOMMU_HAS_FLAG(data->plat_data, STD_AXI_MODE))
997 regval &= ~F_MMU_STANDARD_AXI_MODE_MASK;
998 if (MTK_IOMMU_HAS_FLAG(data->plat_data, OUT_ORDER_WR_EN))
999 regval &= ~F_MMU_IN_ORDER_WR_EN_MASK;
1000 }
1001 writel_relaxed(regval, bank0->base + REG_MMU_MISC_CTRL);
1002
1003 /* Independent settings for each bank */
1004 regval = F_L2_MULIT_HIT_EN |
1005 F_TABLE_WALK_FAULT_INT_EN |
1006 F_PREETCH_FIFO_OVERFLOW_INT_EN |
1007 F_MISS_FIFO_OVERFLOW_INT_EN |
1008 F_PREFETCH_FIFO_ERR_INT_EN |
1009 F_MISS_FIFO_ERR_INT_EN;
1010 writel_relaxed(regval, bankx->base + REG_MMU_INT_CONTROL0);
1011
1012 regval = F_INT_TRANSLATION_FAULT |
1013 F_INT_MAIN_MULTI_HIT_FAULT |
1014 F_INT_INVALID_PA_FAULT |
1015 F_INT_ENTRY_REPLACEMENT_FAULT |
1016 F_INT_TLB_MISS_FAULT |
1017 F_INT_MISS_TRANSACTION_FIFO_FAULT |
1018 F_INT_PRETETCH_TRANSATION_FIFO_FAULT;
1019 writel_relaxed(regval, bankx->base + REG_MMU_INT_MAIN_CONTROL);
1020
1021 if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_LEGACY_IVRP_PADDR))
1022 regval = (data->protect_base >> 1) | (data->enable_4GB << 31);
1023 else
1024 regval = lower_32_bits(data->protect_base) |
1025 upper_32_bits(data->protect_base);
1026 writel_relaxed(regval, bankx->base + REG_MMU_IVRP_PADDR);
1027
1028 if (devm_request_irq(bankx->parent_dev, bankx->irq, mtk_iommu_isr, 0,
1029 dev_name(bankx->parent_dev), (void *)bankx)) {
1030 writel_relaxed(0, bankx->base + REG_MMU_PT_BASE_ADDR);
1031 dev_err(bankx->parent_dev, "Failed @ IRQ-%d Request\n", bankx->irq);
1032 return -ENODEV;
1033 }
1034
1035 return 0;
1036 }
1037
1038 static const struct component_master_ops mtk_iommu_com_ops = {
1039 .bind = mtk_iommu_bind,
1040 .unbind = mtk_iommu_unbind,
1041 };
1042
mtk_iommu_mm_dts_parse(struct device * dev,struct component_match ** match,struct mtk_iommu_data * data)1043 static int mtk_iommu_mm_dts_parse(struct device *dev, struct component_match **match,
1044 struct mtk_iommu_data *data)
1045 {
1046 struct device_node *larbnode, *smicomm_node, *smi_subcomm_node;
1047 struct platform_device *plarbdev, *pcommdev;
1048 struct device_link *link;
1049 int i, larb_nr, ret;
1050
1051 larb_nr = of_count_phandle_with_args(dev->of_node, "mediatek,larbs", NULL);
1052 if (larb_nr < 0)
1053 return larb_nr;
1054 if (larb_nr == 0 || larb_nr > MTK_LARB_NR_MAX)
1055 return -EINVAL;
1056
1057 for (i = 0; i < larb_nr; i++) {
1058 u32 id;
1059
1060 larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i);
1061 if (!larbnode) {
1062 ret = -EINVAL;
1063 goto err_larbdev_put;
1064 }
1065
1066 if (!of_device_is_available(larbnode)) {
1067 of_node_put(larbnode);
1068 continue;
1069 }
1070
1071 ret = of_property_read_u32(larbnode, "mediatek,larb-id", &id);
1072 if (ret)/* The id is consecutive if there is no this property */
1073 id = i;
1074 if (id >= MTK_LARB_NR_MAX) {
1075 of_node_put(larbnode);
1076 ret = -EINVAL;
1077 goto err_larbdev_put;
1078 }
1079
1080 plarbdev = of_find_device_by_node(larbnode);
1081 of_node_put(larbnode);
1082 if (!plarbdev) {
1083 ret = -ENODEV;
1084 goto err_larbdev_put;
1085 }
1086 if (data->larb_imu[id].dev) {
1087 platform_device_put(plarbdev);
1088 ret = -EEXIST;
1089 goto err_larbdev_put;
1090 }
1091 data->larb_imu[id].dev = &plarbdev->dev;
1092
1093 if (!plarbdev->dev.driver) {
1094 ret = -EPROBE_DEFER;
1095 goto err_larbdev_put;
1096 }
1097
1098 component_match_add(dev, match, component_compare_dev, &plarbdev->dev);
1099 platform_device_put(plarbdev);
1100 }
1101
1102 /* Get smi-(sub)-common dev from the last larb. */
1103 smi_subcomm_node = of_parse_phandle(larbnode, "mediatek,smi", 0);
1104 if (!smi_subcomm_node)
1105 return -EINVAL;
1106
1107 /*
1108 * It may have two level smi-common. the node is smi-sub-common if it
1109 * has a new mediatek,smi property. otherwise it is smi-commmon.
1110 */
1111 smicomm_node = of_parse_phandle(smi_subcomm_node, "mediatek,smi", 0);
1112 if (smicomm_node)
1113 of_node_put(smi_subcomm_node);
1114 else
1115 smicomm_node = smi_subcomm_node;
1116
1117 pcommdev = of_find_device_by_node(smicomm_node);
1118 of_node_put(smicomm_node);
1119 if (!pcommdev)
1120 return -ENODEV;
1121 data->smicomm_dev = &pcommdev->dev;
1122
1123 link = device_link_add(data->smicomm_dev, dev,
1124 DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
1125 platform_device_put(pcommdev);
1126 if (!link) {
1127 dev_err(dev, "Unable to link %s.\n", dev_name(data->smicomm_dev));
1128 return -EINVAL;
1129 }
1130 return 0;
1131
1132 err_larbdev_put:
1133 for (i = MTK_LARB_NR_MAX - 1; i >= 0; i--) {
1134 if (!data->larb_imu[i].dev)
1135 continue;
1136 put_device(data->larb_imu[i].dev);
1137 }
1138 return ret;
1139 }
1140
mtk_iommu_probe(struct platform_device * pdev)1141 static int mtk_iommu_probe(struct platform_device *pdev)
1142 {
1143 struct mtk_iommu_data *data;
1144 struct device *dev = &pdev->dev;
1145 struct resource *res;
1146 resource_size_t ioaddr;
1147 struct component_match *match = NULL;
1148 struct regmap *infracfg;
1149 void *protect;
1150 int ret, banks_num, i = 0;
1151 u32 val;
1152 char *p;
1153 struct mtk_iommu_bank_data *bank;
1154 void __iomem *base;
1155
1156 data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
1157 if (!data)
1158 return -ENOMEM;
1159 data->dev = dev;
1160 data->plat_data = of_device_get_match_data(dev);
1161
1162 /* Protect memory. HW will access here while translation fault.*/
1163 protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL);
1164 if (!protect)
1165 return -ENOMEM;
1166 data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN);
1167
1168 if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_4GB_MODE)) {
1169 infracfg = syscon_regmap_lookup_by_phandle(dev->of_node, "mediatek,infracfg");
1170 if (IS_ERR(infracfg)) {
1171 /*
1172 * Legacy devicetrees will not specify a phandle to
1173 * mediatek,infracfg: in that case, we use the older
1174 * way to retrieve a syscon to infra.
1175 *
1176 * This is for retrocompatibility purposes only, hence
1177 * no more compatibles shall be added to this.
1178 */
1179 switch (data->plat_data->m4u_plat) {
1180 case M4U_MT2712:
1181 p = "mediatek,mt2712-infracfg";
1182 break;
1183 case M4U_MT8173:
1184 p = "mediatek,mt8173-infracfg";
1185 break;
1186 default:
1187 p = NULL;
1188 }
1189
1190 infracfg = syscon_regmap_lookup_by_compatible(p);
1191 if (IS_ERR(infracfg))
1192 return PTR_ERR(infracfg);
1193 }
1194
1195 ret = regmap_read(infracfg, REG_INFRA_MISC, &val);
1196 if (ret)
1197 return ret;
1198 data->enable_4GB = !!(val & F_DDR_4GB_SUPPORT_EN);
1199 }
1200
1201 banks_num = data->plat_data->banks_num;
1202 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1203 if (!res)
1204 return -EINVAL;
1205 if (resource_size(res) < banks_num * MTK_IOMMU_BANK_SZ) {
1206 dev_err(dev, "banknr %d. res %pR is not enough.\n", banks_num, res);
1207 return -EINVAL;
1208 }
1209 base = devm_ioremap_resource(dev, res);
1210 if (IS_ERR(base))
1211 return PTR_ERR(base);
1212 ioaddr = res->start;
1213
1214 data->bank = devm_kmalloc(dev, banks_num * sizeof(*data->bank), GFP_KERNEL);
1215 if (!data->bank)
1216 return -ENOMEM;
1217
1218 do {
1219 if (!data->plat_data->banks_enable[i])
1220 continue;
1221 bank = &data->bank[i];
1222 bank->id = i;
1223 bank->base = base + i * MTK_IOMMU_BANK_SZ;
1224 bank->m4u_dom = NULL;
1225
1226 bank->irq = platform_get_irq(pdev, i);
1227 if (bank->irq < 0)
1228 return bank->irq;
1229 bank->parent_dev = dev;
1230 bank->parent_data = data;
1231 spin_lock_init(&bank->tlb_lock);
1232 } while (++i < banks_num);
1233
1234 if (MTK_IOMMU_HAS_FLAG(data->plat_data, HAS_BCLK)) {
1235 data->bclk = devm_clk_get(dev, "bclk");
1236 if (IS_ERR(data->bclk))
1237 return PTR_ERR(data->bclk);
1238 }
1239
1240 pm_runtime_enable(dev);
1241
1242 if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) {
1243 ret = mtk_iommu_mm_dts_parse(dev, &match, data);
1244 if (ret) {
1245 dev_err_probe(dev, ret, "mm dts parse fail\n");
1246 goto out_runtime_disable;
1247 }
1248 } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA)) {
1249 p = data->plat_data->pericfg_comp_str;
1250 data->pericfg = syscon_regmap_lookup_by_compatible(p);
1251 if (IS_ERR(data->pericfg)) {
1252 ret = PTR_ERR(data->pericfg);
1253 goto out_runtime_disable;
1254 }
1255 }
1256
1257 platform_set_drvdata(pdev, data);
1258 mutex_init(&data->mutex);
1259
1260 ret = iommu_device_sysfs_add(&data->iommu, dev, NULL,
1261 "mtk-iommu.%pa", &ioaddr);
1262 if (ret)
1263 goto out_link_remove;
1264
1265 ret = iommu_device_register(&data->iommu, &mtk_iommu_ops, dev);
1266 if (ret)
1267 goto out_sysfs_remove;
1268
1269 if (MTK_IOMMU_HAS_FLAG(data->plat_data, SHARE_PGTABLE)) {
1270 list_add_tail(&data->list, data->plat_data->hw_list);
1271 data->hw_list = data->plat_data->hw_list;
1272 } else {
1273 INIT_LIST_HEAD(&data->hw_list_head);
1274 list_add_tail(&data->list, &data->hw_list_head);
1275 data->hw_list = &data->hw_list_head;
1276 }
1277
1278 if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) {
1279 ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match);
1280 if (ret)
1281 goto out_list_del;
1282 }
1283 return ret;
1284
1285 out_list_del:
1286 list_del(&data->list);
1287 iommu_device_unregister(&data->iommu);
1288 out_sysfs_remove:
1289 iommu_device_sysfs_remove(&data->iommu);
1290 out_link_remove:
1291 if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM))
1292 device_link_remove(data->smicomm_dev, dev);
1293 out_runtime_disable:
1294 pm_runtime_disable(dev);
1295 return ret;
1296 }
1297
mtk_iommu_remove(struct platform_device * pdev)1298 static int mtk_iommu_remove(struct platform_device *pdev)
1299 {
1300 struct mtk_iommu_data *data = platform_get_drvdata(pdev);
1301 struct mtk_iommu_bank_data *bank;
1302 int i;
1303
1304 iommu_device_sysfs_remove(&data->iommu);
1305 iommu_device_unregister(&data->iommu);
1306
1307 list_del(&data->list);
1308
1309 if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) {
1310 device_link_remove(data->smicomm_dev, &pdev->dev);
1311 component_master_del(&pdev->dev, &mtk_iommu_com_ops);
1312 }
1313 pm_runtime_disable(&pdev->dev);
1314 for (i = 0; i < data->plat_data->banks_num; i++) {
1315 bank = &data->bank[i];
1316 if (!bank->m4u_dom)
1317 continue;
1318 devm_free_irq(&pdev->dev, bank->irq, bank);
1319 }
1320 return 0;
1321 }
1322
mtk_iommu_runtime_suspend(struct device * dev)1323 static int __maybe_unused mtk_iommu_runtime_suspend(struct device *dev)
1324 {
1325 struct mtk_iommu_data *data = dev_get_drvdata(dev);
1326 struct mtk_iommu_suspend_reg *reg = &data->reg;
1327 void __iomem *base;
1328 int i = 0;
1329
1330 base = data->bank[i].base;
1331 reg->wr_len_ctrl = readl_relaxed(base + REG_MMU_WR_LEN_CTRL);
1332 reg->misc_ctrl = readl_relaxed(base + REG_MMU_MISC_CTRL);
1333 reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS);
1334 reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG);
1335 reg->vld_pa_rng = readl_relaxed(base + REG_MMU_VLD_PA_RNG);
1336 do {
1337 if (!data->plat_data->banks_enable[i])
1338 continue;
1339 base = data->bank[i].base;
1340 reg->int_control[i] = readl_relaxed(base + REG_MMU_INT_CONTROL0);
1341 reg->int_main_control[i] = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL);
1342 reg->ivrp_paddr[i] = readl_relaxed(base + REG_MMU_IVRP_PADDR);
1343 } while (++i < data->plat_data->banks_num);
1344 clk_disable_unprepare(data->bclk);
1345 return 0;
1346 }
1347
mtk_iommu_runtime_resume(struct device * dev)1348 static int __maybe_unused mtk_iommu_runtime_resume(struct device *dev)
1349 {
1350 struct mtk_iommu_data *data = dev_get_drvdata(dev);
1351 struct mtk_iommu_suspend_reg *reg = &data->reg;
1352 struct mtk_iommu_domain *m4u_dom;
1353 void __iomem *base;
1354 int ret, i = 0;
1355
1356 ret = clk_prepare_enable(data->bclk);
1357 if (ret) {
1358 dev_err(data->dev, "Failed to enable clk(%d) in resume\n", ret);
1359 return ret;
1360 }
1361
1362 /*
1363 * Uppon first resume, only enable the clk and return, since the values of the
1364 * registers are not yet set.
1365 */
1366 if (!reg->wr_len_ctrl)
1367 return 0;
1368
1369 base = data->bank[i].base;
1370 writel_relaxed(reg->wr_len_ctrl, base + REG_MMU_WR_LEN_CTRL);
1371 writel_relaxed(reg->misc_ctrl, base + REG_MMU_MISC_CTRL);
1372 writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS);
1373 writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG);
1374 writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG);
1375 do {
1376 m4u_dom = data->bank[i].m4u_dom;
1377 if (!data->plat_data->banks_enable[i] || !m4u_dom)
1378 continue;
1379 base = data->bank[i].base;
1380 writel_relaxed(reg->int_control[i], base + REG_MMU_INT_CONTROL0);
1381 writel_relaxed(reg->int_main_control[i], base + REG_MMU_INT_MAIN_CONTROL);
1382 writel_relaxed(reg->ivrp_paddr[i], base + REG_MMU_IVRP_PADDR);
1383 writel(m4u_dom->cfg.arm_v7s_cfg.ttbr, base + REG_MMU_PT_BASE_ADDR);
1384 } while (++i < data->plat_data->banks_num);
1385
1386 /*
1387 * Users may allocate dma buffer before they call pm_runtime_get,
1388 * in which case it will lack the necessary tlb flush.
1389 * Thus, make sure to update the tlb after each PM resume.
1390 */
1391 mtk_iommu_tlb_flush_all(data);
1392 return 0;
1393 }
1394
1395 static const struct dev_pm_ops mtk_iommu_pm_ops = {
1396 SET_RUNTIME_PM_OPS(mtk_iommu_runtime_suspend, mtk_iommu_runtime_resume, NULL)
1397 SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
1398 pm_runtime_force_resume)
1399 };
1400
1401 static const struct mtk_iommu_plat_data mt2712_data = {
1402 .m4u_plat = M4U_MT2712,
1403 .flags = HAS_4GB_MODE | HAS_BCLK | HAS_VLD_PA_RNG | SHARE_PGTABLE |
1404 MTK_IOMMU_TYPE_MM,
1405 .hw_list = &m4ulist,
1406 .inv_sel_reg = REG_MMU_INV_SEL_GEN1,
1407 .iova_region = single_domain,
1408 .banks_num = 1,
1409 .banks_enable = {true},
1410 .iova_region_nr = ARRAY_SIZE(single_domain),
1411 .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}},
1412 };
1413
1414 static const struct mtk_iommu_plat_data mt6779_data = {
1415 .m4u_plat = M4U_MT6779,
1416 .flags = HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | WR_THROT_EN |
1417 MTK_IOMMU_TYPE_MM | PGTABLE_PA_35_EN,
1418 .inv_sel_reg = REG_MMU_INV_SEL_GEN2,
1419 .banks_num = 1,
1420 .banks_enable = {true},
1421 .iova_region = single_domain,
1422 .iova_region_nr = ARRAY_SIZE(single_domain),
1423 .larbid_remap = {{0}, {1}, {2}, {3}, {5}, {7, 8}, {10}, {9}},
1424 };
1425
1426 static const struct mtk_iommu_plat_data mt6795_data = {
1427 .m4u_plat = M4U_MT6795,
1428 .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI |
1429 HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM |
1430 TF_PORT_TO_ADDR_MT8173,
1431 .inv_sel_reg = REG_MMU_INV_SEL_GEN1,
1432 .banks_num = 1,
1433 .banks_enable = {true},
1434 .iova_region = single_domain,
1435 .iova_region_nr = ARRAY_SIZE(single_domain),
1436 .larbid_remap = {{0}, {1}, {2}, {3}, {4}}, /* Linear mapping. */
1437 };
1438
1439 static const struct mtk_iommu_plat_data mt8167_data = {
1440 .m4u_plat = M4U_MT8167,
1441 .flags = RESET_AXI | HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM,
1442 .inv_sel_reg = REG_MMU_INV_SEL_GEN1,
1443 .banks_num = 1,
1444 .banks_enable = {true},
1445 .iova_region = single_domain,
1446 .iova_region_nr = ARRAY_SIZE(single_domain),
1447 .larbid_remap = {{0}, {1}, {2}}, /* Linear mapping. */
1448 };
1449
1450 static const struct mtk_iommu_plat_data mt8173_data = {
1451 .m4u_plat = M4U_MT8173,
1452 .flags = HAS_4GB_MODE | HAS_BCLK | RESET_AXI |
1453 HAS_LEGACY_IVRP_PADDR | MTK_IOMMU_TYPE_MM |
1454 TF_PORT_TO_ADDR_MT8173,
1455 .inv_sel_reg = REG_MMU_INV_SEL_GEN1,
1456 .banks_num = 1,
1457 .banks_enable = {true},
1458 .iova_region = single_domain,
1459 .iova_region_nr = ARRAY_SIZE(single_domain),
1460 .larbid_remap = {{0}, {1}, {2}, {3}, {4}, {5}}, /* Linear mapping. */
1461 };
1462
1463 static const struct mtk_iommu_plat_data mt8183_data = {
1464 .m4u_plat = M4U_MT8183,
1465 .flags = RESET_AXI | MTK_IOMMU_TYPE_MM,
1466 .inv_sel_reg = REG_MMU_INV_SEL_GEN1,
1467 .banks_num = 1,
1468 .banks_enable = {true},
1469 .iova_region = single_domain,
1470 .iova_region_nr = ARRAY_SIZE(single_domain),
1471 .larbid_remap = {{0}, {4}, {5}, {6}, {7}, {2}, {3}, {1}},
1472 };
1473
1474 static const struct mtk_iommu_plat_data mt8186_data_mm = {
1475 .m4u_plat = M4U_MT8186,
1476 .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN |
1477 WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM,
1478 .larbid_remap = {{0}, {1, MTK_INVALID_LARBID, 8}, {4}, {7}, {2}, {9, 11, 19, 20},
1479 {MTK_INVALID_LARBID, 14, 16},
1480 {MTK_INVALID_LARBID, 13, MTK_INVALID_LARBID, 17}},
1481 .inv_sel_reg = REG_MMU_INV_SEL_GEN2,
1482 .banks_num = 1,
1483 .banks_enable = {true},
1484 .iova_region = mt8192_multi_dom,
1485 .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom),
1486 };
1487
1488 static const struct mtk_iommu_plat_data mt8192_data = {
1489 .m4u_plat = M4U_MT8192,
1490 .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN |
1491 WR_THROT_EN | IOVA_34_EN | MTK_IOMMU_TYPE_MM,
1492 .inv_sel_reg = REG_MMU_INV_SEL_GEN2,
1493 .banks_num = 1,
1494 .banks_enable = {true},
1495 .iova_region = mt8192_multi_dom,
1496 .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom),
1497 .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20},
1498 {0, 14, 16}, {0, 13, 18, 17}},
1499 };
1500
1501 static const struct mtk_iommu_plat_data mt8195_data_infra = {
1502 .m4u_plat = M4U_MT8195,
1503 .flags = WR_THROT_EN | DCM_DISABLE | STD_AXI_MODE | PM_CLK_AO |
1504 MTK_IOMMU_TYPE_INFRA | IFA_IOMMU_PCIE_SUPPORT,
1505 .pericfg_comp_str = "mediatek,mt8195-pericfg_ao",
1506 .inv_sel_reg = REG_MMU_INV_SEL_GEN2,
1507 .banks_num = 5,
1508 .banks_enable = {true, false, false, false, true},
1509 .banks_portmsk = {[0] = GENMASK(19, 16), /* PCIe */
1510 [4] = GENMASK(31, 20), /* USB */
1511 },
1512 .iova_region = single_domain,
1513 .iova_region_nr = ARRAY_SIZE(single_domain),
1514 };
1515
1516 static const struct mtk_iommu_plat_data mt8195_data_vdo = {
1517 .m4u_plat = M4U_MT8195,
1518 .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN |
1519 WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | MTK_IOMMU_TYPE_MM,
1520 .hw_list = &m4ulist,
1521 .inv_sel_reg = REG_MMU_INV_SEL_GEN2,
1522 .banks_num = 1,
1523 .banks_enable = {true},
1524 .iova_region = mt8192_multi_dom,
1525 .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom),
1526 .larbid_remap = {{2, 0}, {21}, {24}, {7}, {19}, {9, 10, 11},
1527 {13, 17, 15/* 17b */, 25}, {5}},
1528 };
1529
1530 static const struct mtk_iommu_plat_data mt8195_data_vpp = {
1531 .m4u_plat = M4U_MT8195,
1532 .flags = HAS_BCLK | HAS_SUB_COMM_3BITS | OUT_ORDER_WR_EN |
1533 WR_THROT_EN | IOVA_34_EN | SHARE_PGTABLE | MTK_IOMMU_TYPE_MM,
1534 .hw_list = &m4ulist,
1535 .inv_sel_reg = REG_MMU_INV_SEL_GEN2,
1536 .banks_num = 1,
1537 .banks_enable = {true},
1538 .iova_region = mt8192_multi_dom,
1539 .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom),
1540 .larbid_remap = {{1}, {3},
1541 {22, MTK_INVALID_LARBID, MTK_INVALID_LARBID, MTK_INVALID_LARBID, 23},
1542 {8}, {20}, {12},
1543 /* 16: 16a; 29: 16b; 30: CCUtop0; 31: CCUtop1 */
1544 {14, 16, 29, 26, 30, 31, 18},
1545 {4, MTK_INVALID_LARBID, MTK_INVALID_LARBID, MTK_INVALID_LARBID, 6}},
1546 };
1547
1548 static const struct of_device_id mtk_iommu_of_ids[] = {
1549 { .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data},
1550 { .compatible = "mediatek,mt6779-m4u", .data = &mt6779_data},
1551 { .compatible = "mediatek,mt6795-m4u", .data = &mt6795_data},
1552 { .compatible = "mediatek,mt8167-m4u", .data = &mt8167_data},
1553 { .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data},
1554 { .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data},
1555 { .compatible = "mediatek,mt8186-iommu-mm", .data = &mt8186_data_mm}, /* mm: m4u */
1556 { .compatible = "mediatek,mt8192-m4u", .data = &mt8192_data},
1557 { .compatible = "mediatek,mt8195-iommu-infra", .data = &mt8195_data_infra},
1558 { .compatible = "mediatek,mt8195-iommu-vdo", .data = &mt8195_data_vdo},
1559 { .compatible = "mediatek,mt8195-iommu-vpp", .data = &mt8195_data_vpp},
1560 {}
1561 };
1562
1563 static struct platform_driver mtk_iommu_driver = {
1564 .probe = mtk_iommu_probe,
1565 .remove = mtk_iommu_remove,
1566 .driver = {
1567 .name = "mtk-iommu",
1568 .of_match_table = mtk_iommu_of_ids,
1569 .pm = &mtk_iommu_pm_ops,
1570 }
1571 };
1572 module_platform_driver(mtk_iommu_driver);
1573
1574 MODULE_DESCRIPTION("IOMMU API for MediaTek M4U implementations");
1575 MODULE_LICENSE("GPL v2");
1576