1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2016-2018 Etnaviv Project
4  */
5 
6 #include <linux/bitops.h>
7 #include <linux/dma-mapping.h>
8 #include <linux/platform_device.h>
9 #include <linux/sizes.h>
10 #include <linux/slab.h>
11 #include <linux/vmalloc.h>
12 
13 #include "etnaviv_cmdbuf.h"
14 #include "etnaviv_gpu.h"
15 #include "etnaviv_mmu.h"
16 #include "state.xml.h"
17 #include "state_hi.xml.h"
18 
19 #define MMUv2_PTE_PRESENT		BIT(0)
20 #define MMUv2_PTE_EXCEPTION		BIT(1)
21 #define MMUv2_PTE_WRITEABLE		BIT(2)
22 
23 #define MMUv2_MTLB_MASK			0xffc00000
24 #define MMUv2_MTLB_SHIFT		22
25 #define MMUv2_STLB_MASK			0x003ff000
26 #define MMUv2_STLB_SHIFT		12
27 
28 #define MMUv2_MAX_STLB_ENTRIES		1024
29 
30 struct etnaviv_iommuv2_context {
31 	struct etnaviv_iommu_context base;
32 	unsigned short id;
33 	/* M(aster) TLB aka first level pagetable */
34 	u32 *mtlb_cpu;
35 	dma_addr_t mtlb_dma;
36 	/* S(lave) TLB aka second level pagetable */
37 	u32 *stlb_cpu[MMUv2_MAX_STLB_ENTRIES];
38 	dma_addr_t stlb_dma[MMUv2_MAX_STLB_ENTRIES];
39 };
40 
41 static struct etnaviv_iommuv2_context *
to_v2_context(struct etnaviv_iommu_context * context)42 to_v2_context(struct etnaviv_iommu_context *context)
43 {
44 	return container_of(context, struct etnaviv_iommuv2_context, base);
45 }
46 
etnaviv_iommuv2_free(struct etnaviv_iommu_context * context)47 static void etnaviv_iommuv2_free(struct etnaviv_iommu_context *context)
48 {
49 	struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
50 	int i;
51 
52 	drm_mm_takedown(&context->mm);
53 
54 	for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
55 		if (v2_context->stlb_cpu[i])
56 			dma_free_wc(context->global->dev, SZ_4K,
57 				    v2_context->stlb_cpu[i],
58 				    v2_context->stlb_dma[i]);
59 	}
60 
61 	dma_free_wc(context->global->dev, SZ_4K, v2_context->mtlb_cpu,
62 		    v2_context->mtlb_dma);
63 
64 	clear_bit(v2_context->id, context->global->v2.pta_alloc);
65 
66 	vfree(v2_context);
67 }
68 static int
etnaviv_iommuv2_ensure_stlb(struct etnaviv_iommuv2_context * v2_context,int stlb)69 etnaviv_iommuv2_ensure_stlb(struct etnaviv_iommuv2_context *v2_context,
70 			    int stlb)
71 {
72 	if (v2_context->stlb_cpu[stlb])
73 		return 0;
74 
75 	v2_context->stlb_cpu[stlb] =
76 			dma_alloc_wc(v2_context->base.global->dev, SZ_4K,
77 				     &v2_context->stlb_dma[stlb],
78 				     GFP_KERNEL);
79 
80 	if (!v2_context->stlb_cpu[stlb])
81 		return -ENOMEM;
82 
83 	memset32(v2_context->stlb_cpu[stlb], MMUv2_PTE_EXCEPTION,
84 		 SZ_4K / sizeof(u32));
85 
86 	v2_context->mtlb_cpu[stlb] =
87 			v2_context->stlb_dma[stlb] | MMUv2_PTE_PRESENT;
88 
89 	return 0;
90 }
91 
etnaviv_iommuv2_map(struct etnaviv_iommu_context * context,unsigned long iova,phys_addr_t paddr,size_t size,int prot)92 static int etnaviv_iommuv2_map(struct etnaviv_iommu_context *context,
93 			       unsigned long iova, phys_addr_t paddr,
94 			       size_t size, int prot)
95 {
96 	struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
97 	int mtlb_entry, stlb_entry, ret;
98 	u32 entry = lower_32_bits(paddr) | MMUv2_PTE_PRESENT;
99 
100 	if (size != SZ_4K)
101 		return -EINVAL;
102 
103 	if (IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
104 		entry |= (upper_32_bits(paddr) & 0xff) << 4;
105 
106 	if (prot & ETNAVIV_PROT_WRITE)
107 		entry |= MMUv2_PTE_WRITEABLE;
108 
109 	mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
110 	stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
111 
112 	ret = etnaviv_iommuv2_ensure_stlb(v2_context, mtlb_entry);
113 	if (ret)
114 		return ret;
115 
116 	v2_context->stlb_cpu[mtlb_entry][stlb_entry] = entry;
117 
118 	return 0;
119 }
120 
etnaviv_iommuv2_unmap(struct etnaviv_iommu_context * context,unsigned long iova,size_t size)121 static size_t etnaviv_iommuv2_unmap(struct etnaviv_iommu_context *context,
122 				    unsigned long iova, size_t size)
123 {
124 	struct etnaviv_iommuv2_context *etnaviv_domain = to_v2_context(context);
125 	int mtlb_entry, stlb_entry;
126 
127 	if (size != SZ_4K)
128 		return -EINVAL;
129 
130 	mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
131 	stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
132 
133 	etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = MMUv2_PTE_EXCEPTION;
134 
135 	return SZ_4K;
136 }
137 
etnaviv_iommuv2_dump_size(struct etnaviv_iommu_context * context)138 static size_t etnaviv_iommuv2_dump_size(struct etnaviv_iommu_context *context)
139 {
140 	struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
141 	size_t dump_size = SZ_4K;
142 	int i;
143 
144 	for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++)
145 		if (v2_context->mtlb_cpu[i] & MMUv2_PTE_PRESENT)
146 			dump_size += SZ_4K;
147 
148 	return dump_size;
149 }
150 
etnaviv_iommuv2_dump(struct etnaviv_iommu_context * context,void * buf)151 static void etnaviv_iommuv2_dump(struct etnaviv_iommu_context *context, void *buf)
152 {
153 	struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
154 	int i;
155 
156 	memcpy(buf, v2_context->mtlb_cpu, SZ_4K);
157 	buf += SZ_4K;
158 	for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++)
159 		if (v2_context->mtlb_cpu[i] & MMUv2_PTE_PRESENT) {
160 			memcpy(buf, v2_context->stlb_cpu[i], SZ_4K);
161 			buf += SZ_4K;
162 		}
163 }
164 
etnaviv_iommuv2_restore_nonsec(struct etnaviv_gpu * gpu,struct etnaviv_iommu_context * context)165 static void etnaviv_iommuv2_restore_nonsec(struct etnaviv_gpu *gpu,
166 	struct etnaviv_iommu_context *context)
167 {
168 	struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
169 	u16 prefetch;
170 
171 	/* If the MMU is already enabled the state is still there. */
172 	if (gpu_read(gpu, VIVS_MMUv2_CONTROL) & VIVS_MMUv2_CONTROL_ENABLE)
173 		return;
174 
175 	if (gpu->mmu_context)
176 		etnaviv_iommu_context_put(gpu->mmu_context);
177 	gpu->mmu_context = etnaviv_iommu_context_get(context);
178 
179 	prefetch = etnaviv_buffer_config_mmuv2(gpu,
180 				(u32)v2_context->mtlb_dma,
181 				(u32)context->global->bad_page_dma);
182 	etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer),
183 			     prefetch);
184 	etnaviv_gpu_wait_idle(gpu, 100);
185 
186 	gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE);
187 }
188 
etnaviv_iommuv2_restore_sec(struct etnaviv_gpu * gpu,struct etnaviv_iommu_context * context)189 static void etnaviv_iommuv2_restore_sec(struct etnaviv_gpu *gpu,
190 	struct etnaviv_iommu_context *context)
191 {
192 	struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
193 	u16 prefetch;
194 
195 	/* If the MMU is already enabled the state is still there. */
196 	if (gpu_read(gpu, VIVS_MMUv2_SEC_CONTROL) & VIVS_MMUv2_SEC_CONTROL_ENABLE)
197 		return;
198 
199 	if (gpu->mmu_context)
200 		etnaviv_iommu_context_put(gpu->mmu_context);
201 	gpu->mmu_context = etnaviv_iommu_context_get(context);
202 
203 	gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_LOW,
204 		  lower_32_bits(context->global->v2.pta_dma));
205 	gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_HIGH,
206 		  upper_32_bits(context->global->v2.pta_dma));
207 	gpu_write(gpu, VIVS_MMUv2_PTA_CONTROL, VIVS_MMUv2_PTA_CONTROL_ENABLE);
208 
209 	gpu_write(gpu, VIVS_MMUv2_NONSEC_SAFE_ADDR_LOW,
210 		  lower_32_bits(context->global->bad_page_dma));
211 	gpu_write(gpu, VIVS_MMUv2_SEC_SAFE_ADDR_LOW,
212 		  lower_32_bits(context->global->bad_page_dma));
213 	gpu_write(gpu, VIVS_MMUv2_SAFE_ADDRESS_CONFIG,
214 		  VIVS_MMUv2_SAFE_ADDRESS_CONFIG_NON_SEC_SAFE_ADDR_HIGH(
215 		  upper_32_bits(context->global->bad_page_dma)) |
216 		  VIVS_MMUv2_SAFE_ADDRESS_CONFIG_SEC_SAFE_ADDR_HIGH(
217 		  upper_32_bits(context->global->bad_page_dma)));
218 
219 	context->global->v2.pta_cpu[v2_context->id] = v2_context->mtlb_dma |
220 				 	 VIVS_MMUv2_CONFIGURATION_MODE_MODE4_K;
221 
222 	/* trigger a PTA load through the FE */
223 	prefetch = etnaviv_buffer_config_pta(gpu, v2_context->id);
224 	etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer),
225 			     prefetch);
226 	etnaviv_gpu_wait_idle(gpu, 100);
227 
228 	gpu_write(gpu, VIVS_MMUv2_SEC_CONTROL, VIVS_MMUv2_SEC_CONTROL_ENABLE);
229 }
230 
etnaviv_iommuv2_get_mtlb_addr(struct etnaviv_iommu_context * context)231 u32 etnaviv_iommuv2_get_mtlb_addr(struct etnaviv_iommu_context *context)
232 {
233 	struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
234 
235 	return v2_context->mtlb_dma;
236 }
237 
etnaviv_iommuv2_get_pta_id(struct etnaviv_iommu_context * context)238 unsigned short etnaviv_iommuv2_get_pta_id(struct etnaviv_iommu_context *context)
239 {
240 	struct etnaviv_iommuv2_context *v2_context = to_v2_context(context);
241 
242 	return v2_context->id;
243 }
etnaviv_iommuv2_restore(struct etnaviv_gpu * gpu,struct etnaviv_iommu_context * context)244 static void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu,
245 				    struct etnaviv_iommu_context *context)
246 {
247 	switch (gpu->sec_mode) {
248 	case ETNA_SEC_NONE:
249 		etnaviv_iommuv2_restore_nonsec(gpu, context);
250 		break;
251 	case ETNA_SEC_KERNEL:
252 		etnaviv_iommuv2_restore_sec(gpu, context);
253 		break;
254 	default:
255 		WARN(1, "unhandled GPU security mode\n");
256 		break;
257 	}
258 }
259 
260 const struct etnaviv_iommu_ops etnaviv_iommuv2_ops = {
261 	.free = etnaviv_iommuv2_free,
262 	.map = etnaviv_iommuv2_map,
263 	.unmap = etnaviv_iommuv2_unmap,
264 	.dump_size = etnaviv_iommuv2_dump_size,
265 	.dump = etnaviv_iommuv2_dump,
266 	.restore = etnaviv_iommuv2_restore,
267 };
268 
269 struct etnaviv_iommu_context *
etnaviv_iommuv2_context_alloc(struct etnaviv_iommu_global * global)270 etnaviv_iommuv2_context_alloc(struct etnaviv_iommu_global *global)
271 {
272 	struct etnaviv_iommuv2_context *v2_context;
273 	struct etnaviv_iommu_context *context;
274 
275 	v2_context = vzalloc(sizeof(*v2_context));
276 	if (!v2_context)
277 		return NULL;
278 
279 	mutex_lock(&global->lock);
280 	v2_context->id = find_first_zero_bit(global->v2.pta_alloc,
281 					     ETNAVIV_PTA_ENTRIES);
282 	if (v2_context->id < ETNAVIV_PTA_ENTRIES) {
283 		set_bit(v2_context->id, global->v2.pta_alloc);
284 	} else {
285 		mutex_unlock(&global->lock);
286 		goto out_free;
287 	}
288 	mutex_unlock(&global->lock);
289 
290 	v2_context->mtlb_cpu = dma_alloc_wc(global->dev, SZ_4K,
291 					    &v2_context->mtlb_dma, GFP_KERNEL);
292 	if (!v2_context->mtlb_cpu)
293 		goto out_free_id;
294 
295 	memset32(v2_context->mtlb_cpu, MMUv2_PTE_EXCEPTION,
296 		 MMUv2_MAX_STLB_ENTRIES);
297 
298 	global->v2.pta_cpu[v2_context->id] = v2_context->mtlb_dma;
299 
300 	context = &v2_context->base;
301 	context->global = global;
302 	kref_init(&context->refcount);
303 	mutex_init(&context->lock);
304 	INIT_LIST_HEAD(&context->mappings);
305 	drm_mm_init(&context->mm, SZ_4K, (u64)SZ_1G * 4 - SZ_4K);
306 
307 	return context;
308 
309 out_free_id:
310 	clear_bit(v2_context->id, global->v2.pta_alloc);
311 out_free:
312 	vfree(v2_context);
313 	return NULL;
314 }
315