1 // SPDX-License-Identifier: GPL-2.0
2 
3 /*
4  * Copyright 2016-2022 HabanaLabs, Ltd.
5  * All Rights Reserved.
6  */
7 
8 #include <linux/slab.h>
9 
10 #include "../habanalabs.h"
11 
12 #include <trace/events/habanalabs.h>
13 
14 /**
15  * hl_mmu_get_funcs() - get MMU functions structure
16  * @hdev: habanalabs device structure.
17  * @pgt_residency: page table residency.
18  * @is_dram_addr: true if we need HMMU functions
19  *
20  * @return appropriate MMU functions structure
21  */
hl_mmu_get_funcs(struct hl_device * hdev,int pgt_residency,bool is_dram_addr)22 static struct hl_mmu_funcs *hl_mmu_get_funcs(struct hl_device *hdev, int pgt_residency,
23 									bool is_dram_addr)
24 {
25 	return &hdev->mmu_func[pgt_residency];
26 }
27 
hl_is_dram_va(struct hl_device * hdev,u64 virt_addr)28 bool hl_is_dram_va(struct hl_device *hdev, u64 virt_addr)
29 {
30 	struct asic_fixed_properties *prop = &hdev->asic_prop;
31 
32 	return hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size,
33 					prop->dmmu.start_addr,
34 					prop->dmmu.end_addr);
35 }
36 
37 /**
38  * hl_mmu_init() - initialize the MMU module.
39  * @hdev: habanalabs device structure.
40  *
41  * Return: 0 for success, non-zero for failure.
42  */
hl_mmu_init(struct hl_device * hdev)43 int hl_mmu_init(struct hl_device *hdev)
44 {
45 	int rc = -EOPNOTSUPP;
46 
47 	if (!hdev->mmu_enable)
48 		return 0;
49 
50 	mutex_init(&hdev->mmu_lock);
51 
52 	if (hdev->mmu_func[MMU_DR_PGT].init != NULL) {
53 		rc = hdev->mmu_func[MMU_DR_PGT].init(hdev);
54 		if (rc)
55 			return rc;
56 	}
57 
58 	if (hdev->mmu_func[MMU_HR_PGT].init != NULL) {
59 		rc = hdev->mmu_func[MMU_HR_PGT].init(hdev);
60 		if (rc)
61 			goto fini_dr_mmu;
62 	}
63 
64 	return 0;
65 
66 fini_dr_mmu:
67 	if (hdev->mmu_func[MMU_DR_PGT].fini != NULL)
68 		hdev->mmu_func[MMU_DR_PGT].fini(hdev);
69 
70 	return rc;
71 }
72 
73 /**
74  * hl_mmu_fini() - release the MMU module.
75  * @hdev: habanalabs device structure.
76  *
77  * This function does the following:
78  * - Disable MMU in H/W.
79  * - Free the pgt_infos pool.
80  *
81  * All contexts should be freed before calling this function.
82  */
hl_mmu_fini(struct hl_device * hdev)83 void hl_mmu_fini(struct hl_device *hdev)
84 {
85 	if (!hdev->mmu_enable)
86 		return;
87 
88 	if (hdev->mmu_func[MMU_DR_PGT].fini != NULL)
89 		hdev->mmu_func[MMU_DR_PGT].fini(hdev);
90 
91 	if (hdev->mmu_func[MMU_HR_PGT].fini != NULL)
92 		hdev->mmu_func[MMU_HR_PGT].fini(hdev);
93 
94 	mutex_destroy(&hdev->mmu_lock);
95 }
96 
97 /**
98  * hl_mmu_ctx_init() - initialize a context for using the MMU module.
99  * @ctx: pointer to the context structure to initialize.
100  *
101  * Initialize a mutex to protect the concurrent mapping flow, a hash to hold all
102  * page tables hops related to this context.
103  * Return: 0 on success, non-zero otherwise.
104  */
hl_mmu_ctx_init(struct hl_ctx * ctx)105 int hl_mmu_ctx_init(struct hl_ctx *ctx)
106 {
107 	struct hl_device *hdev = ctx->hdev;
108 	int rc = -EOPNOTSUPP;
109 
110 	if (!hdev->mmu_enable)
111 		return 0;
112 
113 	if (hdev->mmu_func[MMU_DR_PGT].ctx_init != NULL) {
114 		rc = hdev->mmu_func[MMU_DR_PGT].ctx_init(ctx);
115 		if (rc)
116 			return rc;
117 	}
118 
119 	if (hdev->mmu_func[MMU_HR_PGT].ctx_init != NULL) {
120 		rc = hdev->mmu_func[MMU_HR_PGT].ctx_init(ctx);
121 		if (rc)
122 			goto fini_dr_ctx;
123 	}
124 
125 	return 0;
126 
127 fini_dr_ctx:
128 	if (hdev->mmu_func[MMU_DR_PGT].fini != NULL)
129 		hdev->mmu_func[MMU_DR_PGT].fini(hdev);
130 
131 	return rc;
132 }
133 
134 /*
135  * hl_mmu_ctx_fini - disable a ctx from using the mmu module
136  *
137  * @ctx: pointer to the context structure
138  *
139  * This function does the following:
140  * - Free any pgts which were not freed yet
141  * - Free the mutex
142  * - Free DRAM default page mapping hops
143  */
hl_mmu_ctx_fini(struct hl_ctx * ctx)144 void hl_mmu_ctx_fini(struct hl_ctx *ctx)
145 {
146 	struct hl_device *hdev = ctx->hdev;
147 
148 	if (!hdev->mmu_enable)
149 		return;
150 
151 	if (hdev->mmu_func[MMU_DR_PGT].ctx_fini != NULL)
152 		hdev->mmu_func[MMU_DR_PGT].ctx_fini(ctx);
153 
154 	if (hdev->mmu_func[MMU_HR_PGT].ctx_fini != NULL)
155 		hdev->mmu_func[MMU_HR_PGT].ctx_fini(ctx);
156 }
157 
158 /*
159  * hl_mmu_get_real_page_size - get real page size to use in map/unmap operation
160  *
161  * @hdev: pointer to device data.
162  * @mmu_prop: MMU properties.
163  * @page_size: page size
164  * @real_page_size: set here the actual page size to use for the operation
165  * @is_dram_addr: true if DRAM address, otherwise false.
166  *
167  * @return 0 on success, otherwise non 0 error code
168  *
169  * note that this is general implementation that can fit most MMU arch. but as this is used as an
170  * MMU function:
171  * 1. it shall not be called directly- only from mmu_func structure instance
172  * 2. each MMU may modify the implementation internally
173  */
hl_mmu_get_real_page_size(struct hl_device * hdev,struct hl_mmu_properties * mmu_prop,u32 page_size,u32 * real_page_size,bool is_dram_addr)174 int hl_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop,
175 				u32 page_size, u32 *real_page_size, bool is_dram_addr)
176 {
177 	/*
178 	 * The H/W handles mapping of specific page sizes. Hence if the page
179 	 * size is bigger, we break it to sub-pages and map them separately.
180 	 */
181 	if ((page_size % mmu_prop->page_size) == 0) {
182 		*real_page_size = mmu_prop->page_size;
183 		return 0;
184 	}
185 
186 	dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n",
187 						page_size, mmu_prop->page_size >> 10);
188 
189 	return -EFAULT;
190 }
191 
hl_mmu_get_prop(struct hl_device * hdev,u32 page_size,bool is_dram_addr)192 static struct hl_mmu_properties *hl_mmu_get_prop(struct hl_device *hdev, u32 page_size,
193 							bool is_dram_addr)
194 {
195 	struct asic_fixed_properties *prop = &hdev->asic_prop;
196 
197 	if (is_dram_addr)
198 		return &prop->dmmu;
199 	else if ((page_size % prop->pmmu_huge.page_size) == 0)
200 		return &prop->pmmu_huge;
201 
202 	return &prop->pmmu;
203 }
204 
205 /*
206  * hl_mmu_unmap_page - unmaps a virtual addr
207  *
208  * @ctx: pointer to the context structure
209  * @virt_addr: virt addr to map from
210  * @page_size: size of the page to unmap
211  * @flush_pte: whether to do a PCI flush
212  *
213  * This function does the following:
214  * - Check that the virt addr is mapped
215  * - Unmap the virt addr and frees pgts if possible
216  * - Returns 0 on success, -EINVAL if the given addr is not mapped
217  *
218  * Because this function changes the page tables in the device and because it
219  * changes the MMU hash, it must be protected by a lock.
220  * However, because it maps only a single page, the lock should be implemented
221  * in a higher level in order to protect the entire mapping of the memory area
222  *
223  * For optimization reasons PCI flush may be requested once after unmapping of
224  * large area.
225  */
hl_mmu_unmap_page(struct hl_ctx * ctx,u64 virt_addr,u32 page_size,bool flush_pte)226 int hl_mmu_unmap_page(struct hl_ctx *ctx, u64 virt_addr, u32 page_size, bool flush_pte)
227 {
228 	struct hl_device *hdev = ctx->hdev;
229 	struct hl_mmu_properties *mmu_prop;
230 	struct hl_mmu_funcs *mmu_funcs;
231 	int i, pgt_residency, rc = 0;
232 	u32 real_page_size, npages;
233 	u64 real_virt_addr;
234 	bool is_dram_addr;
235 
236 	if (!hdev->mmu_enable)
237 		return 0;
238 
239 	is_dram_addr = hl_is_dram_va(hdev, virt_addr);
240 	mmu_prop = hl_mmu_get_prop(hdev, page_size, is_dram_addr);
241 
242 	pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT;
243 	mmu_funcs = hl_mmu_get_funcs(hdev, pgt_residency, is_dram_addr);
244 
245 	rc = hdev->asic_funcs->mmu_get_real_page_size(hdev, mmu_prop, page_size, &real_page_size,
246 							is_dram_addr);
247 	if (rc)
248 		return rc;
249 
250 	npages = page_size / real_page_size;
251 	real_virt_addr = virt_addr;
252 
253 	for (i = 0 ; i < npages ; i++) {
254 		rc = mmu_funcs->unmap(ctx, real_virt_addr, is_dram_addr);
255 		if (rc)
256 			break;
257 
258 		real_virt_addr += real_page_size;
259 	}
260 
261 	if (flush_pte)
262 		mmu_funcs->flush(ctx);
263 
264 	if (trace_habanalabs_mmu_unmap_enabled() && !rc)
265 		trace_habanalabs_mmu_unmap(hdev->dev, virt_addr, 0, page_size, flush_pte);
266 
267 	return rc;
268 }
269 
270 /*
271  * hl_mmu_map_page - maps a virtual addr to physical addr
272  *
273  * @ctx: pointer to the context structure
274  * @virt_addr: virt addr to map from
275  * @phys_addr: phys addr to map to
276  * @page_size: physical page size
277  * @flush_pte: whether to do a PCI flush
278  *
279  * This function does the following:
280  * - Check that the virt addr is not mapped
281  * - Allocate pgts as necessary in order to map the virt addr to the phys
282  * - Returns 0 on success, -EINVAL if addr is already mapped, or -ENOMEM.
283  *
284  * Because this function changes the page tables in the device and because it
285  * changes the MMU hash, it must be protected by a lock.
286  * However, because it maps only a single page, the lock should be implemented
287  * in a higher level in order to protect the entire mapping of the memory area
288  *
289  * For optimization reasons PCI flush may be requested once after mapping of
290  * large area.
291  */
hl_mmu_map_page(struct hl_ctx * ctx,u64 virt_addr,u64 phys_addr,u32 page_size,bool flush_pte)292 int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size,
293 			bool flush_pte)
294 {
295 	int i, rc, pgt_residency, mapped_cnt = 0;
296 	struct hl_device *hdev = ctx->hdev;
297 	struct hl_mmu_properties *mmu_prop;
298 	u64 real_virt_addr, real_phys_addr;
299 	struct hl_mmu_funcs *mmu_funcs;
300 	u32 real_page_size, npages;
301 	bool is_dram_addr;
302 
303 
304 	if (!hdev->mmu_enable)
305 		return 0;
306 
307 	is_dram_addr = hl_is_dram_va(hdev, virt_addr);
308 	mmu_prop = hl_mmu_get_prop(hdev, page_size, is_dram_addr);
309 
310 	pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT;
311 	mmu_funcs = hl_mmu_get_funcs(hdev, pgt_residency, is_dram_addr);
312 
313 	rc = hdev->asic_funcs->mmu_get_real_page_size(hdev, mmu_prop, page_size, &real_page_size,
314 							is_dram_addr);
315 	if (rc)
316 		return rc;
317 
318 	/*
319 	 * Verify that the phys and virt addresses are aligned with the
320 	 * MMU page size (in dram this means checking the address and MMU
321 	 * after scrambling)
322 	 */
323 	if ((is_dram_addr &&
324 			((hdev->asic_funcs->scramble_addr(hdev, phys_addr) &
325 				(mmu_prop->page_size - 1)) ||
326 			(hdev->asic_funcs->scramble_addr(hdev, virt_addr) &
327 				(mmu_prop->page_size - 1)))) ||
328 		(!is_dram_addr && ((phys_addr & (real_page_size - 1)) ||
329 				(virt_addr & (real_page_size - 1)))))
330 		dev_crit(hdev->dev,
331 			"Mapping address 0x%llx with virtual address 0x%llx and page size of 0x%x is erroneous! Addresses must be divisible by page size",
332 			phys_addr, virt_addr, real_page_size);
333 
334 	npages = page_size / real_page_size;
335 	real_virt_addr = virt_addr;
336 	real_phys_addr = phys_addr;
337 
338 	for (i = 0 ; i < npages ; i++) {
339 		rc = mmu_funcs->map(ctx, real_virt_addr, real_phys_addr, real_page_size,
340 										is_dram_addr);
341 		if (rc)
342 			goto err;
343 
344 		real_virt_addr += real_page_size;
345 		real_phys_addr += real_page_size;
346 		mapped_cnt++;
347 	}
348 
349 	if (flush_pte)
350 		mmu_funcs->flush(ctx);
351 
352 	trace_habanalabs_mmu_map(hdev->dev, virt_addr, phys_addr, page_size, flush_pte);
353 
354 	return 0;
355 
356 err:
357 	real_virt_addr = virt_addr;
358 	for (i = 0 ; i < mapped_cnt ; i++) {
359 		if (mmu_funcs->unmap(ctx, real_virt_addr, is_dram_addr))
360 			dev_warn_ratelimited(hdev->dev,
361 				"failed to unmap va: 0x%llx\n", real_virt_addr);
362 
363 		real_virt_addr += real_page_size;
364 	}
365 
366 	mmu_funcs->flush(ctx);
367 
368 	return rc;
369 }
370 
371 /*
372  * hl_mmu_map_contiguous - implements a wrapper for hl_mmu_map_page
373  *                         for mapping contiguous physical memory
374  *
375  * @ctx: pointer to the context structure
376  * @virt_addr: virt addr to map from
377  * @phys_addr: phys addr to map to
378  * @size: size to map
379  *
380  */
hl_mmu_map_contiguous(struct hl_ctx * ctx,u64 virt_addr,u64 phys_addr,u32 size)381 int hl_mmu_map_contiguous(struct hl_ctx *ctx, u64 virt_addr,
382 					u64 phys_addr, u32 size)
383 {
384 	struct hl_device *hdev = ctx->hdev;
385 	struct asic_fixed_properties *prop = &hdev->asic_prop;
386 	u64 curr_va, curr_pa;
387 	u32 page_size;
388 	bool flush_pte;
389 	int rc = 0, off;
390 
391 	if (hl_mem_area_inside_range(virt_addr, size,
392 			prop->dmmu.start_addr, prop->dmmu.end_addr))
393 		page_size = prop->dmmu.page_size;
394 	else if (hl_mem_area_inside_range(virt_addr, size,
395 			prop->pmmu.start_addr, prop->pmmu.end_addr))
396 		page_size = prop->pmmu.page_size;
397 	else if (hl_mem_area_inside_range(virt_addr, size,
398 			prop->pmmu_huge.start_addr, prop->pmmu_huge.end_addr))
399 		page_size = prop->pmmu_huge.page_size;
400 	else
401 		return -EINVAL;
402 
403 	for (off = 0 ; off < size ; off += page_size) {
404 		curr_va = virt_addr + off;
405 		curr_pa = phys_addr + off;
406 		flush_pte = (off + page_size) >= size;
407 		rc = hl_mmu_map_page(ctx, curr_va, curr_pa, page_size,
408 								flush_pte);
409 		if (rc) {
410 			dev_err(hdev->dev,
411 				"Map failed for va 0x%llx to pa 0x%llx\n",
412 				curr_va, curr_pa);
413 			/* last mapping failed so don't try to unmap it - reduce off by page_size */
414 			off -= page_size;
415 			goto unmap;
416 		}
417 	}
418 
419 	return rc;
420 
421 unmap:
422 	for (; off >= 0 ; off -= page_size) {
423 		curr_va = virt_addr + off;
424 		flush_pte = (off - (s32) page_size) < 0;
425 		if (hl_mmu_unmap_page(ctx, curr_va, page_size, flush_pte))
426 			dev_warn_ratelimited(hdev->dev,
427 				"failed to unmap va 0x%llx\n", curr_va);
428 	}
429 
430 	return rc;
431 }
432 
433 /*
434  * hl_mmu_unmap_contiguous - implements a wrapper for hl_mmu_unmap_page
435  *                           for unmapping contiguous physical memory
436  *
437  * @ctx: pointer to the context structure
438  * @virt_addr: virt addr to unmap
439  * @size: size to unmap
440  *
441  */
hl_mmu_unmap_contiguous(struct hl_ctx * ctx,u64 virt_addr,u32 size)442 int hl_mmu_unmap_contiguous(struct hl_ctx *ctx, u64 virt_addr, u32 size)
443 {
444 	struct hl_device *hdev = ctx->hdev;
445 	struct asic_fixed_properties *prop = &hdev->asic_prop;
446 	u64 curr_va;
447 	u32 page_size;
448 	bool flush_pte;
449 	int rc = 0, off;
450 
451 	if (hl_mem_area_inside_range(virt_addr, size,
452 			prop->dmmu.start_addr, prop->dmmu.end_addr))
453 		page_size = prop->dmmu.page_size;
454 	else if (hl_mem_area_inside_range(virt_addr, size,
455 			prop->pmmu.start_addr, prop->pmmu.end_addr))
456 		page_size = prop->pmmu.page_size;
457 	else if (hl_mem_area_inside_range(virt_addr, size,
458 			prop->pmmu_huge.start_addr, prop->pmmu_huge.end_addr))
459 		page_size = prop->pmmu_huge.page_size;
460 	else
461 		return -EINVAL;
462 
463 	for (off = 0 ; off < size ; off += page_size) {
464 		curr_va = virt_addr + off;
465 		flush_pte = (off + page_size) >= size;
466 		rc = hl_mmu_unmap_page(ctx, curr_va, page_size, flush_pte);
467 		if (rc)
468 			dev_warn_ratelimited(hdev->dev,
469 				"Unmap failed for va 0x%llx\n", curr_va);
470 	}
471 
472 	return rc;
473 }
474 
475 /*
476  * hl_mmu_swap_out - marks all mapping of the given ctx as swapped out
477  *
478  * @ctx: pointer to the context structure
479  *
480  */
hl_mmu_swap_out(struct hl_ctx * ctx)481 void hl_mmu_swap_out(struct hl_ctx *ctx)
482 {
483 	struct hl_device *hdev = ctx->hdev;
484 
485 	if (!hdev->mmu_enable)
486 		return;
487 
488 	if (hdev->mmu_func[MMU_DR_PGT].swap_out != NULL)
489 		hdev->mmu_func[MMU_DR_PGT].swap_out(ctx);
490 
491 	if (hdev->mmu_func[MMU_HR_PGT].swap_out != NULL)
492 		hdev->mmu_func[MMU_HR_PGT].swap_out(ctx);
493 }
494 
495 /*
496  * hl_mmu_swap_in - marks all mapping of the given ctx as swapped in
497  *
498  * @ctx: pointer to the context structure
499  *
500  */
hl_mmu_swap_in(struct hl_ctx * ctx)501 void hl_mmu_swap_in(struct hl_ctx *ctx)
502 {
503 	struct hl_device *hdev = ctx->hdev;
504 
505 	if (!hdev->mmu_enable)
506 		return;
507 
508 	if (hdev->mmu_func[MMU_DR_PGT].swap_in != NULL)
509 		hdev->mmu_func[MMU_DR_PGT].swap_in(ctx);
510 
511 	if (hdev->mmu_func[MMU_HR_PGT].swap_in != NULL)
512 		hdev->mmu_func[MMU_HR_PGT].swap_in(ctx);
513 }
514 
hl_mmu_pa_page_with_offset(struct hl_ctx * ctx,u64 virt_addr,struct hl_mmu_hop_info * hops,u64 * phys_addr)515 static void hl_mmu_pa_page_with_offset(struct hl_ctx *ctx, u64 virt_addr,
516 						struct hl_mmu_hop_info *hops,
517 						u64 *phys_addr)
518 {
519 	struct asic_fixed_properties *prop = &ctx->hdev->asic_prop;
520 	u64 offset_mask, addr_mask, hop_shift, tmp_phys_addr;
521 	struct hl_mmu_properties *mmu_prop;
522 
523 	/* last hop holds the phys address and flags */
524 	if (hops->unscrambled_paddr)
525 		tmp_phys_addr = hops->unscrambled_paddr;
526 	else
527 		tmp_phys_addr = hops->hop_info[hops->used_hops - 1].hop_pte_val;
528 
529 	if (hops->range_type == HL_VA_RANGE_TYPE_HOST_HUGE)
530 		mmu_prop = &prop->pmmu_huge;
531 	else if (hops->range_type == HL_VA_RANGE_TYPE_HOST)
532 		mmu_prop = &prop->pmmu;
533 	else /* HL_VA_RANGE_TYPE_DRAM */
534 		mmu_prop = &prop->dmmu;
535 
536 	if ((hops->range_type == HL_VA_RANGE_TYPE_DRAM) &&
537 			!is_power_of_2(prop->dram_page_size)) {
538 		u64 dram_page_size, dram_base, abs_phys_addr, abs_virt_addr,
539 			page_id, page_start;
540 		u32 page_off;
541 
542 		/*
543 		 * Bit arithmetics cannot be used for non power of two page
544 		 * sizes. In addition, since bit arithmetics is not used,
545 		 * we cannot ignore dram base. All that shall be considered.
546 		 */
547 
548 		dram_page_size = prop->dram_page_size;
549 		dram_base = prop->dram_base_address;
550 		abs_phys_addr = tmp_phys_addr - dram_base;
551 		abs_virt_addr = virt_addr - dram_base;
552 		page_id = DIV_ROUND_DOWN_ULL(abs_phys_addr, dram_page_size);
553 		page_start = page_id * dram_page_size;
554 		div_u64_rem(abs_virt_addr, dram_page_size, &page_off);
555 
556 		*phys_addr = page_start + page_off + dram_base;
557 	} else {
558 		/*
559 		 * find the correct hop shift field in hl_mmu_properties
560 		 * structure in order to determine the right masks
561 		 * for the page offset.
562 		 */
563 		hop_shift = mmu_prop->hop_shifts[hops->used_hops - 1];
564 		offset_mask = (1ull << hop_shift) - 1;
565 		addr_mask = ~(offset_mask);
566 		*phys_addr = (tmp_phys_addr & addr_mask) |
567 				(virt_addr & offset_mask);
568 	}
569 }
570 
hl_mmu_va_to_pa(struct hl_ctx * ctx,u64 virt_addr,u64 * phys_addr)571 int hl_mmu_va_to_pa(struct hl_ctx *ctx, u64 virt_addr, u64 *phys_addr)
572 {
573 	struct hl_mmu_hop_info hops;
574 	int rc;
575 
576 	memset(&hops, 0, sizeof(hops));
577 
578 	rc = hl_mmu_get_tlb_info(ctx, virt_addr, &hops);
579 	if (rc)
580 		return rc;
581 
582 	hl_mmu_pa_page_with_offset(ctx, virt_addr, &hops,  phys_addr);
583 
584 	return 0;
585 }
586 
hl_mmu_get_tlb_info(struct hl_ctx * ctx,u64 virt_addr,struct hl_mmu_hop_info * hops)587 int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr,
588 			struct hl_mmu_hop_info *hops)
589 {
590 	struct hl_device *hdev = ctx->hdev;
591 	struct asic_fixed_properties *prop;
592 	struct hl_mmu_properties *mmu_prop;
593 	struct hl_mmu_funcs *mmu_funcs;
594 	int pgt_residency, rc;
595 	bool is_dram_addr;
596 
597 	if (!hdev->mmu_enable)
598 		return -EOPNOTSUPP;
599 
600 	prop = &hdev->asic_prop;
601 	hops->scrambled_vaddr = virt_addr;      /* assume no scrambling */
602 
603 	is_dram_addr = hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size,
604 								prop->dmmu.start_addr,
605 								prop->dmmu.end_addr);
606 
607 	/* host-residency is the same in PMMU and PMMU huge, no need to distinguish here */
608 	mmu_prop = is_dram_addr ? &prop->dmmu : &prop->pmmu;
609 	pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT;
610 	mmu_funcs = hl_mmu_get_funcs(hdev, pgt_residency, is_dram_addr);
611 
612 	mutex_lock(&hdev->mmu_lock);
613 	rc = mmu_funcs->get_tlb_info(ctx, virt_addr, hops);
614 	mutex_unlock(&hdev->mmu_lock);
615 
616 	if (rc)
617 		return rc;
618 
619 	/* add page offset to physical address */
620 	if (hops->unscrambled_paddr)
621 		hl_mmu_pa_page_with_offset(ctx, virt_addr, hops, &hops->unscrambled_paddr);
622 
623 	return 0;
624 }
625 
hl_mmu_if_set_funcs(struct hl_device * hdev)626 int hl_mmu_if_set_funcs(struct hl_device *hdev)
627 {
628 	if (!hdev->mmu_enable)
629 		return 0;
630 
631 	switch (hdev->asic_type) {
632 	case ASIC_GOYA:
633 	case ASIC_GAUDI:
634 	case ASIC_GAUDI_SEC:
635 		hl_mmu_v1_set_funcs(hdev, &hdev->mmu_func[MMU_DR_PGT]);
636 		break;
637 	case ASIC_GAUDI2:
638 	case ASIC_GAUDI2_SEC:
639 		/* MMUs in Gaudi2 are always host resident */
640 		hl_mmu_v2_hr_set_funcs(hdev, &hdev->mmu_func[MMU_HR_PGT]);
641 		break;
642 	default:
643 		dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
644 			hdev->asic_type);
645 		return -EOPNOTSUPP;
646 	}
647 
648 	return 0;
649 }
650 
651 /**
652  * hl_mmu_scramble_addr() - The generic mmu address scrambling routine.
653  * @hdev: pointer to device data.
654  * @addr: The address to scramble.
655  *
656  * Return: The scrambled address.
657  */
hl_mmu_scramble_addr(struct hl_device * hdev,u64 addr)658 u64 hl_mmu_scramble_addr(struct hl_device *hdev, u64 addr)
659 {
660 	return addr;
661 }
662 
663 /**
664  * hl_mmu_descramble_addr() - The generic mmu address descrambling
665  * routine.
666  * @hdev: pointer to device data.
667  * @addr: The address to descramble.
668  *
669  * Return: The un-scrambled address.
670  */
hl_mmu_descramble_addr(struct hl_device * hdev,u64 addr)671 u64 hl_mmu_descramble_addr(struct hl_device *hdev, u64 addr)
672 {
673 	return addr;
674 }
675 
hl_mmu_invalidate_cache(struct hl_device * hdev,bool is_hard,u32 flags)676 int hl_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags)
677 {
678 	int rc;
679 
680 	rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags);
681 	if (rc)
682 		dev_err_ratelimited(hdev->dev, "MMU cache invalidation failed\n");
683 
684 	return rc;
685 }
686 
hl_mmu_invalidate_cache_range(struct hl_device * hdev,bool is_hard,u32 flags,u32 asid,u64 va,u64 size)687 int hl_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard,
688 					u32 flags, u32 asid, u64 va, u64 size)
689 {
690 	int rc;
691 
692 	rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, is_hard, flags,
693 								asid, va, size);
694 	if (rc)
695 		dev_err_ratelimited(hdev->dev, "MMU cache range invalidation failed\n");
696 
697 	return rc;
698 }
699 
hl_mmu_prefetch_work_function(struct work_struct * work)700 static void hl_mmu_prefetch_work_function(struct work_struct *work)
701 {
702 	struct hl_prefetch_work *pfw = container_of(work, struct hl_prefetch_work, pf_work);
703 	struct hl_ctx *ctx = pfw->ctx;
704 	struct hl_device *hdev = ctx->hdev;
705 
706 	if (!hl_device_operational(hdev, NULL))
707 		goto put_ctx;
708 
709 	mutex_lock(&hdev->mmu_lock);
710 
711 	hdev->asic_funcs->mmu_prefetch_cache_range(ctx, pfw->flags, pfw->asid, pfw->va, pfw->size);
712 
713 	mutex_unlock(&hdev->mmu_lock);
714 
715 put_ctx:
716 	/*
717 	 * context was taken in the common mmu prefetch function- see comment there about
718 	 * context handling.
719 	 */
720 	hl_ctx_put(ctx);
721 	kfree(pfw);
722 }
723 
hl_mmu_prefetch_cache_range(struct hl_ctx * ctx,u32 flags,u32 asid,u64 va,u64 size)724 int hl_mmu_prefetch_cache_range(struct hl_ctx *ctx, u32 flags, u32 asid, u64 va, u64 size)
725 {
726 	struct hl_prefetch_work *handle_pf_work;
727 
728 	handle_pf_work = kmalloc(sizeof(*handle_pf_work), GFP_KERNEL);
729 	if (!handle_pf_work)
730 		return -ENOMEM;
731 
732 	INIT_WORK(&handle_pf_work->pf_work, hl_mmu_prefetch_work_function);
733 	handle_pf_work->ctx = ctx;
734 	handle_pf_work->va = va;
735 	handle_pf_work->size = size;
736 	handle_pf_work->flags = flags;
737 	handle_pf_work->asid = asid;
738 
739 	/*
740 	 * as actual prefetch is done in a WQ we must get the context (and put it
741 	 * at the end of the work function)
742 	 */
743 	hl_ctx_get(ctx);
744 	queue_work(ctx->hdev->pf_wq, &handle_pf_work->pf_work);
745 
746 	return 0;
747 }
748 
hl_mmu_get_next_hop_addr(struct hl_ctx * ctx,u64 curr_pte)749 u64 hl_mmu_get_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte)
750 {
751 	return (curr_pte & PAGE_PRESENT_MASK) ? (curr_pte & HOP_PHYS_ADDR_MASK) : ULLONG_MAX;
752 }
753 
754 /**
755  * hl_mmu_get_hop_pte_phys_addr() - extract PTE address from HOP
756  * @ctx: pointer to the context structure to initialize.
757  * @mmu_prop: MMU properties.
758  * @hop_idx: HOP index.
759  * @hop_addr: HOP address.
760  * @virt_addr: virtual address fro the translation.
761  *
762  * @return the matching PTE value on success, otherwise U64_MAX.
763  */
hl_mmu_get_hop_pte_phys_addr(struct hl_ctx * ctx,struct hl_mmu_properties * mmu_prop,u8 hop_idx,u64 hop_addr,u64 virt_addr)764 u64 hl_mmu_get_hop_pte_phys_addr(struct hl_ctx *ctx, struct hl_mmu_properties *mmu_prop,
765 					u8 hop_idx, u64 hop_addr, u64 virt_addr)
766 {
767 	u64 mask, shift;
768 
769 	if (hop_idx >= mmu_prop->num_hops) {
770 		dev_err_ratelimited(ctx->hdev->dev, "Invalid hop index %d\n", hop_idx);
771 		return U64_MAX;
772 	}
773 
774 	shift = mmu_prop->hop_shifts[hop_idx];
775 	mask = mmu_prop->hop_masks[hop_idx];
776 
777 	return hop_addr + ctx->hdev->asic_prop.mmu_pte_size * ((virt_addr & mask) >> shift);
778 }
779 
mmu_dma_mem_free_from_chunk(struct gen_pool * pool,struct gen_pool_chunk * chunk,void * data)780 static void mmu_dma_mem_free_from_chunk(struct gen_pool *pool,
781 					struct gen_pool_chunk *chunk,
782 					void *data)
783 {
784 	struct hl_device *hdev = (struct hl_device *)data;
785 
786 	hl_asic_dma_free_coherent(hdev, (chunk->end_addr - chunk->start_addr) + 1,
787 					(void *)chunk->start_addr, chunk->phys_addr);
788 }
789 
hl_mmu_hr_flush(struct hl_ctx * ctx)790 void hl_mmu_hr_flush(struct hl_ctx *ctx)
791 {
792 	/* a flush operation requires memory barrier */
793 	mb();
794 }
795 
796 /**
797  * hl_mmu_hr_pool_destroy() - destroy genpool
798  * @hdev: habanalabs device structure.
799  * @hr_priv: MMU HR private data.
800  * @hop_table_size: HOP table size.
801  *
802  * This function does the following:
803  * - free entries allocated for shadow HOP0
804  * - free pool chunks
805  * - free pool
806  */
hl_mmu_hr_pool_destroy(struct hl_device * hdev,struct hl_mmu_hr_priv * hr_priv,u32 hop_table_size)807 static void hl_mmu_hr_pool_destroy(struct hl_device *hdev, struct hl_mmu_hr_priv *hr_priv,
808 					u32 hop_table_size)
809 {
810 	struct asic_fixed_properties *prop = &hdev->asic_prop;
811 	struct gen_pool **pool = &hr_priv->mmu_pgt_pool;
812 	struct pgt_info *hop0_pgt;
813 	int asid;
814 
815 	if (ZERO_OR_NULL_PTR(*pool))
816 		return;
817 
818 	/* Free the Fixed allocation of HOPs0 */
819 	if (hr_priv->mmu_asid_hop0) {
820 		for (asid = 0 ; asid < prop->max_asid ; asid++) {
821 			hop0_pgt = &hr_priv->mmu_asid_hop0[asid];
822 			if (ZERO_OR_NULL_PTR(hop0_pgt->virt_addr))
823 				continue;
824 
825 			gen_pool_free(*pool, (uintptr_t) hop0_pgt->virt_addr, hop_table_size);
826 		}
827 	}
828 
829 	gen_pool_for_each_chunk(*pool, mmu_dma_mem_free_from_chunk, hdev);
830 	gen_pool_destroy(*pool);
831 
832 	/* Make sure that if we arrive here again without init was called we
833 	 * won't cause kernel panic. This can happen for example if we fail
834 	 * during hard reset code at certain points
835 	 */
836 	*pool = NULL;
837 }
838 
839 /**
840  * hl_mmu_hr_init() - initialize the MMU module.
841  * @hdev: habanalabs device structure.
842  * @hr_priv: MMU HR private data.
843  * @hop_table_size: HOP table size.
844  * @pgt_size: memory size allocated for the page table
845  *
846  * @return 0 on success otherwise non-zero error code
847  *
848  * This function does the following:
849  * - Create a pool of pages for pgt_infos.
850  * - Create a shadow table for pgt
851  */
hl_mmu_hr_init(struct hl_device * hdev,struct hl_mmu_hr_priv * hr_priv,u32 hop_table_size,u64 pgt_size)852 int hl_mmu_hr_init(struct hl_device *hdev, struct hl_mmu_hr_priv *hr_priv, u32 hop_table_size,
853 			u64 pgt_size)
854 {
855 	struct asic_fixed_properties *prop = &hdev->asic_prop;
856 	size_t pool_chunk_size = SZ_4M;
857 	struct pgt_info *hop0_pgt;
858 	dma_addr_t dma_addr;
859 	u64 virt_addr;
860 	int i, rc;
861 
862 	/*
863 	 * we set alloc size as PAGE_SIZE (sine dma_alloc_coherent allocation order/size is
864 	 * PAGE_SHIFT/PAGE_SIZE) in order to be able to control the allocations alignment.
865 	 * This way we can call "DMA alloc align" according to dma_alloc granularity and supply
866 	 * allocations with higher-order alignment restrictions
867 	 */
868 	hr_priv->mmu_pgt_pool = gen_pool_create(PAGE_SHIFT, -1);
869 	if (ZERO_OR_NULL_PTR(hr_priv->mmu_pgt_pool)) {
870 		dev_err(hdev->dev, "Failed to create hr page pool\n");
871 		return -ENOMEM;
872 	}
873 
874 	hr_priv->mmu_asid_hop0 = kvcalloc(prop->max_asid, sizeof(struct pgt_info), GFP_KERNEL);
875 	if (ZERO_OR_NULL_PTR(hr_priv->mmu_asid_hop0)) {
876 		dev_err(hdev->dev, "Failed to allocate hr-mmu hop0 table\n");
877 		rc = -ENOMEM;
878 		goto destroy_mmu_pgt_pool;
879 	}
880 
881 	for (i = 0 ; i < pgt_size ; i += pool_chunk_size) {
882 		virt_addr = (uintptr_t) hl_asic_dma_alloc_coherent(hdev, pool_chunk_size,
883 									&dma_addr,
884 									GFP_KERNEL | __GFP_ZERO);
885 		if (ZERO_OR_NULL_PTR(virt_addr)) {
886 			dev_err(hdev->dev,
887 				"Failed to allocate memory for host-resident page pool\n");
888 			rc = -ENOMEM;
889 			goto destroy_mmu_pgt_pool;
890 		}
891 
892 		rc = gen_pool_add_virt(hr_priv->mmu_pgt_pool, virt_addr, (phys_addr_t) dma_addr,
893 						pool_chunk_size, -1);
894 		if (rc) {
895 			dev_err(hdev->dev, "Failed to fill host-resident page pool\n");
896 			goto destroy_mmu_pgt_pool;
897 		}
898 	}
899 
900 	for (i = 0 ; i < prop->max_asid ; i++) {
901 		hop0_pgt = &hr_priv->mmu_asid_hop0[i];
902 		hop0_pgt->virt_addr = (uintptr_t)
903 					gen_pool_dma_zalloc_align(hr_priv->mmu_pgt_pool,
904 								hop_table_size,
905 								(dma_addr_t *) &hop0_pgt->phys_addr,
906 								hop_table_size);
907 		if (!hop0_pgt->virt_addr) {
908 			dev_err(hdev->dev, "Failed to allocate HOP from pgt pool\n");
909 			rc = -ENOMEM;
910 			goto destroy_mmu_pgt_pool;
911 		}
912 	}
913 
914 	/* MMU H/W init will be done in device hw_init() */
915 
916 	return 0;
917 
918 destroy_mmu_pgt_pool:
919 	hl_mmu_hr_pool_destroy(hdev, hr_priv, hop_table_size);
920 	if (!ZERO_OR_NULL_PTR(hr_priv->mmu_asid_hop0))
921 		kvfree(hr_priv->mmu_asid_hop0);
922 
923 	return rc;
924 }
925 
926 /**
927  * hl_mmu_hr_fini() - release the MMU module.
928  * @hdev: habanalabs device structure.
929  * @hr_priv: MMU host resident private info.
930  * @hop_table_size: HOP table size
931  *
932  * This function does the following:
933  * - Disable MMU in H/W.
934  * - Free the pgt_infos pool.
935  *
936  * All contexts should be freed before calling this function.
937  */
hl_mmu_hr_fini(struct hl_device * hdev,struct hl_mmu_hr_priv * hr_priv,u32 hop_table_size)938 void hl_mmu_hr_fini(struct hl_device *hdev, struct hl_mmu_hr_priv *hr_priv, u32 hop_table_size)
939 {
940 	/* MMU H/W fini was already done in device hw_fini() */
941 
942 	hl_mmu_hr_pool_destroy(hdev, hr_priv, hop_table_size);
943 
944 	if (!ZERO_OR_NULL_PTR(hr_priv->mmu_asid_hop0)) {
945 		kvfree(hr_priv->mmu_asid_hop0);
946 
947 		/* Make sure that if we arrive here again without init was
948 		 * called we won't cause kernel panic. This can happen for
949 		 * example if we fail during hard reset code at certain points
950 		 */
951 		hr_priv->mmu_asid_hop0 = NULL;
952 	}
953 }
954 
955 /**
956  * hl_mmu_hr_free_hop_remove_pgt() - free HOP and remove PGT from hash
957  * @pgt_info: page table info structure.
958  * @hr_priv: MMU HR private data.
959  * @hop_table_size: HOP table size.
960  */
hl_mmu_hr_free_hop_remove_pgt(struct pgt_info * pgt_info,struct hl_mmu_hr_priv * hr_priv,u32 hop_table_size)961 void hl_mmu_hr_free_hop_remove_pgt(struct pgt_info *pgt_info, struct hl_mmu_hr_priv *hr_priv,
962 					u32 hop_table_size)
963 {
964 	gen_pool_free(hr_priv->mmu_pgt_pool, pgt_info->virt_addr, hop_table_size);
965 	hash_del(&pgt_info->node);
966 	kfree(pgt_info);
967 }
968 
969 /**
970  * hl_mmu_hr_pte_phys_to_virt() - translate PTE phys addr to virt addr
971  * @ctx: pointer to the context structure
972  * @pgt: pgt_info for the HOP hosting the PTE
973  * @phys_pte_addr: phys address of the PTE
974  * @hop_table_size: HOP table size
975  *
976  * @return PTE virtual address
977  *
978  * The function use the pgt_info to get HOP base virt addr and obtain the PTE's virt addr
979  * by adding the PTE offset.
980  */
hl_mmu_hr_pte_phys_to_virt(struct hl_ctx * ctx,struct pgt_info * pgt,u64 phys_pte_addr,u32 hop_table_size)981 u64 hl_mmu_hr_pte_phys_to_virt(struct hl_ctx *ctx, struct pgt_info *pgt,
982 							u64 phys_pte_addr, u32 hop_table_size)
983 {
984 	u64 page_mask = (hop_table_size - 1);
985 	u64 pte_offset = phys_pte_addr & page_mask;
986 
987 	return pgt->virt_addr + pte_offset;
988 }
989 
990 /**
991  * hl_mmu_hr_write_pte() - write HR PTE
992  * @ctx: pointer to the context structure
993  * @pgt_info: HOP's page table info structure
994  * @phys_pte_addr: phys PTE address
995  * @val: raw PTE data
996  * @hop_table_size: HOP table size
997  */
hl_mmu_hr_write_pte(struct hl_ctx * ctx,struct pgt_info * pgt_info,u64 phys_pte_addr,u64 val,u32 hop_table_size)998 void hl_mmu_hr_write_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr,
999 								u64 val, u32 hop_table_size)
1000 {
1001 	/*
1002 	 * The value to write is the phys address of the next hop +
1003 	 * flags at the 12 LSBs.
1004 	 */
1005 	u64 virt_addr = hl_mmu_hr_pte_phys_to_virt(ctx, pgt_info, phys_pte_addr, hop_table_size);
1006 
1007 	*((u64 *) (uintptr_t) virt_addr) = val;
1008 }
1009 
1010 /**
1011  * hl_mmu_hr_clear_pte() - clear HR PTE
1012  * @ctx: pointer to the context structure
1013  * @pgt_info: HOP's page table info structure
1014  * @phys_pte_addr: phys PTE address
1015  * @hop_table_size: HOP table size
1016  */
hl_mmu_hr_clear_pte(struct hl_ctx * ctx,struct pgt_info * pgt_info,u64 phys_pte_addr,u32 hop_table_size)1017 void hl_mmu_hr_clear_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr,
1018 						u32 hop_table_size)
1019 {
1020 	/* no need to transform the value to physical address */
1021 	hl_mmu_hr_write_pte(ctx, pgt_info, phys_pte_addr, 0, hop_table_size);
1022 }
1023 
1024 /**
1025  * hl_mmu_hr_put_pte() - put HR PTE and remove it if necessary (no more PTEs)
1026  * @ctx: pointer to the context structure
1027  * @pgt_info: HOP's page table info structure
1028  * @hr_priv: HR MMU private info
1029  * @hop_table_size: HOP table size
1030  *
1031  * @return number of PTEs still in the HOP
1032  */
hl_mmu_hr_put_pte(struct hl_ctx * ctx,struct pgt_info * pgt_info,struct hl_mmu_hr_priv * hr_priv,u32 hop_table_size)1033 int hl_mmu_hr_put_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info,
1034 						struct hl_mmu_hr_priv *hr_priv,
1035 						u32 hop_table_size)
1036 {
1037 	int num_of_ptes_left;
1038 
1039 	pgt_info->num_of_ptes--;
1040 
1041 	/*
1042 	 * Need to save the number of ptes left because free_hop might free
1043 	 * the pgt_info
1044 	 */
1045 	num_of_ptes_left = pgt_info->num_of_ptes;
1046 	if (!num_of_ptes_left)
1047 		hl_mmu_hr_free_hop_remove_pgt(pgt_info, hr_priv, hop_table_size);
1048 
1049 	return num_of_ptes_left;
1050 }
1051 
1052 /**
1053  * hl_mmu_hr_get_pte() - increase PGT PTE count
1054  * @ctx: pointer to the context structure
1055  * @hr_func: host resident functions
1056  * @phys_hop_addr: HOP phys address
1057  */
hl_mmu_hr_get_pte(struct hl_ctx * ctx,struct hl_hr_mmu_funcs * hr_func,u64 phys_hop_addr)1058 void hl_mmu_hr_get_pte(struct hl_ctx *ctx, struct hl_hr_mmu_funcs *hr_func, u64 phys_hop_addr)
1059 {
1060 	hr_func->get_pgt_info(ctx, phys_hop_addr)->num_of_ptes++;
1061 }
1062 
1063 /**
1064  * hl_mmu_hr_get_next_hop_pgt_info() - get pgt_info structure for the next HOP
1065  * @ctx: pointer to the context structure.
1066  * @hr_func: host resident functions.
1067  * @curr_pte: current PTE value.
1068  *
1069  * @return pgt_info structure on success, otherwise NULL.
1070  */
hl_mmu_hr_get_next_hop_pgt_info(struct hl_ctx * ctx,struct hl_hr_mmu_funcs * hr_func,u64 curr_pte)1071 struct pgt_info *hl_mmu_hr_get_next_hop_pgt_info(struct hl_ctx *ctx,
1072 							struct hl_hr_mmu_funcs *hr_func,
1073 							u64 curr_pte)
1074 {
1075 	u64 next_hop_phys_addr = hl_mmu_get_next_hop_addr(ctx, curr_pte);
1076 
1077 	if (next_hop_phys_addr == ULLONG_MAX)
1078 		return NULL;
1079 
1080 	return hr_func->get_pgt_info(ctx, next_hop_phys_addr);
1081 }
1082 
1083 /**
1084  * hl_mmu_hr_alloc_hop() - allocate HOP
1085  * @ctx: pointer to the context structure.
1086  * @hr_priv: host resident private info structure.
1087  * @hr_func: host resident functions.
1088  * @mmu_prop: MMU properties.
1089  *
1090  * @return pgt_info structure associated with the allocated HOP on success, otherwise NULL.
1091  */
hl_mmu_hr_alloc_hop(struct hl_ctx * ctx,struct hl_mmu_hr_priv * hr_priv,struct hl_hr_mmu_funcs * hr_func,struct hl_mmu_properties * mmu_prop)1092 struct pgt_info *hl_mmu_hr_alloc_hop(struct hl_ctx *ctx, struct hl_mmu_hr_priv *hr_priv,
1093 							struct hl_hr_mmu_funcs *hr_func,
1094 							struct hl_mmu_properties *mmu_prop)
1095 {
1096 	struct hl_device *hdev = ctx->hdev;
1097 	struct pgt_info *pgt_info;
1098 	dma_addr_t phys_addr;
1099 	void *virt_addr;
1100 	int i, retry = 1;
1101 
1102 	pgt_info = kmalloc(sizeof(*pgt_info), GFP_KERNEL);
1103 	if (!pgt_info)
1104 		return NULL;
1105 
1106 	for (i = 0; i <= retry; i++) {
1107 		virt_addr = gen_pool_dma_zalloc_align(hr_priv->mmu_pgt_pool,
1108 							mmu_prop->hop_table_size,
1109 							&phys_addr,
1110 							mmu_prop->hop_table_size);
1111 		if (virt_addr)
1112 			break;
1113 
1114 		/* No memory in pool - get some and try again */
1115 		virt_addr = hl_asic_dma_alloc_coherent(hdev, SZ_2M, &phys_addr,
1116 							GFP_KERNEL | __GFP_ZERO);
1117 		if (ZERO_OR_NULL_PTR(virt_addr))
1118 			break;
1119 
1120 		if (gen_pool_add_virt(hr_priv->mmu_pgt_pool, (unsigned long)virt_addr,
1121 								phys_addr, SZ_2M, -1)) {
1122 			hl_asic_dma_free_coherent(hdev, SZ_2M, virt_addr, phys_addr);
1123 			virt_addr = NULL;
1124 			break;
1125 		}
1126 	}
1127 
1128 	if (ZERO_OR_NULL_PTR(virt_addr)) {
1129 		dev_err(hdev->dev, "failed to allocate page\n");
1130 		goto pool_alloc_err;
1131 	}
1132 
1133 	pgt_info->phys_addr = phys_addr;
1134 	pgt_info->shadow_addr = (unsigned long) NULL;
1135 	pgt_info->virt_addr = (unsigned long)virt_addr;
1136 	pgt_info->ctx = ctx;
1137 	pgt_info->num_of_ptes = 0;
1138 	hr_func->add_pgt_info(ctx, pgt_info, phys_addr);
1139 
1140 	return pgt_info;
1141 
1142 pool_alloc_err:
1143 	kfree(pgt_info);
1144 
1145 	return NULL;
1146 }
1147 
1148 /**
1149  * hl_mmu_hr_get_alloc_next_hop() - get the next HOP, allocate it if it does not exist
1150  * @ctx: pointer to the context structure.
1151  * @hr_priv: host resident private info structure.
1152  * @hr_func: host resident functions.
1153  * @mmu_prop: MMU properties.
1154  * @curr_pte: current PTE value.
1155  * @is_new_hop: set to true if HOP is new (caller responsibility to set it to false).
1156  *
1157  * @return pgt_info structure associated with the allocated HOP on success, otherwise NULL.
1158  */
hl_mmu_hr_get_alloc_next_hop(struct hl_ctx * ctx,struct hl_mmu_hr_priv * hr_priv,struct hl_hr_mmu_funcs * hr_func,struct hl_mmu_properties * mmu_prop,u64 curr_pte,bool * is_new_hop)1159 struct pgt_info *hl_mmu_hr_get_alloc_next_hop(struct hl_ctx *ctx,
1160 							struct hl_mmu_hr_priv *hr_priv,
1161 							struct hl_hr_mmu_funcs *hr_func,
1162 							struct hl_mmu_properties *mmu_prop,
1163 							u64 curr_pte, bool *is_new_hop)
1164 {
1165 	u64 hop_addr = hl_mmu_get_next_hop_addr(ctx, curr_pte);
1166 
1167 	if (hop_addr != ULLONG_MAX)
1168 		return hr_func->get_pgt_info(ctx, hop_addr);
1169 
1170 	*is_new_hop = true;
1171 	return hl_mmu_hr_alloc_hop(ctx, hr_priv, hr_func, mmu_prop);
1172 }
1173 
1174 /**
1175  * hl_mmu_hr_get_tlb_info() - get the TLB info (info for a specific mapping)
1176  * @ctx: pointer to the context structure.
1177  * @virt_addr: the virt address for which to get info.
1178  * @hops: HOPs info structure.
1179  * @hr_func: host resident functions.
1180  *
1181  * @return 0 on success, otherwise non 0 error code..
1182  */
hl_mmu_hr_get_tlb_info(struct hl_ctx * ctx,u64 virt_addr,struct hl_mmu_hop_info * hops,struct hl_hr_mmu_funcs * hr_func)1183 int hl_mmu_hr_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops,
1184 								struct hl_hr_mmu_funcs *hr_func)
1185 {
1186 	/* using 6 HOPs as this is the maximum number of HOPs */
1187 	struct pgt_info *hops_pgt_info[MMU_ARCH_6_HOPS] = { NULL };
1188 	struct hl_device *hdev = ctx->hdev;
1189 	struct hl_mmu_properties *mmu_prop;
1190 	int rc, i, used_hops;
1191 	bool is_huge;
1192 
1193 	rc = hr_func->get_tlb_mapping_params(hdev, &mmu_prop, hops, virt_addr, &is_huge);
1194 	if (rc)
1195 		return rc;
1196 
1197 	used_hops = mmu_prop->num_hops;
1198 
1199 	/* huge pages use one less hop */
1200 	if (is_huge)
1201 		used_hops--;
1202 
1203 	hops->scrambled_vaddr = hdev->asic_funcs->scramble_addr(hdev, virt_addr);
1204 
1205 	for (i = 0 ; i < used_hops ; i++) {
1206 		if (i == 0)
1207 			hops_pgt_info[i] = hr_func->get_hop0_pgt_info(ctx);
1208 		else
1209 			hops_pgt_info[i] = hl_mmu_hr_get_next_hop_pgt_info(ctx, hr_func,
1210 								hops->hop_info[i - 1].hop_pte_val);
1211 
1212 		if (!hops_pgt_info[i])
1213 			return -EFAULT;
1214 
1215 		hops->hop_info[i].hop_addr = hops_pgt_info[i]->phys_addr;
1216 		hops->hop_info[i].hop_pte_addr =
1217 				hl_mmu_get_hop_pte_phys_addr(ctx, mmu_prop, i,
1218 								hops->hop_info[i].hop_addr,
1219 								hops->scrambled_vaddr);
1220 		hops->hop_info[i].hop_pte_val = *(u64 *) (uintptr_t)
1221 						hl_mmu_hr_pte_phys_to_virt(ctx, hops_pgt_info[i],
1222 								hops->hop_info[i].hop_pte_addr,
1223 								mmu_prop->hop_table_size);
1224 
1225 		if (!(hops->hop_info[i].hop_pte_val & PAGE_PRESENT_MASK))
1226 			return -EFAULT;
1227 
1228 		if (hops->hop_info[i].hop_pte_val & mmu_prop->last_mask)
1229 			break;
1230 	}
1231 
1232 	/* if passed over all hops then no last hop was found */
1233 	if (i == mmu_prop->num_hops)
1234 		return -EFAULT;
1235 
1236 	if (hops->scrambled_vaddr != virt_addr)
1237 		hops->unscrambled_paddr = hdev->asic_funcs->descramble_addr
1238 				(hdev, hops->hop_info[i].hop_pte_val);
1239 	else
1240 		hops->unscrambled_paddr = hops->hop_info[i].hop_pte_val;
1241 
1242 	hops->used_hops = i + 1;
1243 
1244 	return 0;
1245 }
1246 
1247