42 static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)  in ivpu_mmu_pgtable_init()  argument
46 	pgtable->pgd_dma_ptr = dma_alloc_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pgd_dma,  in ivpu_mmu_pgtable_init()
48 	if (!pgtable->pgd_dma_ptr)  in ivpu_mmu_pgtable_init()
51 	pgtable->pgd_dma = pgd_dma;  in ivpu_mmu_pgtable_init()
63 static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable)  in ivpu_mmu_pgtables_free()  argument
70 		pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];  in ivpu_mmu_pgtables_free()
71 		pud_dma = pgtable->pgd_dma_ptr[pgd_idx];  in ivpu_mmu_pgtables_free()
77 			pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];  in ivpu_mmu_pgtables_free()
78 			pmd_dma = pgtable->pud_ptrs[pgd_idx][pud_idx];  in ivpu_mmu_pgtables_free()
84 				pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];  in ivpu_mmu_pgtables_free()
85 				pte_dma = pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx];  in ivpu_mmu_pgtables_free()
90 			kfree(pgtable->pte_ptrs[pgd_idx][pud_idx]);  in ivpu_mmu_pgtables_free()
94 		kfree(pgtable->pmd_ptrs[pgd_idx]);  in ivpu_mmu_pgtables_free()
95 		kfree(pgtable->pte_ptrs[pgd_idx]);  in ivpu_mmu_pgtables_free()
99 	ivpu_mmu_pgtable_free(vdev, pgtable->pgd_dma_ptr, pgtable->pgd_dma);  in ivpu_mmu_pgtables_free()
103 ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx)  in ivpu_mmu_ensure_pud()  argument
105 	u64 *pud_dma_ptr = pgtable->pud_ptrs[pgd_idx];  in ivpu_mmu_ensure_pud()
115 	drm_WARN_ON(&vdev->drm, pgtable->pmd_ptrs[pgd_idx]);  in ivpu_mmu_ensure_pud()
116 	pgtable->pmd_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);  in ivpu_mmu_ensure_pud()
117 	if (!pgtable->pmd_ptrs[pgd_idx])  in ivpu_mmu_ensure_pud()
120 	drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx]);  in ivpu_mmu_ensure_pud()
121 	pgtable->pte_ptrs[pgd_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);  in ivpu_mmu_ensure_pud()
122 	if (!pgtable->pte_ptrs[pgd_idx])  in ivpu_mmu_ensure_pud()
125 	pgtable->pud_ptrs[pgd_idx] = pud_dma_ptr;  in ivpu_mmu_ensure_pud()
126 	pgtable->pgd_dma_ptr[pgd_idx] = pud_dma | IVPU_MMU_ENTRY_VALID;  in ivpu_mmu_ensure_pud()
131 	kfree(pgtable->pmd_ptrs[pgd_idx]);  in ivpu_mmu_ensure_pud()
139 ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx,  in ivpu_mmu_ensure_pmd()  argument
142 	u64 *pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx];  in ivpu_mmu_ensure_pmd()
152 	drm_WARN_ON(&vdev->drm, pgtable->pte_ptrs[pgd_idx][pud_idx]);  in ivpu_mmu_ensure_pmd()
153 	pgtable->pte_ptrs[pgd_idx][pud_idx] = kzalloc(IVPU_MMU_PGTABLE_SIZE, GFP_KERNEL);  in ivpu_mmu_ensure_pmd()
154 	if (!pgtable->pte_ptrs[pgd_idx][pud_idx])  in ivpu_mmu_ensure_pmd()
157 	pgtable->pmd_ptrs[pgd_idx][pud_idx] = pmd_dma_ptr;  in ivpu_mmu_ensure_pmd()
158 	pgtable->pud_ptrs[pgd_idx][pud_idx] = pmd_dma | IVPU_MMU_ENTRY_VALID;  in ivpu_mmu_ensure_pmd()
168 ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable,  in ivpu_mmu_ensure_pte()  argument
171 	u64 *pte_dma_ptr = pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx];  in ivpu_mmu_ensure_pte()
181 	pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma_ptr;  in ivpu_mmu_ensure_pte()
182 	pgtable->pmd_ptrs[pgd_idx][pud_idx][pmd_idx] = pte_dma | IVPU_MMU_ENTRY_VALID;  in ivpu_mmu_ensure_pte()
198 	if (!ivpu_mmu_ensure_pud(vdev, &ctx->pgtable, pgd_idx))  in ivpu_mmu_context_map_page()
202 	if (!ivpu_mmu_ensure_pmd(vdev, &ctx->pgtable, pgd_idx, pud_idx))  in ivpu_mmu_context_map_page()
206 	pte = ivpu_mmu_ensure_pte(vdev, &ctx->pgtable, pgd_idx, pud_idx, pmd_idx);  in ivpu_mmu_context_map_page()
249 	ctx->pgtable.pte_ptrs[pgd_idx][pud_idx][pmd_idx][pte_idx] = IVPU_MMU_ENTRY_INVALID;  in ivpu_mmu_context_unmap_page()
255 	struct ivpu_mmu_pgtable *pgtable = &ctx->pgtable;  in ivpu_mmu_context_flush_page_tables()  local
272 				clflush_cache_range(pgtable->pte_ptrs[pgd_idx][pud_idx][pmd_idx],  in ivpu_mmu_context_flush_page_tables()
276 			clflush_cache_range(pgtable->pmd_ptrs[pgd_idx][pud_idx],  in ivpu_mmu_context_flush_page_tables()
279 		clflush_cache_range(pgtable->pud_ptrs[pgd_idx], IVPU_MMU_PGTABLE_SIZE);  in ivpu_mmu_context_flush_page_tables()
281 	clflush_cache_range(pgtable->pgd_dma_ptr, IVPU_MMU_PGTABLE_SIZE);  in ivpu_mmu_context_flush_page_tables()
426 	ret = ivpu_mmu_pgtable_init(vdev, &ctx->pgtable);  in ivpu_mmu_context_init()
446 	if (drm_WARN_ON(&vdev->drm, !ctx->pgtable.pgd_dma_ptr))  in ivpu_mmu_context_fini()
450 	ivpu_mmu_pgtables_free(vdev, &ctx->pgtable);  in ivpu_mmu_context_fini()
453 	ctx->pgtable.pgd_dma_ptr = NULL;  in ivpu_mmu_context_fini()
454 	ctx->pgtable.pgd_dma = 0;  in ivpu_mmu_context_fini()
492 	ret = ivpu_mmu_set_pgtable(vdev, ctx_id, &ctx->pgtable);  in ivpu_mmu_user_context_init()