44 static inline bool is_large_pte(u64 pte)  in is_large_pte()  argument
46 	return (pte & IOMMU_PAGE_PSE);  in is_large_pte()
64 static inline void *get_pgtable_pte(u64 pte)  in get_pgtable_pte()  argument
66 	return iommu_phys_to_virt(pte & PM_ADDR_MASK);  in get_pgtable_pte()
71 	u64 pte;  in set_pte_attr()  local
73 	pte = __sme_set(paddr & PM_ADDR_MASK);  in set_pte_attr()
74 	pte |= IOMMU_PAGE_PRESENT | IOMMU_PAGE_USER;  in set_pte_attr()
75 	pte |= IOMMU_PAGE_ACCESS | IOMMU_PAGE_DIRTY;  in set_pte_attr()
78 		pte |= IOMMU_PAGE_RW;  in set_pte_attr()
82 		pte |= IOMMU_PAGE_PSE;  in set_pte_attr()
84 	return pte;  in set_pte_attr()
144 	u64 *pte, *page;  in v2_alloc_pte()  local
149 	pte = &pgd[PM_LEVEL_INDEX(level, iova)];  in v2_alloc_pte()
155 		__pte = *pte;  in v2_alloc_pte()
159 			cmpxchg64(pte, *pte, 0ULL);  in v2_alloc_pte()
171 			if (cmpxchg64(pte, __pte, __npte) != __pte)  in v2_alloc_pte()
180 		pte = get_pgtable_pte(__pte);  in v2_alloc_pte()
181 		pte = &pte[PM_LEVEL_INDEX(level, iova)];  in v2_alloc_pte()
185 	if (IOMMU_PTE_PRESENT(*pte)) {  in v2_alloc_pte()
189 		__pte = get_pgtable_pte(*pte);  in v2_alloc_pte()
190 		cmpxchg64(pte, *pte, 0ULL);  in v2_alloc_pte()
197 	return pte;  in v2_alloc_pte()
207 	u64 *pte;  in fetch_pte()  local
211 	pte = &pgtable->pgd[PM_LEVEL_INDEX(level, iova)];  in fetch_pte()
217 		if (!IOMMU_PTE_PRESENT(*pte))  in fetch_pte()
221 		pte = get_pgtable_pte(*pte);  in fetch_pte()
222 		pte = &pte[PM_LEVEL_INDEX(level - 1, iova)];  in fetch_pte()
225 		if (is_large_pte(*pte)) {  in fetch_pte()
239 	return pte;  in fetch_pte()
248 	u64 *pte;  in iommu_v2_map_pages()  local
265 		pte = v2_alloc_pte(pdom->iop.pgd, iova, map_size, &updated);  in iommu_v2_map_pages()
266 		if (!pte) {  in iommu_v2_map_pages()
271 		*pte = set_pte_attr(paddr, map_size, prot);  in iommu_v2_map_pages()
303 	u64 *pte;  in iommu_v2_unmap_pages()  local
309 		pte = fetch_pte(pgtable, iova, &unmap_size);  in iommu_v2_unmap_pages()
310 		if (!pte)  in iommu_v2_unmap_pages()
313 		*pte = 0ULL;  in iommu_v2_unmap_pages()
326 	u64 *pte, __pte;  in iommu_v2_iova_to_phys()  local
328 	pte = fetch_pte(pgtable, iova, &pte_pgsize);  in iommu_v2_iova_to_phys()
329 	if (!pte || !IOMMU_PTE_PRESENT(*pte))  in iommu_v2_iova_to_phys()
333 	__pte = __sme_clr(*pte & PM_ADDR_MASK);  in iommu_v2_iova_to_phys()